From 0c493cfa19e0cfc8d2a95ed34fe338b83a7816a1 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Wed, 17 Jun 2026 16:37:41 -0700 Subject: [PATCH] Add deprecated download dedupe cleanup (#2715) * feat: add deprecated download dedupe cleanup * chore: refresh vulnerable transitive overrides --- bun.lock | 109 +++++------------- convex/maintenance.test.ts | 181 +++++++++++++++++++++++++++++ convex/maintenance.ts | 225 +++++++++++++++++++++++++++++++++++++ package.json | 1 + 4 files changed, 435 insertions(+), 81 deletions(-) diff --git a/bun.lock b/bun.lock index 781bfb27..91829268 100644 --- a/bun.lock +++ b/bun.lock @@ -150,6 +150,7 @@ "overrides": { "ast-v8-to-istanbul": "1.0.4", "dompurify": "3.4.10", + "esbuild": "0.28.1", "next": "16.2.6", "postcss": "8.5.12", "ws": "8.21.0", @@ -237,57 +238,57 @@ "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w=="], - "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.1", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ=="], - "@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="], + "@esbuild/android-arm": ["@esbuild/android-arm@0.28.1", "", { "os": "android", "cpu": "arm" }, "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ=="], - "@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.0", "", { "os": "android", "cpu": "arm64" }, "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw=="], + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.1", "", { "os": "android", "cpu": "arm64" }, "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg=="], - "@esbuild/android-x64": ["@esbuild/android-x64@0.28.0", "", { "os": "android", "cpu": "x64" }, "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA=="], + "@esbuild/android-x64": ["@esbuild/android-x64@0.28.1", "", { "os": "android", "cpu": "x64" }, "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng=="], - "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q=="], + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q=="], - "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ=="], + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ=="], - "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q=="], + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw=="], - "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw=="], + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ=="], - "@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw=="], + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.1", "", { "os": "linux", "cpu": "arm" }, "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ=="], - "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A=="], + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g=="], - "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ=="], + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.1", "", { "os": "linux", "cpu": "ia32" }, "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w=="], - "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg=="], + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg=="], - "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w=="], + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ=="], - "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg=="], + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ=="], - "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ=="], + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ=="], - "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q=="], + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag=="], - "@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.0", "", { "os": "linux", "cpu": "x64" }, "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ=="], + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.1", "", { "os": "linux", "cpu": "x64" }, "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA=="], - "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw=="], + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw=="], - "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.0", "", { "os": "none", "cpu": "x64" }, "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw=="], + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.1", "", { "os": "none", "cpu": "x64" }, "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg=="], - "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g=="], + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.1", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q=="], - "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA=="], + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw=="], - "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w=="], + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg=="], - "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw=="], + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.1", "", { "os": "sunos", "cpu": "x64" }, "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ=="], - "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA=="], + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA=="], - "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA=="], + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg=="], - "@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw=="], + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.1", "", { "os": "win32", "cpu": "x64" }, "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A=="], "@exodus/bytes": ["@exodus/bytes@1.15.1", "", { "peerDependencies": { "@noble/hashes": "^1.8.0 || ^2.0.0" }, "optionalPeers": ["@noble/hashes"] }, "sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q=="], @@ -1177,7 +1178,7 @@ "es-toolkit": ["es-toolkit@1.47.0", "", {}, "sha512-n1GuoD0WEQZMBk5tttoZSqwgyLx01oqa5XsBmCHwPyNe1S9jPBEmtR2pSgp2kJuWE3ciFZ6yRHmY4pM4C3OOkw=="], - "esbuild": ["esbuild@0.28.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.0", "@esbuild/android-arm": "0.28.0", "@esbuild/android-arm64": "0.28.0", "@esbuild/android-x64": "0.28.0", "@esbuild/darwin-arm64": "0.28.0", "@esbuild/darwin-x64": "0.28.0", "@esbuild/freebsd-arm64": "0.28.0", "@esbuild/freebsd-x64": "0.28.0", "@esbuild/linux-arm": "0.28.0", "@esbuild/linux-arm64": "0.28.0", "@esbuild/linux-ia32": "0.28.0", "@esbuild/linux-loong64": "0.28.0", "@esbuild/linux-mips64el": "0.28.0", "@esbuild/linux-ppc64": "0.28.0", "@esbuild/linux-riscv64": "0.28.0", "@esbuild/linux-s390x": "0.28.0", "@esbuild/linux-x64": "0.28.0", "@esbuild/netbsd-arm64": "0.28.0", "@esbuild/netbsd-x64": "0.28.0", "@esbuild/openbsd-arm64": "0.28.0", "@esbuild/openbsd-x64": "0.28.0", "@esbuild/openharmony-arm64": "0.28.0", "@esbuild/sunos-x64": "0.28.0", "@esbuild/win32-arm64": "0.28.0", "@esbuild/win32-ia32": "0.28.0", "@esbuild/win32-x64": "0.28.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw=="], + "esbuild": ["esbuild@0.28.1", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.1", "@esbuild/android-arm": "0.28.1", "@esbuild/android-arm64": "0.28.1", "@esbuild/android-x64": "0.28.1", "@esbuild/darwin-arm64": "0.28.1", "@esbuild/darwin-x64": "0.28.1", "@esbuild/freebsd-arm64": "0.28.1", "@esbuild/freebsd-x64": "0.28.1", "@esbuild/linux-arm": "0.28.1", "@esbuild/linux-arm64": "0.28.1", "@esbuild/linux-ia32": "0.28.1", "@esbuild/linux-loong64": "0.28.1", "@esbuild/linux-mips64el": "0.28.1", "@esbuild/linux-ppc64": "0.28.1", "@esbuild/linux-riscv64": "0.28.1", "@esbuild/linux-s390x": "0.28.1", "@esbuild/linux-x64": "0.28.1", "@esbuild/netbsd-arm64": "0.28.1", "@esbuild/netbsd-x64": "0.28.1", "@esbuild/openbsd-arm64": "0.28.1", "@esbuild/openbsd-x64": "0.28.1", "@esbuild/openharmony-arm64": "0.28.1", "@esbuild/sunos-x64": "0.28.1", "@esbuild/win32-arm64": "0.28.1", "@esbuild/win32-ia32": "0.28.1", "@esbuild/win32-x64": "0.28.1" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw=="], "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], @@ -1909,8 +1910,6 @@ "conf/semver": ["semver@7.8.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-wnilbGyMxzbY7dNOl7jpKbLSjcfeweJWU5j4+u5qW+6/wuGD9KzIGOyZnQVSBM9E7DtWaaH3CyHkppYrKYoxwg=="], - "convex/esbuild": ["esbuild@0.27.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.0", "@esbuild/android-arm": "0.27.0", "@esbuild/android-arm64": "0.27.0", "@esbuild/android-x64": "0.27.0", "@esbuild/darwin-arm64": "0.27.0", "@esbuild/darwin-x64": "0.27.0", "@esbuild/freebsd-arm64": "0.27.0", "@esbuild/freebsd-x64": "0.27.0", "@esbuild/linux-arm": "0.27.0", "@esbuild/linux-arm64": "0.27.0", "@esbuild/linux-ia32": "0.27.0", "@esbuild/linux-loong64": "0.27.0", "@esbuild/linux-mips64el": "0.27.0", "@esbuild/linux-ppc64": "0.27.0", "@esbuild/linux-riscv64": "0.27.0", "@esbuild/linux-s390x": "0.27.0", "@esbuild/linux-x64": "0.27.0", "@esbuild/netbsd-arm64": "0.27.0", "@esbuild/netbsd-x64": "0.27.0", "@esbuild/openbsd-arm64": "0.27.0", "@esbuild/openbsd-x64": "0.27.0", "@esbuild/openharmony-arm64": "0.27.0", "@esbuild/sunos-x64": "0.27.0", "@esbuild/win32-arm64": "0.27.0", "@esbuild/win32-ia32": "0.27.0", "@esbuild/win32-x64": "0.27.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA=="], - "cytoscape-fcose/cose-base": ["cose-base@2.2.0", "", { "dependencies": { "layout-base": "^2.0.0" } }, "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g=="], "d3-dsv/commander": ["commander@7.2.0", "", {}, "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw=="], @@ -1965,58 +1964,6 @@ "accepts/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], - "convex/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.27.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A=="], - - "convex/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.27.0", "", { "os": "android", "cpu": "arm" }, "sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ=="], - - "convex/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.27.0", "", { "os": "android", "cpu": "arm64" }, "sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ=="], - - "convex/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.27.0", "", { "os": "android", "cpu": "x64" }, "sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q=="], - - "convex/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.27.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg=="], - - "convex/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.27.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g=="], - - "convex/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.27.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw=="], - - "convex/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.27.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g=="], - - "convex/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.27.0", "", { "os": "linux", "cpu": "arm" }, "sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ=="], - - "convex/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.27.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ=="], - - "convex/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.27.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw=="], - - "convex/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.27.0", "", { "os": "linux", "cpu": "none" }, "sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg=="], - - "convex/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.27.0", "", { "os": "linux", "cpu": "none" }, "sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg=="], - - "convex/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.27.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA=="], - - "convex/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.27.0", "", { "os": "linux", "cpu": "none" }, "sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ=="], - - "convex/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.27.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w=="], - - "convex/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.27.0", "", { "os": "linux", "cpu": "x64" }, "sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw=="], - - "convex/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.27.0", "", { "os": "none", "cpu": "arm64" }, "sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w=="], - - "convex/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.27.0", "", { "os": "none", "cpu": "x64" }, "sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA=="], - - "convex/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.27.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ=="], - - "convex/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.27.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A=="], - - "convex/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.27.0", "", { "os": "none", "cpu": "arm64" }, "sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA=="], - - "convex/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.27.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA=="], - - "convex/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.27.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg=="], - - "convex/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.27.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ=="], - - "convex/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.27.0", "", { "os": "win32", "cpu": "x64" }, "sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg=="], - "cytoscape-fcose/cose-base/layout-base": ["layout-base@2.0.1", "", {}, "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg=="], "d3-sankey/d3-array/internmap": ["internmap@1.0.1", "", {}, "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw=="], diff --git a/convex/maintenance.test.ts b/convex/maintenance.test.ts index 2fe932d5..bb8c4b47 100644 --- a/convex/maintenance.test.ts +++ b/convex/maintenance.test.ts @@ -35,6 +35,10 @@ vi.mock("./_generated/api", () => ({ nominateUserForEmptySkillSpamInternal: Symbol("nominateUserForEmptySkillSpamInternal"), cleanupEmptySkillsInternal: Symbol("cleanupEmptySkillsInternal"), nominateEmptySkillSpammersInternal: Symbol("nominateEmptySkillSpammersInternal"), + deleteDeprecatedDownloadDedupesBatchInternal: Symbol( + "deleteDeprecatedDownloadDedupesBatchInternal", + ), + cleanupDeprecatedDownloadDedupesInternal: Symbol("cleanupDeprecatedDownloadDedupesInternal"), }, skills: { backfillLatestSkillModerationInternal: Symbol("skills.backfillLatestSkillModerationInternal"), @@ -61,6 +65,8 @@ const { cleanupDeprecatedRateLimitShardsInternalHandler, cleanupEmptySkillsInternalHandler, deleteDeprecatedRateLimitShardsBatchInternalHandler, + cleanupDeprecatedDownloadDedupesInternalHandler, + deleteDeprecatedDownloadDedupesBatchInternalHandler, nominateEmptySkillSpammersInternalHandler, repairLegacyPublisherOwnershipForUserHandler, upsertSkillBadgeRecordInternal, @@ -1261,6 +1267,181 @@ describe("maintenance fingerprint backfill", () => { }); }); +describe("maintenance deprecated download dedupe cleanup", () => { + it("dry-runs one indexed batch without deleting rows", async () => { + const rows = [ + { _id: "downloadDedupes:1", hourStart: 100 }, + { _id: "downloadDedupes:2", hourStart: 200 }, + ]; + const take = vi.fn(async (limit: number) => rows.slice(0, limit)); + const order = vi.fn(() => ({ take })); + const withIndex = vi.fn(() => ({ order })); + const query = vi.fn(() => ({ withIndex })); + const deleteRow = vi.fn(); + + const result = await deleteDeprecatedDownloadDedupesBatchInternalHandler( + { + db: { + query, + delete: deleteRow, + }, + } as never, + { + dryRun: true, + batchSize: 10, + }, + ); + + expect(result).toEqual({ + ok: true, + dryRun: true, + scanned: 2, + deleted: 0, + isDone: true, + oldestHourStart: 100, + newestHourStart: 200, + }); + expect(query).toHaveBeenCalledWith("downloadDedupes"); + expect(withIndex).toHaveBeenCalledWith("by_hour", expect.any(Function)); + expect(order).toHaveBeenCalledWith("asc"); + expect(take).toHaveBeenCalledWith(10); + expect(deleteRow).not.toHaveBeenCalled(); + }); + + it("requires explicit table confirmation before deleting rows", async () => { + await expect( + deleteDeprecatedDownloadDedupesBatchInternalHandler( + { + db: { + query: vi.fn(), + delete: vi.fn(), + }, + } as never, + { + dryRun: false, + batchSize: 10, + }, + ), + ).rejects.toThrow('confirmTable="downloadDedupes"'); + }); + + it("deletes a confirmed indexed batch", async () => { + const rows = [ + { _id: "downloadDedupes:1", hourStart: 100 }, + { _id: "downloadDedupes:2", hourStart: 100 }, + ]; + const take = vi.fn(async (limit: number) => rows.slice(0, limit)); + const order = vi.fn(() => ({ take })); + const withIndex = vi.fn(() => ({ order })); + const query = vi.fn(() => ({ withIndex })); + const deleteRow = vi.fn(); + + const result = await deleteDeprecatedDownloadDedupesBatchInternalHandler( + { + db: { + query, + delete: deleteRow, + }, + } as never, + { + dryRun: false, + batchSize: 2, + confirmTable: "downloadDedupes", + }, + ); + + expect(result).toMatchObject({ + ok: true, + dryRun: false, + scanned: 2, + deleted: 2, + isDone: false, + oldestHourStart: 100, + newestHourStart: 100, + }); + expect(deleteRow).toHaveBeenCalledTimes(2); + expect(deleteRow).toHaveBeenNthCalledWith(1, "downloadDedupes:1"); + expect(deleteRow).toHaveBeenNthCalledWith(2, "downloadDedupes:2"); + }); + + it("runs exactly one dry-run batch even when maxBatches is larger", async () => { + const runMutation = vi.fn(async () => ({ + ok: true, + dryRun: true, + scanned: 1_000, + deleted: 0, + isDone: false, + oldestHourStart: 100, + newestHourStart: 200, + })); + + const result = await cleanupDeprecatedDownloadDedupesInternalHandler( + { runMutation, scheduler: { runAfter: vi.fn() } } as never, + { dryRun: true, batchSize: 1_000, maxBatches: 50 }, + ); + + expect(result).toMatchObject({ + ok: true, + dryRun: true, + batchSize: 1_000, + maxBatches: 1, + batches: 1, + scanned: 1_000, + deleted: 0, + isDone: false, + scheduledNext: false, + }); + expect(runMutation).toHaveBeenCalledTimes(1); + }); + + it("schedules a continuation when requested and the cleanup is incomplete", async () => { + const runMutation = vi.fn(async () => ({ + ok: true, + dryRun: false, + scanned: 1_000, + deleted: 1_000, + isDone: false, + oldestHourStart: 100, + newestHourStart: 200, + })); + const runAfter = vi.fn(); + + const result = await cleanupDeprecatedDownloadDedupesInternalHandler( + { runMutation, scheduler: { runAfter } } as never, + { + dryRun: false, + batchSize: 1_000, + maxBatches: 1, + continueOnIncomplete: true, + confirmTable: "downloadDedupes", + }, + ); + + expect(result).toMatchObject({ + ok: true, + dryRun: false, + batchSize: 1_000, + maxBatches: 1, + batches: 1, + scanned: 1_000, + deleted: 1_000, + isDone: false, + scheduledNext: true, + }); + expect(runAfter).toHaveBeenCalledWith( + 0, + internal.maintenance.cleanupDeprecatedDownloadDedupesInternal, + { + dryRun: false, + batchSize: 1_000, + maxBatches: 1, + continueOnIncomplete: true, + confirmTable: "downloadDedupes", + }, + ); + }); +}); + describe("maintenance empty skill cleanup", () => { it("dryRun detects empty skills and returns nominations", async () => { const runQuery = vi.fn().mockImplementation(async (endpoint: unknown) => { diff --git a/convex/maintenance.ts b/convex/maintenance.ts index 2dcc396e..cccefa09 100644 --- a/convex/maintenance.ts +++ b/convex/maintenance.ts @@ -34,6 +34,11 @@ const DEFAULT_DEPRECATED_RATE_LIMIT_SHARD_CLEANUP_BATCH_SIZE = 500; const MAX_DEPRECATED_RATE_LIMIT_SHARD_CLEANUP_BATCH_SIZE = 1_000; const DEFAULT_DEPRECATED_RATE_LIMIT_SHARD_CLEANUP_MAX_BATCHES = 1; const MAX_DEPRECATED_RATE_LIMIT_SHARD_CLEANUP_MAX_BATCHES = 100; +const DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_CONFIRM_TABLE = "downloadDedupes"; +const DEFAULT_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_BATCH_SIZE = 1_000; +const MAX_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_BATCH_SIZE = 2_000; +const DEFAULT_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_MAX_BATCHES = 1; +const MAX_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_MAX_BATCHES = 100; const DEFAULT_EMPTY_SKILL_MAX_README_BYTES = 8000; const DEFAULT_EMPTY_SKILL_NOMINATION_THRESHOLD = 3; const PLATFORM_SKILL_LICENSE = "MIT-0" as const; @@ -91,6 +96,35 @@ type PublisherStatsBackfillPageResult = { isDone: boolean; }; +type DeprecatedDownloadDedupeCleanupBatchResult = { + ok: true; + dryRun: boolean; + scanned: number; + deleted: number; + isDone: boolean; + oldestHourStart: number | null; + newestHourStart: number | null; +}; + +type DeprecatedDownloadDedupeCleanupBatchTiming = DeprecatedDownloadDedupeCleanupBatchResult & { + batch: number; + elapsedMs: number; +}; + +type DeprecatedDownloadDedupeCleanupResult = { + ok: true; + dryRun: boolean; + batchSize: number; + maxBatches: number; + batches: number; + scanned: number; + deleted: number; + isDone: boolean; + scheduledNext: boolean; + elapsedMs: number; + batchTimings: DeprecatedDownloadDedupeCleanupBatchTiming[]; +}; + type UserOwnedSkillsBackfillPageResult = { items: Array, "stats" | "softDeletedAt">>; cursor: string | null; @@ -518,6 +552,197 @@ export type PublisherStatsBackfillActionResult = { cursor: string | null; }; +export type DeprecatedDownloadDedupesCleanupActionArgs = { + dryRun?: boolean; + batchSize?: number; + maxBatches?: number; + continueOnIncomplete?: boolean; + confirmTable?: string; +}; + +export async function deleteDeprecatedDownloadDedupesBatchInternalHandler( + ctx: Pick, + args: { + dryRun?: boolean; + batchSize?: number; + confirmTable?: string; + }, +): Promise { + const dryRun = Boolean(args.dryRun); + if (!dryRun && args.confirmTable !== DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_CONFIRM_TABLE) { + throw new ConvexError( + `Refusing to delete deprecated download dedupe rows without confirmTable="${DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_CONFIRM_TABLE}"`, + ); + } + + const batchSize = clampInt( + args.batchSize ?? DEFAULT_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_BATCH_SIZE, + 1, + MAX_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_BATCH_SIZE, + ); + const rows = await ctx.db + .query("downloadDedupes") + .withIndex("by_hour", (q) => q) + .order("asc") + .take(batchSize); + + if (!dryRun) { + for (const row of rows) { + await ctx.db.delete(row._id); + } + } + + return { + ok: true as const, + dryRun, + scanned: rows.length, + deleted: dryRun ? 0 : rows.length, + isDone: rows.length < batchSize, + oldestHourStart: rows[0]?.hourStart ?? null, + newestHourStart: rows.at(-1)?.hourStart ?? null, + }; +} + +export const deleteDeprecatedDownloadDedupesBatchInternal = internalMutation({ + args: { + dryRun: v.optional(v.boolean()), + batchSize: v.optional(v.number()), + confirmTable: v.optional(v.string()), + }, + handler: deleteDeprecatedDownloadDedupesBatchInternalHandler, +}); + +export async function cleanupDeprecatedDownloadDedupesInternalHandler( + ctx: ActionCtx, + args: DeprecatedDownloadDedupesCleanupActionArgs, +): Promise { + const dryRun = Boolean(args.dryRun); + if (!dryRun && args.confirmTable !== DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_CONFIRM_TABLE) { + throw new ConvexError( + `Refusing to delete deprecated download dedupe rows without confirmTable="${DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_CONFIRM_TABLE}"`, + ); + } + + const batchSize = clampInt( + args.batchSize ?? DEFAULT_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_BATCH_SIZE, + 1, + MAX_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_BATCH_SIZE, + ); + const maxBatches = dryRun + ? 1 + : clampInt( + args.maxBatches ?? DEFAULT_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_MAX_BATCHES, + 1, + MAX_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_MAX_BATCHES, + ); + const startedAt = Date.now(); + const batchTimings: DeprecatedDownloadDedupeCleanupBatchTiming[] = []; + let scanned = 0; + let deleted = 0; + let isDone = false; + + for (let batch = 1; batch <= maxBatches; batch += 1) { + const batchStartedAt = Date.now(); + const result = (await ctx.runMutation( + internal.maintenance.deleteDeprecatedDownloadDedupesBatchInternal, + { + dryRun, + batchSize, + confirmTable: args.confirmTable, + }, + )) as DeprecatedDownloadDedupeCleanupBatchResult; + const timing = { + ...result, + batch, + elapsedMs: Date.now() - batchStartedAt, + }; + batchTimings.push(timing); + scanned += result.scanned; + deleted += result.deleted; + isDone = result.isDone; + if (result.isDone || result.scanned === 0) break; + } + + const shouldContinue = !dryRun && !isDone && Boolean(args.continueOnIncomplete); + if (shouldContinue) { + await ctx.scheduler.runAfter(0, internal.maintenance.cleanupDeprecatedDownloadDedupesInternal, { + dryRun: false, + batchSize, + maxBatches, + continueOnIncomplete: true, + confirmTable: DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_CONFIRM_TABLE, + }); + } + + return { + ok: true as const, + dryRun, + batchSize, + maxBatches, + batches: batchTimings.length, + scanned, + deleted, + isDone, + scheduledNext: shouldContinue, + elapsedMs: Date.now() - startedAt, + batchTimings, + }; +} + +export const cleanupDeprecatedDownloadDedupesInternal = internalAction({ + args: { + dryRun: v.optional(v.boolean()), + batchSize: v.optional(v.number()), + maxBatches: v.optional(v.number()), + continueOnIncomplete: v.optional(v.boolean()), + confirmTable: v.optional(v.string()), + }, + handler: cleanupDeprecatedDownloadDedupesInternalHandler, +}); + +export const cleanupDeprecatedDownloadDedupes: ReturnType = action({ + args: { + dryRun: v.optional(v.boolean()), + batchSize: v.optional(v.number()), + maxBatches: v.optional(v.number()), + continueOnIncomplete: v.optional(v.boolean()), + confirmTable: v.optional(v.string()), + }, + handler: async (ctx, args): Promise => { + const { user } = await requireUserFromAction(ctx); + assertRole(user, ["admin"]); + return ctx.runAction( + internal.maintenance.cleanupDeprecatedDownloadDedupesInternal, + args, + ) as Promise; + }, +}); + +export const scheduleDeprecatedDownloadDedupesCleanup: ReturnType = action({ + args: { + batchSize: v.optional(v.number()), + maxBatches: v.optional(v.number()), + confirmTable: v.string(), + }, + handler: async (ctx, args) => { + const { user } = await requireUserFromAction(ctx); + assertRole(user, ["admin"]); + if (args.confirmTable !== DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_CONFIRM_TABLE) { + throw new ConvexError( + `Refusing to schedule deprecated download dedupe cleanup without confirmTable="${DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_CONFIRM_TABLE}"`, + ); + } + await ctx.scheduler.runAfter(0, internal.maintenance.cleanupDeprecatedDownloadDedupesInternal, { + dryRun: false, + batchSize: args.batchSize ?? DEFAULT_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_BATCH_SIZE, + maxBatches: args.maxBatches ?? DEFAULT_DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_MAX_BATCHES, + continueOnIncomplete: true, + confirmTable: DEPRECATED_DOWNLOAD_DEDUPE_CLEANUP_CONFIRM_TABLE, + }); + return { ok: true as const }; + }, +}); + export async function backfillSkillSummariesInternalHandler( ctx: ActionCtx, args: BackfillActionArgs, diff --git a/package.json b/package.json index f348690b..fd21977d 100644 --- a/package.json +++ b/package.json @@ -154,6 +154,7 @@ "overrides": { "ast-v8-to-istanbul": "1.0.4", "dompurify": "3.4.10", + "esbuild": "0.28.1", "next": "16.2.6", "postcss": "8.5.12", "ws": "8.21.0"