From 1148023b1afba57cc9ed8f8be0a12287fa16d357 Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Wed, 1 Jan 2025 11:25:05 +0000 Subject: [PATCH] feat(api): ratelimiting --- api/middleware/ratelimit.ts | 17 +++++ api/routes/single-page.ts | 136 ++++++++++++---------------------- api/tsconfig.json | 7 +- api/worker-configuration.d.ts | 6 ++ biome.json | 8 +- docs/.vitepress/README.md | 3 + nitro.config.ts | 6 +- package.json | 3 + pnpm-lock.yaml | 93 +++++++++-------------- wrangler.toml | 10 ++- 10 files changed, 139 insertions(+), 150 deletions(-) create mode 100644 api/middleware/ratelimit.ts create mode 100644 api/worker-configuration.d.ts diff --git a/api/middleware/ratelimit.ts b/api/middleware/ratelimit.ts new file mode 100644 index 000000000..200319b49 --- /dev/null +++ b/api/middleware/ratelimit.ts @@ -0,0 +1,17 @@ +export default defineEventHandler(async (event) => { + const { cloudflare } = event.context + + // FIXME: THIS IS NOT RECOMMENDED. BUT I WILL USE IT FOR NOW + // Not recommended: many users may share a single IP, especially on mobile networks + // or when using privacy-enabling proxies + const ipAddress = getHeader(event, 'CF-Connecting-IP') ?? '' + + const { success } = await // KILL YOURSELF + (cloudflare.env as unknown as Env).RATE_LIMITER.limit({ + key: ipAddress + }) + + if (!success) { + throw createError('Failure – global rate limit exceeded') + } +}) diff --git a/api/routes/single-page.ts b/api/routes/single-page.ts index 5ca8105bf..b36e305d7 100644 --- a/api/routes/single-page.ts +++ b/api/routes/single-page.ts @@ -13,99 +13,61 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +const files = ( + [ + 'adblockvpnguide.md', + 'ai.md', + 'android-iosguide.md', + 'audiopiracyguide.md', + 'beginners-guide.md', + 'devtools.md', + 'downloadpiracyguide.md', + 'edupiracyguide.md', + 'file-tools.md', + 'gaming-tools.md', + 'gamingpiracyguide.md', + 'img-tools.md', + 'internet-tools.md', + 'linuxguide.md', + 'miscguide.md', + 'non-english.md', + 'readingpiracyguide.md', + 'social-media-tools.md', + 'storage.md', + 'system-tools.md', + 'text-tools.md', + 'torrentpiracyguide.md', + 'unsafesites.md', + 'video-tools.md', + 'videopiracyguide.md' + ] as const +).map((file) => ({ + name: file, + url: `https://raw.githubusercontent.com/fmhy/edit/main/docs/${file}` +})) -import { fetcher } from 'itty-fetcher' -import { createStorage } from 'unstorage' -import cloudflareKVBindingDriver from 'unstorage/drivers/cloudflare-kv-binding' +export default defineCachedEventHandler( + async (event) => { + let body = '\n' -// Look inside the docs directory -const GITHUB_REPO = 'https://api.github.com/repos/fmhy/edit/contents/docs/' -const EXCLUDE_FILES = [ - 'README.md', - 'index.md', - 'feedback.md', - 'posts.md', - 'sandbox.md' -] -const EXCLUDE_DIRECTORIES = ['posts/'] - -interface File { - name: string - path: string - sha: string - size: number - url: string - html_url: string - git_url: string - download_url: string | null - type: string - _links: { - self: string - git: string - html: string - } -} - -export default defineEventHandler(async (event) => { - const markdownStorage = createStorage({ - driver: cloudflareKVBindingDriver({ binding: 'STORAGE' }) - }) - - let body = '\n' - const f = fetcher({ - headers: { - 'User-Agent': 'taskylizard' - } - }) - - try { - // Fetch the list of files in the repository - const indexCacheKey = "INDEX" - let files = await markdownStorage.getItem(indexCacheKey) - - if (!files) { - files = await f.get(GITHUB_REPO) - await markdownStorage.setItem(indexCacheKey, files, { ttl: 60 * 60 * 24 * 7 }) - } - - // Filter out the excluded files and non-markdown files - const markdownFiles = files.filter((file: File) => { - const isExcludedFile = EXCLUDE_FILES.includes(file.name) - const isInExcludedDirectory = EXCLUDE_DIRECTORIES.some((dir) => - file.path.startsWith(dir) - ) - const isMarkdownFile = file.name.endsWith('.md') - - return isMarkdownFile && !isExcludedFile && !isInExcludedDirectory - }) - - // Fetch and concatenate the contents of the markdown files with caching const contents = await Promise.all( - markdownFiles.map(async (file: File) => { - const cached = await markdownStorage.getItem(file.name) - if (cached) return cached - - const content = await f.get(file.download_url) - if (content) { - await markdownStorage.setItem(file.name, content, { ttl: 60 * 60 }) - } + files.map(async (file) => { + const content = await $fetch(file.url) return content }) ) - body += contents.join('\n\n') - } catch (error) { - return { - status: 500, - body: `Error fetching markdown files: ${error.message}` - } - } - // biome-ignore lint/correctness/noUndeclaredVariables: - appendResponseHeaders(event, { - 'content-type': 'text/markdown;charset=utf-8', - 'cache-control': 'public, max-age=3600' - }) - return body -}) + appendResponseHeaders(event, { + 'content-type': 'text/markdown;charset=utf-8', + 'cache-control': 'public, max-age=7200' + }) + return body + }, + { + maxAge: 60 * 60, + name: 'single-page', + getKey: () => 'default' /* Can be extended in the future */ + } +) diff --git a/api/tsconfig.json b/api/tsconfig.json index e7468d88c..ab45f2118 100644 --- a/api/tsconfig.json +++ b/api/tsconfig.json @@ -1,3 +1,8 @@ { - "extends": "../.nitro/types/tsconfig.json" + "extends": "../.nitro/types/tsconfig.json", + "compilerOptions": { + "types": [ + "@cloudflare/workers-types" + ] + } } diff --git a/api/worker-configuration.d.ts b/api/worker-configuration.d.ts new file mode 100644 index 000000000..b6e97d8f0 --- /dev/null +++ b/api/worker-configuration.d.ts @@ -0,0 +1,6 @@ +// Generated by Wrangler by running `wrangler types api/worker-configuration.d.ts` + +interface Env { + STORAGE: KVNamespace; + RATE_LIMITER: RateLimit; +} diff --git a/biome.json b/biome.json index 1c5184365..352c3f444 100644 --- a/biome.json +++ b/biome.json @@ -1,6 +1,9 @@ { "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json", - "extends": ["@taskylizard/biome-config", "./.cache/imports.json"], + "extends": [ + "@taskylizard/biome-config", + "./.cache/imports.json" + ], "files": { "ignore": [ "docs/.vitepress/**/*.vue", @@ -42,6 +45,9 @@ }, "linter": { "rules": { + "correctness": { + "noUndeclaredVariables": "off" + }, "style": { "useFilenamingConvention": "off", "noDefaultExport": "off" diff --git a/docs/.vitepress/README.md b/docs/.vitepress/README.md index e2da1674c..86b0515a4 100644 --- a/docs/.vitepress/README.md +++ b/docs/.vitepress/README.md @@ -1,3 +1,6 @@ +> [!NOTE] +> The website is no longer getting new features. It is now in maintenance mode. Please do not open issues or PRs. + This is the website source code to be used with [VitePress](https://vitepress.dev/). Licensed under the Apache License v2.0, see [LICENSE](./LICENSE) for more information. diff --git a/nitro.config.ts b/nitro.config.ts index e35947e88..05405ed78 100644 --- a/nitro.config.ts +++ b/nitro.config.ts @@ -13,8 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -//https://nitro.unjs.io/config + +import nitroCloudflareBindings from 'nitro-cloudflare-dev' +import { defineNitroConfig } from 'nitropack/config' + export default defineNitroConfig({ + modules: [nitroCloudflareBindings], preset: 'cloudflare_module', compatibilityDate: '2024-11-01', runtimeConfig: { diff --git a/package.json b/package.json index 709c1a8f8..7b17345ee 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "api:dev": "nitropack dev", "api:prepare": "nitropack prepare", "api:preview": "node .output/server/index.mjs", + "api:typegen": "wrangler types api/worker-configuration.d.ts", "docs:build": "vitepress build docs/", "docs:dev": "vitepress dev docs/", "docs:preview": "vitepress preview docs/", @@ -41,6 +42,7 @@ }, "devDependencies": { "@biomejs/biome": "^1.9.3", + "@cloudflare/workers-types": "^4.20241230.0", "@ianvs/prettier-plugin-sort-imports": "^4.3.1", "@iconify-json/carbon": "^1.2.3", "@iconify-json/heroicons-solid": "^1.2.0", @@ -51,6 +53,7 @@ "@taskylizard/biome-config": "^1.0.5", "@types/node": "^20.16.12", "@types/nprogress": "^0.2.3", + "nitro-cloudflare-dev": "^0.2.1", "prettier": "^3.3.3", "prettier-plugin-pkgsort": "^0.2.1", "prettier-plugin-tailwindcss": "^0.6.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2fd7174e4..72ba6c1d2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: consola: specifier: ^3.2.3 version: 3.2.3 + express-rate-limit: + specifier: ^7.5.0 + version: 7.5.0 feed: specifier: ^4.2.2 version: 4.2.2 @@ -60,6 +63,9 @@ importers: '@biomejs/biome': specifier: ^1.9.3 version: 1.9.3 + '@cloudflare/workers-types': + specifier: ^4.20241230.0 + version: 4.20241230.0 '@ianvs/prettier-plugin-sort-imports': specifier: ^4.3.1 version: 4.3.1(@vue/compiler-sfc@3.5.12)(prettier@3.3.3) @@ -90,6 +96,9 @@ importers: '@types/nprogress': specifier: ^0.2.3 version: 0.2.3 + nitro-cloudflare-dev: + specifier: ^0.2.1 + version: 0.2.1 prettier: specifier: ^3.3.3 version: 3.3.3 @@ -122,7 +131,7 @@ importers: version: 1.2.0(rollup@4.29.1)(vite@5.4.11(@types/node@20.16.12)(sass@1.80.1)(terser@5.34.1)) wrangler: specifier: ^3.99.0 - version: 3.99.0 + version: 3.99.0(@cloudflare/workers-types@4.20241230.0) packages: @@ -459,6 +468,9 @@ packages: cpu: [x64] os: [win32] + '@cloudflare/workers-types@4.20241230.0': + resolution: {integrity: sha512-dtLD4jY35Lb750cCVyO1i/eIfdZJg2Z0i+B1RYX6BVeRPlgaHx/H18ImKAkYmy0g09Ow8R2jZy3hIxMgXun0WQ==} + '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -2154,9 +2166,6 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} - cookie-es@1.0.0: - resolution: {integrity: sha512-mWYvfOLrfEc996hlKcdABeIiPHUPC6DM2QYZdGGOvhOTbA3tjm2eBwqlJpoFdjC89NI4Qt6h0Pu06Mp+1Pj5OQ==} - cookie-es@1.2.2: resolution: {integrity: sha512-+W7VmiVINB+ywl1HGXJXmrqkOhpKrIiVZV6tQuV54ZyQC7MMuBt81Vc336GMLoHBq5hV/F9eXgt5Mnx0Rha5Fg==} @@ -2315,9 +2324,6 @@ packages: resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} engines: {node: '>=8'} - defu@6.1.3: - resolution: {integrity: sha512-Vy2wmG3NTkmHNg/kzpuvHhkqeIx3ODWqasgCRbKtbXEN0G+HpEEv9BtJLp7ZG1CZloFaC41Ah3ZFbq7aqCqMeQ==} - defu@6.1.4: resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} @@ -2333,9 +2339,6 @@ packages: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} - destr@2.0.2: - resolution: {integrity: sha512-65AlobnZMiCET00KaFFjUefxDX0khFA/E4myqZ7a6Sq1yZtR8+FVIvilVX66vF2uobSumxooYZChiRPCKNqhmg==} - destr@2.0.3: resolution: {integrity: sha512-2N3BOUU4gYMpTP24s5rF5iP7BDr7uNTCs4ozw3kf/eKfvWSIu93GEBi5m427YoyJoeOzQ5smuu4nNAPGb8idSQ==} @@ -2485,6 +2488,12 @@ packages: resolution: {integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==} engines: {node: '>=6'} + express-rate-limit@7.5.0: + resolution: {integrity: sha512-eB5zbQh5h+VenMPM3fh+nw1YExi5nMr6HUCR62ELSP11huvxm/Uir1H1QEyTkk5QX6A58pX6NmaTMceKZ0Eodg==} + engines: {node: '>= 16'} + peerDependencies: + express: ^4.11 || 5 || ^5.0.0-beta.1 + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2653,9 +2662,6 @@ packages: h3@1.13.0: resolution: {integrity: sha512-vFEAu/yf8UMUcB4s43OaDaigcqpQd14yanmOsn+NcRX3/guSKncyE2rOYhq8RIchgJrPSs/QiIddnTTR1ddiAg==} - h3@1.8.2: - resolution: {integrity: sha512-1Ca0orJJlCaiFY68BvzQtP2lKLk46kcLAxVM8JgYbtm2cUg6IY7pjpYgWMwUvDO9QI30N5JAukOKoT8KD3Q0PQ==} - has-flag@3.0.0: resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} engines: {node: '>=4'} @@ -2749,9 +2755,6 @@ packages: resolution: {integrity: sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==} engines: {node: '>=12.22.0'} - iron-webcrypto@0.10.1: - resolution: {integrity: sha512-QGOS8MRMnj/UiOa+aMIgfyHcvkhqNUsUxb1XzskENvbo+rEfp6TOwqd1KPuDzXC4OnGHcMSVxDGRoilqB8ViqA==} - iron-webcrypto@1.2.1: resolution: {integrity: sha512-feOM6FaSr6rEABp/eDfVseKyTMDt+KGpeB35SkVn9Tyn0CqvVsY3EwI0v5i8nMHyJnzCIQf7nsy3p41TPkJZhg==} @@ -3126,6 +3129,9 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + nitro-cloudflare-dev@0.2.1: + resolution: {integrity: sha512-zHAN21dp+As0ldkAr5tWTop/I721j7MssZG6qb7a7EMorFwdRIhyTUwltr2L6v4qT4209S4eb2S9rszP1fxS7A==} + nitro-cors@0.7.1: resolution: {integrity: sha512-c/3d6L2vsGWtCdCwxbiItmnxTQZFE4+iUclvC7q4QBEEwPefBPmxCNiUNgNvtNmPhFkTmUf7LVfMeByvuv+6Ow==} @@ -3436,9 +3442,6 @@ packages: queue-tick@1.0.1: resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} - radix3@1.1.0: - resolution: {integrity: sha512-pNsHDxbGORSvuSScqNJ+3Km6QAVqk8CfsCBIEoDgpqLrkD2f3QM4I7d1ozJJ172OmIcoUcerZaNWqtLkRXTV3A==} - radix3@1.1.2: resolution: {integrity: sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA==} @@ -3878,9 +3881,6 @@ packages: engines: {node: '>=14.17'} hasBin: true - ufo@1.3.1: - resolution: {integrity: sha512-uY/99gMLIOlJPwATcMVYfqDSxUR9//AUcgZMzwfSTJPDKzA1S8mX4VLqa+fiAtveraQUBCz4FFcwVZBGbwBXIw==} - ufo@1.3.2: resolution: {integrity: sha512-o+ORpgGwaYQXgqGDwd+hkS4PuZ3QnmqMMxRuajK/a38L6fTpcE5GPIfrf+L/KemFzfUpeUQc1rRS1iDBozvnFA==} @@ -3912,9 +3912,6 @@ packages: unenv@1.10.0: resolution: {integrity: sha512-wY5bskBQFL9n3Eca5XnhH6KbUo/tfvkwm9OpcdCvLaeA7piBNbavbOKJySEwQ1V0RH6HvNlSAFRTpvTqgKRQXQ==} - unenv@1.7.4: - resolution: {integrity: sha512-fjYsXYi30It0YCQYqLOcT6fHfMXsBr2hw9XC7ycf8rTG7Xxpe3ZssiqUnD0khrjiZEmkBXWLwm42yCSCH46fMw==} - unicode-trie@2.0.0: resolution: {integrity: sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ==} @@ -4689,6 +4686,8 @@ snapshots: '@cloudflare/workerd-windows-64@1.20241218.0': optional: true + '@cloudflare/workers-types@4.20241230.0': {} + '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -6137,8 +6136,6 @@ snapshots: convert-source-map@2.0.0: {} - cookie-es@1.0.0: {} - cookie-es@1.2.2: {} cookie@0.7.2: {} @@ -6245,8 +6242,6 @@ snapshots: define-lazy-prop@2.0.0: {} - defu@6.1.3: {} - defu@6.1.4: {} denque@2.1.0: {} @@ -6255,8 +6250,6 @@ snapshots: dequal@2.0.3: {} - destr@2.0.2: {} - destr@2.0.3: {} destroy@1.2.0: {} @@ -6455,6 +6448,8 @@ snapshots: exit-hook@2.2.1: {} + express-rate-limit@7.5.0: {} + fast-deep-equal@3.1.3: {} fast-fifo@1.3.2: {} @@ -6647,17 +6642,6 @@ snapshots: uncrypto: 0.1.3 unenv: 1.10.0 - h3@1.8.2: - dependencies: - cookie-es: 1.0.0 - defu: 6.1.3 - destr: 2.0.2 - iron-webcrypto: 0.10.1 - radix3: 1.1.0 - ufo: 1.3.1 - uncrypto: 0.1.3 - unenv: 1.7.4 - has-flag@3.0.0: {} has-flag@4.0.0: {} @@ -6765,8 +6749,6 @@ snapshots: transitivePeerDependencies: - supports-color - iron-webcrypto@0.10.1: {} - iron-webcrypto@1.2.1: {} is-arrayish@0.2.1: {} @@ -7098,9 +7080,15 @@ snapshots: nanoid@3.3.8: {} + nitro-cloudflare-dev@0.2.1: + dependencies: + consola: 3.2.3 + mlly: 1.7.2 + pkg-types: 1.2.1 + nitro-cors@0.7.1: dependencies: - h3: 1.8.2 + h3: 1.13.0 ufo: 1.3.2 nitropack@2.10.4(typescript@5.6.3): @@ -7426,8 +7414,6 @@ snapshots: queue-tick@1.0.1: {} - radix3@1.1.0: {} - radix3@1.1.2: {} randombytes@2.1.0: @@ -7949,8 +7935,6 @@ snapshots: typescript@5.6.3: {} - ufo@1.3.1: {} - ufo@1.3.2: {} ufo@1.5.4: {} @@ -7997,14 +7981,6 @@ snapshots: node-fetch-native: 1.6.4 pathe: 1.1.2 - unenv@1.7.4: - dependencies: - consola: 3.2.3 - defu: 6.1.4 - mime: 3.0.0 - node-fetch-native: 1.6.4 - pathe: 1.1.2 - unicode-trie@2.0.0: dependencies: pako: 0.2.9 @@ -8294,7 +8270,7 @@ snapshots: '@cloudflare/workerd-linux-arm64': 1.20241218.0 '@cloudflare/workerd-windows-64': 1.20241218.0 - wrangler@3.99.0: + wrangler@3.99.0(@cloudflare/workers-types@4.20241230.0): dependencies: '@cloudflare/kv-asset-handler': 0.3.4 '@esbuild-plugins/node-globals-polyfill': 0.2.3(esbuild@0.17.19) @@ -8314,6 +8290,7 @@ snapshots: workerd: 1.20241218.0 xxhash-wasm: 1.0.2 optionalDependencies: + '@cloudflare/workers-types': 4.20241230.0 fsevents: 2.3.3 transitivePeerDependencies: - bufferutil diff --git a/wrangler.toml b/wrangler.toml index 63243b028..0c5c28fd4 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -2,7 +2,7 @@ name = "api" main = ".output/server/index.mjs" workers_dev = false account_id = "02f3b11d8d1017a20f95de4ba88fb5d6" -compatibility_flags = [ "nodejs_compat" ] +compatibility_flags = ["nodejs_compat"] compatibility_date = "2024-11-01" routes = [ @@ -16,5 +16,11 @@ id = "6f18adea26a64d6b8858ffbdfd3f4cf2" [[unsafe.bindings]] name = "RATE_LIMITER" type = "ratelimit" -namespace_id = "69420" +# An identifier you define, that is unique to your Cloudflare account. +# Must be an integer. +namespace_id = "1001" + +# Limit: the number of tokens allowed within a given period in a single +# Cloudflare location +# Period: the duration of the period, in seconds. Must be either 10 or 60 simple = { limit = 100, period = 60 }