From 2423324540d823d31ccd17f787860732a1d36062 Mon Sep 17 00:00:00 2001 From: desu-bot Date: Thu, 16 Jan 2025 03:25:20 +0000 Subject: [PATCH] chore: update public repo --- package.json | 3 + pnpm-lock.yaml | 132 +++++++++++++++++ scripts/media/fwmc-radio.ts | 40 +++++ scripts/media/soundcloud-dl.ts | 262 +++++++++++++++++++++++++++++++++ utils/fs.ts | 4 + utils/misc.ts | 6 + utils/opus.ts | 30 ++++ utils/strings.ts | 54 +++++++ 8 files changed, 531 insertions(+) create mode 100644 scripts/media/fwmc-radio.ts create mode 100644 scripts/media/soundcloud-dl.ts create mode 100644 utils/opus.ts create mode 100644 utils/strings.ts diff --git a/package.json b/package.json index 4336100..8e7bf7c 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,11 @@ "dependencies": { "@faker-js/faker": "^9.3.0", "@fuman/io": "^0.0.4", + "@fuman/net": "^0.0.9", "@fuman/node": "^0.0.4", "@mtcute/node": "^0.19.1", "@types/plist": "^3.0.5", + "@types/spinnies": "^0.5.3", "cheerio": "^1.0.0", "es-main": "^1.3.0", "filesize": "^10.1.6", @@ -19,6 +21,7 @@ "nanoid": "^5.0.9", "plist": "^3.1.0", "qrcode-terminal": "^0.12.0", + "spinnies": "^0.5.1", "tough-cookie": "^5.0.0", "tough-cookie-file-store": "^2.0.3", "undici": "^7.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7616233..de66943 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@fuman/io': specifier: ^0.0.4 version: 0.0.4 + '@fuman/net': + specifier: ^0.0.9 + version: 0.0.9 '@fuman/node': specifier: ^0.0.4 version: 0.0.4 @@ -23,6 +26,9 @@ importers: '@types/plist': specifier: ^3.0.5 version: 3.0.5 + '@types/spinnies': + specifier: ^0.5.3 + version: 0.5.3 cheerio: specifier: ^1.0.0 version: 1.0.0 @@ -47,6 +53,9 @@ importers: qrcode-terminal: specifier: ^0.12.0 version: 0.12.0 + spinnies: + specifier: ^0.5.1 + version: 0.5.1 tough-cookie: specifier: ^5.0.0 version: 5.0.0 @@ -264,9 +273,15 @@ packages: '@fuman/io@0.0.4': resolution: {integrity: sha512-IXzBJjHTVKyi04WaGtSXE0dhL3QK45ekrEZNfH/V59XQ1WupSqWevfSWd9T07rdagc2jtaeu8aJY6bwaiJpdYg==} + '@fuman/io@0.0.8': + resolution: {integrity: sha512-+cRZ2JOMYceNQ2Rh6UYro5XVa11j29Sdd3Yhi4GfxAx6ZwCNIw3P80xcTRwCZSfMPLDNN9Etkq7NIc5v9lpItw==} + '@fuman/net@0.0.4': resolution: {integrity: sha512-a8Isnj+qgRNaqqmBCT6lZ9GZj5F3vQdygN5AzB6GGCbLKcOeH+1u5Twh5CUAW/dM7oogrTWOwCqgvS2XHbjzaQ==} + '@fuman/net@0.0.9': + resolution: {integrity: sha512-asn7VJbT8woVXAFCUMZrdyNZCSsXZclraeVZ6RYJ+T3RwQ+JfMMZtXLLTZ7XHrBPxk8x8hoHOJa/Fnyfm+ggbQ==} + '@fuman/node@0.0.4': resolution: {integrity: sha512-tgwbIceUHWuwh4RTwJRQ1sLjzuIGrWx0SeCrqYhGF+IkI/B7DY0FP2SZykWImkVDtW8IzmdZskPZqiDINRGcNg==} @@ -375,6 +390,9 @@ packages: '@types/plist@3.0.5': resolution: {integrity: sha512-E6OCaRmAe4WDmWNsL/9RMqdkkzDCY1etutkflWk4c+AcjDU07Pcz1fQwTX0TQz+Pxqn9i4L1TU3UFpjnrcDgxA==} + '@types/spinnies@0.5.3': + resolution: {integrity: sha512-HYrOubG2TVgRQRKcW1HJ/1eJIIBpLqDoJo551McJgWdO8xzxnaxu/bPKdqC/7okoEy4ZZjy3I4/DwK1sz2OCog==} + '@types/unist@3.0.3': resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} @@ -485,10 +503,18 @@ packages: ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + ansi-regex@4.1.1: + resolution: {integrity: sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==} + engines: {node: '>=6'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} + ansi-styles@3.2.1: + resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} + engines: {node: '>=4'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} @@ -553,6 +579,10 @@ packages: ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + chalk@2.4.2: + resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} + engines: {node: '>=4'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -578,14 +608,24 @@ packages: resolution: {integrity: sha512-GfisEZEJvzKrmGWkvfhgzcz/BllN1USeqD2V6tg14OAOgaCD2Z/PUEuxnAZ/nPvmaHRG7a8y77p1T/IRQ4D1Hw==} engines: {node: '>=4'} + cli-cursor@3.1.0: + resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} + engines: {node: '>=8'} + cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} + color-convert@1.9.3: + resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} + color-name@1.1.3: + resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} + color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} @@ -1032,6 +1072,10 @@ packages: graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} + has-flag@3.0.0: + resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} + engines: {node: '>=4'} + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -1320,6 +1364,10 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} @@ -1383,6 +1431,10 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -1558,6 +1610,10 @@ packages: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} hasBin: true + restore-cursor@3.1.0: + resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} + engines: {node: '>=8'} + reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -1592,6 +1648,9 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} @@ -1623,6 +1682,9 @@ packages: spdx-license-ids@3.0.20: resolution: {integrity: sha512-jg25NiDV/1fLtSgEgyvVyDunvaNHbuwF9lfNV17gSmPFAlYzdfNBlLtLzXTevwkPj7DhGbmN9VnmJIgLnhvaBw==} + spinnies@0.5.1: + resolution: {integrity: sha512-WpjSXv9NQz0nU3yCT9TFEOfpFrXADY9C5fG6eAJqixLhvTX1jP3w92Y8IE5oafIe42nlF9otjhllnXN/QCaB3A==} + stable-hash@0.0.4: resolution: {integrity: sha512-LjdcbuBeLcdETCrPn9i8AYAZ1eCtu4ECAWtP7UleOiZ9LzVxRzzUZEoZ8zB24nhkQnDWyET0I+3sWokSDS3E7g==} @@ -1633,6 +1695,10 @@ packages: string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + strip-ansi@5.2.0: + resolution: {integrity: sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==} + engines: {node: '>=6'} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -1649,6 +1715,10 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + supports-color@5.5.0: + resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} + engines: {node: '>=4'} + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -2040,11 +2110,20 @@ snapshots: dependencies: '@fuman/utils': 0.0.4 + '@fuman/io@0.0.8': + dependencies: + '@fuman/utils': 0.0.4 + '@fuman/net@0.0.4': dependencies: '@fuman/io': 0.0.4 '@fuman/utils': 0.0.4 + '@fuman/net@0.0.9': + dependencies: + '@fuman/io': 0.0.8 + '@fuman/utils': 0.0.4 + '@fuman/node@0.0.4': dependencies: '@fuman/io': 0.0.4 @@ -2182,6 +2261,8 @@ snapshots: '@types/node': 22.10.0 xmlbuilder: 15.1.1 + '@types/spinnies@0.5.3': {} + '@types/unist@3.0.3': {} '@typescript-eslint/eslint-plugin@8.16.0(@typescript-eslint/parser@8.16.0(eslint@9.15.0)(typescript@5.7.2))(eslint@9.15.0)(typescript@5.7.2)': @@ -2320,8 +2401,14 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 + ansi-regex@4.1.1: {} + ansi-regex@5.0.1: {} + ansi-styles@3.2.1: + dependencies: + color-convert: 1.9.3 + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 @@ -2388,6 +2475,12 @@ snapshots: ccount@2.0.1: {} + chalk@2.4.2: + dependencies: + ansi-styles: 3.2.1 + escape-string-regexp: 1.0.5 + supports-color: 5.5.0 + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -2426,16 +2519,26 @@ snapshots: dependencies: escape-string-regexp: 1.0.5 + cli-cursor@3.1.0: + dependencies: + restore-cursor: 3.1.0 + cliui@8.0.1: dependencies: string-width: 4.2.3 strip-ansi: 6.0.1 wrap-ansi: 7.0.0 + color-convert@1.9.3: + dependencies: + color-name: 1.1.3 + color-convert@2.0.1: dependencies: color-name: 1.1.4 + color-name@1.1.3: {} + color-name@1.1.4: {} comment-parser@1.4.1: {} @@ -2948,6 +3051,8 @@ snapshots: graphemer@1.4.0: {} + has-flag@3.0.0: {} + has-flag@4.0.0: {} hasown@2.0.2: @@ -3393,6 +3498,8 @@ snapshots: braces: 3.0.3 picomatch: 2.3.1 + mimic-fn@2.1.0: {} + mimic-response@3.1.0: {} min-indent@1.0.1: {} @@ -3449,6 +3556,10 @@ snapshots: dependencies: wrappy: 1.0.2 + onetime@5.1.2: + dependencies: + mimic-fn: 2.1.0 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -3637,6 +3748,11 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 + restore-cursor@3.1.0: + dependencies: + onetime: 5.1.2 + signal-exit: 3.0.7 + reusify@1.0.4: {} run-parallel@1.2.0: @@ -3663,6 +3779,8 @@ snapshots: shebang-regex@3.0.0: {} + signal-exit@3.0.7: {} + simple-concat@1.0.1: {} simple-get@4.0.1: @@ -3696,6 +3814,12 @@ snapshots: spdx-license-ids@3.0.20: {} + spinnies@0.5.1: + dependencies: + chalk: 2.4.2 + cli-cursor: 3.1.0 + strip-ansi: 5.2.0 + stable-hash@0.0.4: {} string-width@4.2.3: @@ -3708,6 +3832,10 @@ snapshots: dependencies: safe-buffer: 5.2.1 + strip-ansi@5.2.0: + dependencies: + ansi-regex: 4.1.1 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 @@ -3720,6 +3848,10 @@ snapshots: strip-json-comments@3.1.1: {} + supports-color@5.5.0: + dependencies: + has-flag: 3.0.0 + supports-color@7.2.0: dependencies: has-flag: 4.0.0 diff --git a/scripts/media/fwmc-radio.ts b/scripts/media/fwmc-radio.ts new file mode 100644 index 0000000..a12dc4d --- /dev/null +++ b/scripts/media/fwmc-radio.ts @@ -0,0 +1,40 @@ +import { mkdir } from 'node:fs/promises' +import { asyncPool } from '@fuman/utils' +import json5 from 'json5' +import Spinnies from 'spinnies' +import { z } from 'zod' +import { downloadFile, ffetch } from '../../utils/fetch.ts' +import { fileExists } from '../../utils/fs.ts' +import { parseJsObject } from '../../utils/strings.ts' + +const $ = await ffetch('https://fwmc-ai.github.io/radio/').cheerio() + +const script = $('script:icontains(const playlist =)').html()! + +const playlistJs = parseJsObject(`[${script.split('const playlist = [').at(-1)!}`)! +const playlist = z.array( + z.object({ + id: z.string(), + title: z.string(), + file: z.string(), + cover: z.string(), + category: z.enum(['original', 'cover']), + lyrics: z.string(), + }), +).parse(json5.parse(playlistJs)) + +const spinnies = new Spinnies() + +await mkdir('assets/fwmc-radio', { recursive: true }) + +await asyncPool(playlist, async (item) => { + const dlPath = `assets/fwmc-radio/${item.id}.mp3` + if (await fileExists(dlPath)) return + + spinnies.add(item.id, { text: item.title }) + await downloadFile(new URL(item.file, 'https://fwmc-ai.github.io/radio/').toString(), dlPath) + spinnies.remove(item.id) +}) + +console.log('done') +spinnies.stopAll() diff --git a/scripts/media/soundcloud-dl.ts b/scripts/media/soundcloud-dl.ts new file mode 100644 index 0000000..fc640dd --- /dev/null +++ b/scripts/media/soundcloud-dl.ts @@ -0,0 +1,262 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { ffetchAddons } from '@fuman/fetch' +import { asyncPool, base64 } from '@fuman/utils' +import { load } from 'cheerio' +import Spinnies from 'spinnies' +import { ProxyAgent } from 'undici' +import { z } from 'zod' +import { $ } from 'zx' +import { downloadFile, ffetch as ffetchBase } from '../../utils/fetch.ts' +import { sanitizeFilename } from '../../utils/fs.ts' +import { chunks, getEnv } from '../../utils/misc.ts' +import { generateOpusImageBlob } from '../../utils/opus.ts' + +const ffetchApi = ffetchBase.extend({ + baseUrl: 'https://api-v2.soundcloud.com', + // @ts-expect-error lol fixme + query: { + client_id: '4BowhSywvkJtklODQDzjNMq9sK9wyDJ4', + app_version: '1736857534', + app_locale: 'en', + }, + addons: [ + ffetchAddons.rateLimitHandler(), + ], + rateLimit: { + isRejected(res) { + return res.status === 429 + }, + defaultWaitTime: 10_000, + }, + headers: { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36', + 'Authorization': `OAuth ${getEnv('SOUNDCLOUD_TOKEN')}`, + }, +}) +const ffetchHtml = ffetchBase.extend({ + baseUrl: 'https://soundcloud.com', + headers: { + Cookie: `oauth_token=${getEnv('SOUNDCLOUD_TOKEN')}`, + }, + extra: { + // @ts-expect-error lol fixme + dispatcher: new ProxyAgent('http://127.0.0.1:7891'), + }, +}) + +const ScTrack = z.object({ + id: z.number(), + kind: z.literal('track'), + title: z.string(), + duration: z.number(), + permalink_url: z.string(), + artwork_url: z.string().transform(s => s.replace('-large.jpg', '-t500x500.jpg')).nullable(), + media: z.object({ + transcodings: z.array(z.object({ + url: z.string(), + preset: z.string(), + format: z.object({ + protocol: z.string(), + mime_type: z.string(), + }), + quality: z.string(), + is_legacy_transcoding: z.boolean(), + })), + }), + track_authorization: z.string(), + user: z.object({ + username: z.string(), + permalink: z.string(), + }), +}) +type ScTrack = z.infer + +const ScPlaylist = z.object({ + id: z.number(), + title: z.string(), + duration: z.number(), + permalink_url: z.string(), + genre: z.string(), + description: z.string().nullable(), + track_count: z.number(), + user: z.object({ + username: z.string(), + }), + tracks: z.array(z.union([ + ScTrack, + z.object({ + id: z.number(), + kind: z.literal('track'), + }), + ])), +}) +type ScPlaylist = z.infer + +function extractHydrationData(html: string) { + const $ = load(html) + const script = $('script:contains(window.__sc_hydration = )') + return JSON.parse(script.html()!.replace('window.__sc_hydration = ', '').slice(0, -1)) +} + +async function fetchTrackByUrl(url: string) { + const html = await ffetchHtml(url).text() + const hydrationData = extractHydrationData(html) + const track = hydrationData.find(it => it.hydratable === 'sound') + if (!track) throw new Error('no track found') + + return ScTrack.parse(track.data) +} + +async function fetchPlaylistByUrl(url: string) { + const html = await ffetchHtml(url).text() + const hydrationData = extractHydrationData(html) + const playlist = hydrationData.find(it => it.hydratable === 'playlist') + if (!playlist) throw new Error('no playlist found') + + return ScPlaylist.parse(playlist.data) +} + +async function fetchTracksById(trackIds: number[]) { + return ffetchApi('/tracks', { + query: { + ids: trackIds.join(','), + }, + }).parsedJson(z.array(ScTrack)) +} + +async function downloadTrack(track: ScTrack, opts: { + /* download destination (filename without extension) */ + destination: string +}) { + const artworkPath = join('assets', `sc-tmp-${track.id}.jpg`) + const artworkBytes = track.artwork_url ? new Uint8Array(await ffetchHtml(track.artwork_url).arrayBuffer()) : null + + // find the best transcoding + const transcoding = track.media.transcodings.sort((a, b) => { + // prefer non-legacy transcodings + if (a.is_legacy_transcoding && !b.is_legacy_transcoding) return -1 + if (!a.is_legacy_transcoding && b.is_legacy_transcoding) return 1 + + // prefer hq + if (a.quality === 'sq' && b.quality === 'hq') return -1 + if (a.quality === 'hq' && b.quality === 'sq') return 1 + + // prefer opus + if (a.preset === 'opus_0_0' && b.preset !== 'opus_0_0') return -1 + if (a.preset !== 'opus_0_0' && b.preset === 'opus_0_0') return 1 + + return 0 + })[0] + + const { url: hlsUrl } = await ffetchApi(transcoding.url, { + query: { + track_authorization: track.track_authorization, + }, + }).parsedJson(z.object({ + url: z.string(), + })) + + const ext = transcoding.format.mime_type.match(/^audio\/(\w+)(;|$)/)![1] + const filename = `${opts.destination}.${ext}` + + const params: string[] = [ + '-y', + '-i', + hlsUrl, + '-c', + 'copy', + ] + + if (ext === 'mp3') { + if (artworkBytes) { + await writeFile(artworkPath, artworkBytes) + params.push( + '-i', + artworkPath, + '-map', + '0:a', + '-map', + '1:0', + ) + } + params.push( + '-id3v2_version', + '3', + '-metadata:s:v', + 'title="Album cover"', + '-metadata:s:v', + 'comment="Cover (front)"', + ) + } else if (ext === 'ogg' && artworkBytes) { + const blob = base64.encode(await generateOpusImageBlob(artworkBytes)) + params.push( + '-metadata', + `metadata_block_picture=${blob}`, + ) + } + + params.push( + '-metadata', + `title=${track.title}`, + '-metadata', + `artist=${track.user.username}`, + filename, + ) + + await $`ffmpeg ${params}`.quiet(true) +} + +async function downloadPlaylist(playlist: ScPlaylist) { + const tracks: ScTrack[] = [] + const tracksToFetch = new Set() + const trackIdToPosition = new Map() + + for (let i = 0; i < playlist.tracks.length; i++) { + const track = playlist.tracks[i] + trackIdToPosition.set(track.id, i + 1) + if ('user' in track) { + tracks.push(track) + } else { + tracksToFetch.add(track.id) + } + } + + const spinnies = new Spinnies() + + if (tracksToFetch.size) { + let remaining = tracksToFetch.size + spinnies.add('fetching', { text: `fetching ${remaining} tracks` }) + await asyncPool(chunks(Array.from(tracksToFetch), 20), async (ids) => { + const res = await fetchTracksById(Array.from(ids)) + for (const track of res) { + tracks.push(track) + } + remaining -= ids.length + spinnies.update('fetching', { text: `fetching ${remaining} tracks` }) + }) + spinnies.succeed('fetching') + } + + const destDir = join('assets/soundcloud-dl', sanitizeFilename(`${playlist.user.username} - ${playlist.title}`)) + await mkdir(destDir, { recursive: true }) + + const posPadSize = Math.ceil(Math.log10(tracks.length)) + + await asyncPool(tracks, async (track) => { + const position = trackIdToPosition.get(track.id)! + const filename = `${position.toString().padStart(posPadSize, '0')}. ${track.user.username} - ${track.title}` + + spinnies.add(`${track.id}`, { text: filename }) + await downloadTrack(track, { + destination: join(destDir, filename), + }) + + spinnies.remove(`${track.id}`) + }, { limit: 8 }) + + console.log('done') + spinnies.stopAll() +} + +await downloadPlaylist(await fetchPlaylistByUrl('https://soundcloud.com/user-398958278/sets/l2grace')) diff --git a/utils/fs.ts b/utils/fs.ts index 90834ab..11ab17f 100644 --- a/utils/fs.ts +++ b/utils/fs.ts @@ -17,3 +17,7 @@ export async function directoryExists(path: string): Promise { return false } } + +export function sanitizeFilename(filename: string) { + return filename.replace(/[/\\?%*:|"<>]/g, '_') +} diff --git a/utils/misc.ts b/utils/misc.ts index e38398d..c3ae40c 100644 --- a/utils/misc.ts +++ b/utils/misc.ts @@ -8,3 +8,9 @@ export function getEnv(key: string, parser?: (value: string) => T): T | strin if (!parser) return value return parser(value) } + +export function* chunks(arr: T[], size: number) { + for (let i = 0; i < arr.length; i += size) { + yield arr.slice(i, i + size) + } +} diff --git a/utils/opus.ts b/utils/opus.ts new file mode 100644 index 0000000..06ff13b --- /dev/null +++ b/utils/opus.ts @@ -0,0 +1,30 @@ +import { Bytes, write } from '@fuman/io' +import { $ } from 'zx' + +export async function generateOpusImageBlob(image: Uint8Array) { + // todo we should probably not use ffprobe here but whatever lol + const proc = $`ffprobe -of json -v error -show_entries stream=codec_name,width,height pipe:0` + proc.stdin.write(image) + proc.stdin.end() + const json = await proc.json() + + const img = json.streams[0] + // https://www.rfc-editor.org/rfc/rfc9639.html#section-8.8 + const mime = img.codec_name === 'mjpeg' ? 'image/jpeg' : 'image/png' + const description = 'Cover Artwork' + + const res = Bytes.alloc(image.length + 128) + write.uint32be(res, 3) // picture type = album cover + write.uint32be(res, mime.length) + write.rawString(res, mime) + write.uint32be(res, description.length) + write.rawString(res, description) + write.uint32be(res, img.width) + write.uint32be(res, img.height) + write.uint32be(res, 0) // color depth + write.uint32be(res, 0) // color index (unused, for gifs) + write.uint32be(res, image.length) + write.bytes(res, image) + + return res.result() +} diff --git a/utils/strings.ts b/utils/strings.ts new file mode 100644 index 0000000..fec4799 --- /dev/null +++ b/utils/strings.ts @@ -0,0 +1,54 @@ +export function parseJsObject(str: string, offset = 0) { + let i = offset + const len = str.length + let start = -1 + let end = -1 + + const depth = { + '{': 0, + '[': 0, + } + + const possibleQuotes = { + '"': true, + '\'': true, + '`': true, + } + let inQuote: string | null = null + let escapeNextQuote = false + + while (i < len) { + const char = str[i] + if (char in possibleQuotes && !escapeNextQuote) { + if (inQuote === null) { + inQuote = char + } else if (char === inQuote) { + inQuote = null + } + } else if (inQuote != null) { + escapeNextQuote = char === '\\' && !escapeNextQuote + } else if (inQuote == null && char in depth) { + if (start === -1) { + start = i + } + depth[char] += 1 + } else if (inQuote == null && ( + char === '}' || char === ']' + )) { + if (char === '}') depth['{'] -= 1 + if (char === ']') depth['['] -= 1 + + if (depth['{'] === 0 && depth['['] === 0) { + end = i + 1 + break + } + } + i += 1 + } + + if (start === -1 && end === -1) return null + if (depth['{'] !== 0 || depth['['] !== 0) throw new SyntaxError('Mismatched brackets') + if (inQuote) throw new SyntaxError('Unclosed string') + + return str.substring(start, end) +}