From bb5311f828f43e87522e1fa084b32a3830a30b9a Mon Sep 17 00:00:00 2001 From: desu-bot Date: Mon, 3 Feb 2025 14:42:22 +0000 Subject: [PATCH] chore: update public repo --- scripts/media/soundcloud-dl.ts | 155 +++++++++++++++++++++++++-------- 1 file changed, 118 insertions(+), 37 deletions(-) diff --git a/scripts/media/soundcloud-dl.ts b/scripts/media/soundcloud-dl.ts index 2732d9e..ac459dc 100644 --- a/scripts/media/soundcloud-dl.ts +++ b/scripts/media/soundcloud-dl.ts @@ -16,7 +16,7 @@ const ffetchApi = ffetchBase.extend({ baseUrl: 'https://api-v2.soundcloud.com', query: { client_id: '4BowhSywvkJtklODQDzjNMq9sK9wyDJ4', - app_version: '1736857534', + app_version: '1738322252', app_locale: 'en', }, addons: [ @@ -83,6 +83,18 @@ const ScPlaylist = z.object({ }) type ScPlaylist = z.infer +const ScUser = z.object({ + id: z.number(), + kind: z.literal('user'), + permalink_url: z.string(), + username: z.string(), + + likes_count: z.number(), + track_count: z.number(), + playlist_likes_count: z.number(), +}) +type ScUser = z.infer + const ScLike = z.object({ created_at: z.string(), kind: z.literal('like'), @@ -96,15 +108,6 @@ function extractHydrationData(html: string) { return JSON.parse(script.html()!.replace('window.__sc_hydration = ', '').slice(0, -1)) } -async function fetchTrackByUrl(url: string) { - const html = await ffetchHtml(url).text() - const hydrationData = extractHydrationData(html) - const track = hydrationData.find(it => it.hydratable === 'sound') - if (!track) throw new Error('no track found') - - return ScTrack.parse(track.data) -} - async function fetchPlaylistByUrl(url: string) { const html = await ffetchHtml(url).text() const hydrationData = extractHydrationData(html) @@ -318,11 +321,7 @@ async function downloadLikes(username: string) { const hydrationData = extractHydrationData(userPage) const user = hydrationData.find(it => it.hydratable === 'user') if (!user) throw new Error('no user found') - const userData = z.object({ - likes_count: z.number(), - playlist_likes_count: z.number(), - id: z.number(), - }).parse(user.data) + const userData = ScUser.parse(user.data) const tracks: ScTrack[] = [] const playlists: ScPlaylist[] = [] @@ -366,7 +365,7 @@ async function downloadLikes(username: string) { spinnies.succeed('collect', { text: `collected ${tracks.length} tracks and ${playlists.length} playlists` }) spinnies.add('tracks', { text: 'downloading tracks...' }) - const downloaded = 0 + let downloaded = 0 const updateTracksSpinner = () => { spinnies.update('tracks', { text: `[${downloaded}/${tracks.length}] downloading tracks...` }) } @@ -375,21 +374,22 @@ async function downloadLikes(username: string) { const baseDir = join('assets/soundcloud-dl', `${sanitizeFilename(username)}-likes`) await mkdir(baseDir, { recursive: true }) - // await asyncPool(tracks, async (track) => { - // const filename = `${track.user.username} - ${track.title}` - // spinnies.add(`${track.id}`, { text: filename }) - // await downloadTrack(track, { - // destination: join(baseDir, sanitizeFilename(filename)), - // onRateLimit: (wait) => { - // spinnies.update(`${track.id}`, { text: `[rate limit ${Math.floor(wait / 1000)}s] ${filename}` }) - // }, - // onCdnRateLimit: () => { - // spinnies.update(`${track.id}`, { text: `[cdn rate limit] ${filename}` }) - // }, - // }) - // spinnies.remove(`${track.id}`) - // updateTracksSpinner() - // }) + await asyncPool(tracks, async (track) => { + const filename = `${track.user.username} - ${track.title}` + spinnies.add(`${track.id}`, { text: filename }) + await downloadTrack(track, { + destination: join(baseDir, sanitizeFilename(filename)), + onRateLimit: (wait) => { + spinnies.update(`${track.id}`, { text: `[rate limit ${Math.floor(wait / 1000)}s] ${filename}` }) + }, + onCdnRateLimit: () => { + spinnies.update(`${track.id}`, { text: `[cdn rate limit] ${filename}` }) + }, + }) + spinnies.remove(`${track.id}`) + downloaded += 1 + updateTracksSpinner() + }) spinnies.succeed('tracks', { text: `downloaded ${downloaded} tracks` }) spinnies.stopAll() @@ -404,6 +404,75 @@ async function downloadLikes(username: string) { } } +async function downloadUser(user: ScUser) { + const tracks: ScTrack[] = [] + const spinnies = new Spinnies() + + spinnies.add('collect') + const updateSpinner = () => { + const percent = Math.floor(tracks.length / user.track_count * 100) + spinnies.update('collect', { + text: `[${percent}%] collecting user tracks: ${tracks.length}/${user.track_count}`, + }) + } + updateSpinner() + + let offset = '0' + while (true) { + const res = await ffetchApi(`/users/${user.id}/tracks`, { + query: { + limit: 100, + offset, + linked_partitioning: '1', + }, + }).parsedJson(z.object({ + collection: z.array(ScTrack), + next_href: z.string().nullable(), + })) + + for (const track of res.collection) { + tracks.push(track) + } + + updateSpinner() + + if (!res.next_href) break + offset = new URL(res.next_href).searchParams.get('offset')! + } + + spinnies.succeed('collect', { text: `collected ${tracks.length} tracks` }) + + spinnies.add('tracks', { text: 'downloading tracks...' }) + let downloaded = 0 + const updateTracksSpinner = () => { + spinnies.update('tracks', { text: `[${downloaded}/${tracks.length}] downloading tracks...` }) + } + updateTracksSpinner() + + const baseDir = join('assets/soundcloud-dl', `${sanitizeFilename(user.username)}-tracks`) + await mkdir(baseDir, { recursive: true }) + + await asyncPool(tracks, async (track) => { + const filename = track.title + spinnies.add(`${track.id}`, { text: filename }) + await downloadTrack(track, { + destination: join(baseDir, sanitizeFilename(filename)), + onRateLimit: (wait) => { + spinnies.update(`${track.id}`, { text: `[rate limit ${Math.floor(wait / 1000)}s] ${filename}` }) + }, + onCdnRateLimit: () => { + spinnies.update(`${track.id}`, { text: `[cdn rate limit] ${filename}` }) + }, + }) + downloaded += 1 + spinnies.remove(`${track.id}`) + updateTracksSpinner() + }) + + spinnies.succeed('tracks', { text: `downloaded ${downloaded} tracks` }) + spinnies.stopAll() +} + const url = process.argv[2] ?? await question('url > ') if (!url.startsWith('https://soundcloud.com/')) { console.error('url must start with https://soundcloud.com/') @@ -415,10 +484,22 @@ if (url.match(/^https:\/\/soundcloud.com\/[a-z0-9-]+\/sets\//i)) { } else if (url.match(/^https:\/\/soundcloud.com\/[a-z0-9-]+\/likes/i)) { await downloadLikes(url.match(/^https:\/\/soundcloud.com\/([a-z0-9-]+)\/likes/i)![1]) } else { - const track = await fetchTrackByUrl(url) - const filename = `${track.user.username} - ${track.title}` - console.log('downloading track:', filename) - await downloadTrack(track, { - destination: join('assets/soundcloud-dl', sanitizeFilename(filename)), - }) + const html = await ffetchHtml(url).text() + + const hydrationData = extractHydrationData(html) + const trackData = hydrationData.find(it => it.hydratable === 'sound') + if (trackData) { + const track = ScTrack.parse(trackData.data) + const filename = `${track.user.username} - ${track.title}` + console.log('downloading track:', filename) + await downloadTrack(track, { + destination: join('assets/soundcloud-dl', sanitizeFilename(filename)), + }) + } else { + const userData = hydrationData.find(it => it.hydratable === 'user') + if (userData) { + const user = ScUser.parse(userData.data) + await downloadUser(user) + } + } }