From da3ca482443afec12b7e00cc9b58019532f57bb4 Mon Sep 17 00:00:00 2001 From: desu-bot Date: Thu, 30 Oct 2025 12:03:47 +0000 Subject: [PATCH] chore: update public repo --- scripts/media/tidal-dl.ts | 216 +++++++++++++++++++++++++++----------- utils/media-metadata.ts | 4 +- 2 files changed, 156 insertions(+), 64 deletions(-) diff --git a/scripts/media/tidal-dl.ts b/scripts/media/tidal-dl.ts index 69a08e0..cae999a 100644 --- a/scripts/media/tidal-dl.ts +++ b/scripts/media/tidal-dl.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto' import { mkdir, rm, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' -import { asyncPool, base64, todo, unknownToError, utf8 } from '@fuman/utils' +import { asyncPool, AsyncQueue, base64, todo, unknownToError, utf8 } from '@fuman/utils' import Spinnies from 'spinnies' import { z } from 'zod' import { $, question } from 'zx' @@ -44,7 +44,7 @@ const ffetch = ffetchBase.extend({ // for some reason the request sometimes hangs indefinitely, so we need to timeout timeout: 5000, retry: { - maxRetries: 10, + maxRetries: 3, // onError: (err, req) => { // console.log('%s: error: %s', req.url, err) // return true @@ -162,7 +162,6 @@ async function downloadTrack(options: { }).parsedJson(z.array(z.object({ type: z.string(), contributors: z.array(z.object({ - id: z.number(), name: z.string(), })), }))), @@ -230,31 +229,46 @@ async function downloadTrack(options: { } } -async function fetchAlbumTracks(albumId: number) { - let offset = 0 - const tracks: TidalTrack[] = [] +async function fetchPaginated(params: { + initialOffset?: number + fetch: (offset: number) => Promise<{ items: T[], hasMore: boolean }> +}): Promise { + let offset = params.initialOffset ?? 0 + const items: T[] = [] while (true) { - const res = await ffetch(`https://tidal.com/v1/albums/${albumId}/items`, { query: { - ...COMMON_QUERY, - replace: true, - offset, - limit: 100, - } }).parsedJson(z.object({ - items: z.array(z.object({ - item: TidalTrack, - type: z.literal('track'), - })), - totalNumberOfItems: z.number(), - })) - + const res = await params.fetch(offset) for (const item of res.items) { - tracks.push(item.item) + items.push(item) } - if (tracks.length >= res.totalNumberOfItems) break - offset += 100 + if (!res.hasMore) break + offset += res.items.length } - return tracks + return items +} + +async function fetchAlbumTracks(albumId: number) { + return fetchPaginated({ + fetch: async (offset) => { + const res = await ffetch(`https://tidal.com/v1/albums/${albumId}/items`, { query: { + ...COMMON_QUERY, + replace: true, + offset, + limit: 100, + } }).parsedJson(z.object({ + items: z.array(z.object({ + item: TidalTrack, + type: z.literal('track'), + })), + totalNumberOfItems: z.number(), + })) + + return { + items: res.items.map(it => it.item), + hasMore: res.totalNumberOfItems > offset + res.items.length, + } + }, + }) } async function downloadTrackList(opts: { @@ -272,7 +286,12 @@ async function downloadTrackList(opts: { const firstTrackArtistString = getTrackArtistString(opts.tracks[0]) const isDifferentArtists = opts.tracks.some(it => getTrackArtistString(it) !== firstTrackArtistString) - await asyncPool(opts.tracks, async (track) => { + const retries = new Map() + const queue = new AsyncQueue(opts.tracks) + + let finished = 0 + + await asyncPool(queue, async (track, idx) => { let filename = '' if (opts.includeTrackNumber) { if (isMultiDisc) { @@ -298,10 +317,72 @@ async function downloadTrackList(opts: { opts.onDownloadEnd?.(track, null) } catch (e) { opts.onDownloadEnd?.(track, unknownToError(e)) + + const n = retries.get(track.id) ?? 0 + if (n < 3) { + retries.set(track.id, n + 1) + queue.enqueue(track) + return + } + } + + finished += 1 + + if (finished === opts.tracks.length) { + queue.end() } }, { limit: 8 }) } +async function downloadAlbum(album: TidalAlbum | number) { + const [albumRes, albumTracks] = await Promise.all([ + typeof album === 'number' + ? ffetch(`https://tidal.com/v1/albums/${album}`, { query: COMMON_QUERY }).parsedJson(TidalAlbum) + : Promise.resolve(album), + fetchAlbumTracks(typeof album === 'number' ? album : album.id), + ]) + + console.log(`downloading album ${albumRes.title} with ${albumTracks.length} tracks`) + + const outDir = join('assets/tidal-dl', `${getTrackArtistString(albumRes)} - ${sanitizeFilename(albumRes.title)}`) + await mkdir(outDir, { recursive: true }) + + const albumCoverRes = await ffetch(getAlbumCoverUrl(albumRes.cover)).arrayBuffer() + await writeFile(join(outDir, 'cover.jpg'), new Uint8Array(albumCoverRes)) + + const spinnies = new Spinnies() + spinnies.add('download', { text: 'downloading album...' }) + + const errors = new Map() + await downloadTrackList({ + tracks: albumTracks, + albums: new Map([[albumRes.id, albumRes]]), + albumCoverPaths: new Map([[albumRes.id, join(outDir, 'cover.jpg')]]), + destination: outDir, + includeTrackNumber: true, + onDownloadStart(track) { + spinnies.add(`${track.id}`, { text: getTrackName(track) }) + errors.delete(track.id) + }, + onDownloadEnd(track, error) { + spinnies.remove(`${track.id}`) + if (error) { + errors.set(track.id, error) + } + spinnies.remove(`${track.id}`) + }, + }) + + spinnies.succeed('download', { text: 'downloaded album' }) + + if (errors.size) { + console.error('errors:') + for (const [id, error] of errors) { + console.error(` ${id}: ${error.message}`) + } + } +} + const url = process.argv[2] ?? await question('url or search > ') /* eslint-disable no-cond-assign */ @@ -327,48 +408,57 @@ if ((m = url.match(/\/track\/(\d+)/))) { await rm(tmpAlbumCoverPath) } else if ((m = url.match(/\/album\/(\d+)/))) { - const [albumRes, albumTracks] = await Promise.all([ - ffetch(`https://tidal.com/v1/albums/${m[1]}`, { query: COMMON_QUERY }).parsedJson(TidalAlbum), - fetchAlbumTracks(m[1]), + await downloadAlbum(m[1]) +} else if ((m = url.match(/\/artist\/(\d+)/))) { + const withAppearsOn = (await question('include appears on albums? (y/N) > ')).toLowerCase() === 'y' + + function fetchAlbumList(type: string): Promise { + return fetchPaginated({ + fetch: async (offset) => { + const r = await ffetch(`https://tidal.com/v2/artist/${type}/view-all`, { + query: { + itemId: m[1], + ...COMMON_QUERY, + platform: 'WEB', + limit: 50, + offset, + }, + headers: { + 'x-tidal-client-version': '2025.10.29', + }, + }).parsedJson(z.object({ + items: z.array(z.object({ + type: z.literal('ALBUM'), + data: TidalAlbum, + })), + })) + + return { + items: r.items.map(it => it.data), + hasMore: r.items.length === 50, + } + }, + }) + } + + const [albums, singles, appearsOn] = await Promise.all([ + fetchAlbumList('ARTIST_ALBUMS'), + fetchAlbumList('ARTIST_TOP_SINGLES'), + withAppearsOn ? fetchAlbumList('ARTIST_APPEARS_ON') : Promise.resolve([]), ]) - console.log(`downloading album ${albumRes.title} with ${albumTracks.length} tracks`) + // concat and dedupe + const seenIds = new Set() + const allAlbums: TidalAlbum[] = [] + for (const album of [...albums, ...singles, ...appearsOn]) { + if (seenIds.has(album.id)) continue + seenIds.add(album.id) + allAlbums.push(album) + } + console.log('found %d albums', allAlbums.length) - const outDir = join('assets/tidal-dl', `${getTrackArtistString(albumRes)} - ${sanitizeFilename(albumRes.title)}`) - await mkdir(outDir, { recursive: true }) - - const albumCoverRes = await ffetch(getAlbumCoverUrl(albumRes.cover)).arrayBuffer() - await writeFile(join(outDir, 'cover.jpg'), new Uint8Array(albumCoverRes)) - - const spinnies = new Spinnies() - spinnies.add('download', { text: 'downloading album...' }) - - const errors = new Map() - await downloadTrackList({ - tracks: albumTracks, - albums: new Map([[albumRes.id, albumRes]]), - albumCoverPaths: new Map([[albumRes.id, join(outDir, 'cover.jpg')]]), - destination: outDir, - includeTrackNumber: true, - onDownloadStart(track) { - spinnies.add(`${track.id}`, { text: getTrackName(track) }) - }, - onDownloadEnd(track, error) { - spinnies.remove(`${track.id}`) - if (error) { - errors.set(track.id, error) - } - spinnies.remove(`${track.id}`) - }, - }) - - spinnies.succeed('download', { text: 'downloaded album' }) - - if (errors.size) { - console.error('errors:') - for (const [id, error] of errors) { - console.error(` ${id}: ${error.message}`) - } + for (const album of allAlbums) { + await downloadAlbum(album) } } else { todo('unsupported url') diff --git a/utils/media-metadata.ts b/utils/media-metadata.ts index 3dab934..be5f195 100644 --- a/utils/media-metadata.ts +++ b/utils/media-metadata.ts @@ -100,8 +100,10 @@ export function generateFfmpegMetadataFlags(metadata: Partial((resolve, reject) => { + nodeStream.on('error', reject) + const pipe = nodeStream.pipe(proc.stdin) pipe.on('error', reject) pipe.on('finish', resolve) })