chore: update public repo

scripts/media/tidal-dl.ts

@@ -1,7 +1,7 @@
 import { randomUUID } from 'node:crypto'
 import { mkdir, rm, writeFile } from 'node:fs/promises'
 import { dirname, join } from 'node:path'
-import { asyncPool, base64, todo, unknownToError, utf8 } from '@fuman/utils'
+import { asyncPool, AsyncQueue, base64, todo, unknownToError, utf8 } from '@fuman/utils'
 import Spinnies from 'spinnies'
 import { z } from 'zod'
 import { $, question } from 'zx'
@@ -44,7 +44,7 @@ const ffetch = ffetchBase.extend({
   // for some reason the request sometimes hangs indefinitely, so we need to timeout
   timeout: 5000,
   retry: {
-    maxRetries: 10,
+    maxRetries: 3,
     // onError: (err, req) => {
     //   console.log('%s: error: %s', req.url, err)
     //   return true
@@ -162,7 +162,6 @@ async function downloadTrack(options: {
     }).parsedJson(z.array(z.object({
       type: z.string(),
       contributors: z.array(z.object({
-        id: z.number(),
         name: z.string(),
       })),
     }))),
@@ -230,31 +229,46 @@ async function downloadTrack(options: {
   }
 }
 
-async function fetchAlbumTracks(albumId: number) {
+async function fetchPaginated<T>(params: {
-  let offset = 0
+  initialOffset?: number
-  const tracks: TidalTrack[] = []
+  fetch: (offset: number) => Promise<{ items: T[], hasMore: boolean }>
+}): Promise<T[]> {
+  let offset = params.initialOffset ?? 0
+  const items: T[] = []
   while (true) {
-    const res = await ffetch(`https://tidal.com/v1/albums/${albumId}/items`, { query: {
+    const res = await params.fetch(offset)
-      ...COMMON_QUERY,
-      replace: true,
-      offset,
-      limit: 100,
-    } }).parsedJson(z.object({
-      items: z.array(z.object({
-        item: TidalTrack,
-        type: z.literal('track'),
-      })),
-      totalNumberOfItems: z.number(),
-    }))
-
     for (const item of res.items) {
-      tracks.push(item.item)
+      items.push(item)
     }
-    if (tracks.length >= res.totalNumberOfItems) break
+    if (!res.hasMore) break
-    offset += 100
+    offset += res.items.length
   }
 
-  return tracks
+  return items
+}
+
+async function fetchAlbumTracks(albumId: number) {
+  return fetchPaginated({
+    fetch: async (offset) => {
+      const res = await ffetch(`https://tidal.com/v1/albums/${albumId}/items`, { query: {
+        ...COMMON_QUERY,
+        replace: true,
+        offset,
+        limit: 100,
+      } }).parsedJson(z.object({
+        items: z.array(z.object({
+          item: TidalTrack,
+          type: z.literal('track'),
+        })),
+        totalNumberOfItems: z.number(),
+      }))
+
+      return {
+        items: res.items.map(it => it.item),
+        hasMore: res.totalNumberOfItems > offset + res.items.length,
+      }
+    },
+  })
 }
 
 async function downloadTrackList(opts: {
@@ -272,7 +286,12 @@ async function downloadTrackList(opts: {
   const firstTrackArtistString = getTrackArtistString(opts.tracks[0])
   const isDifferentArtists = opts.tracks.some(it => getTrackArtistString(it) !== firstTrackArtistString)
 
-  await asyncPool(opts.tracks, async (track) => {
+  const retries = new Map<number, number>()
+  const queue = new AsyncQueue(opts.tracks)
+
+  let finished = 0
+
+  await asyncPool(queue, async (track, idx) => {
     let filename = ''
     if (opts.includeTrackNumber) {
       if (isMultiDisc) {
@@ -298,10 +317,72 @@ async function downloadTrackList(opts: {
       opts.onDownloadEnd?.(track, null)
     } catch (e) {
       opts.onDownloadEnd?.(track, unknownToError(e))
+
+      const n = retries.get(track.id) ?? 0
+      if (n < 3) {
+        retries.set(track.id, n + 1)
+        queue.enqueue(track)
+        return
+      }
+    }
+
+    finished += 1
+
+    if (finished === opts.tracks.length) {
+      queue.end()
     }
   }, { limit: 8 })
 }
 
+async function downloadAlbum(album: TidalAlbum | number) {
+  const [albumRes, albumTracks] = await Promise.all([
+    typeof album === 'number'
+      ? ffetch(`https://tidal.com/v1/albums/${album}`, { query: COMMON_QUERY }).parsedJson(TidalAlbum)
+      : Promise.resolve(album),
+    fetchAlbumTracks(typeof album === 'number' ? album : album.id),
+  ])
+
+  console.log(`downloading album ${albumRes.title} with ${albumTracks.length} tracks`)
+
+  const outDir = join('assets/tidal-dl', `${getTrackArtistString(albumRes)} - ${sanitizeFilename(albumRes.title)}`)
+  await mkdir(outDir, { recursive: true })
+
+  const albumCoverRes = await ffetch(getAlbumCoverUrl(albumRes.cover)).arrayBuffer()
+  await writeFile(join(outDir, 'cover.jpg'), new Uint8Array(albumCoverRes))
+
+  const spinnies = new Spinnies()
+  spinnies.add('download', { text: 'downloading album...' })
+
+  const errors = new Map<number, Error>()
+  await downloadTrackList({
+    tracks: albumTracks,
+    albums: new Map([[albumRes.id, albumRes]]),
+    albumCoverPaths: new Map([[albumRes.id, join(outDir, 'cover.jpg')]]),
+    destination: outDir,
+    includeTrackNumber: true,
+    onDownloadStart(track) {
+      spinnies.add(`${track.id}`, { text: getTrackName(track) })
+      errors.delete(track.id)
+    },
+    onDownloadEnd(track, error) {
+      spinnies.remove(`${track.id}`)
+      if (error) {
+        errors.set(track.id, error)
+      }
+      spinnies.remove(`${track.id}`)
+    },
+  })
+
+  spinnies.succeed('download', { text: 'downloaded album' })
+
+  if (errors.size) {
+    console.error('errors:')
+    for (const [id, error] of errors) {
+      console.error(`  ${id}: ${error.message}`)
+    }
+  }
+}
+
 const url = process.argv[2] ?? await question('url or search > ')
 
 /* eslint-disable no-cond-assign */
@@ -327,48 +408,57 @@ if ((m = url.match(/\/track\/(\d+)/))) {
 
   await rm(tmpAlbumCoverPath)
 } else if ((m = url.match(/\/album\/(\d+)/))) {
-  const [albumRes, albumTracks] = await Promise.all([
+  await downloadAlbum(m[1])
-    ffetch(`https://tidal.com/v1/albums/${m[1]}`, { query: COMMON_QUERY }).parsedJson(TidalAlbum),
+} else if ((m = url.match(/\/artist\/(\d+)/))) {
-    fetchAlbumTracks(m[1]),
+  const withAppearsOn = (await question('include appears on albums? (y/N) > ')).toLowerCase() === 'y'
+
+  function fetchAlbumList(type: string): Promise<TidalAlbum[]> {
+    return fetchPaginated({
+      fetch: async (offset) => {
+        const r = await ffetch(`https://tidal.com/v2/artist/${type}/view-all`, {
+          query: {
+            itemId: m[1],
+            ...COMMON_QUERY,
+            platform: 'WEB',
+            limit: 50,
+            offset,
+          },
+          headers: {
+            'x-tidal-client-version': '2025.10.29',
+          },
+        }).parsedJson(z.object({
+          items: z.array(z.object({
+            type: z.literal('ALBUM'),
+            data: TidalAlbum,
+          })),
+        }))
+
+        return {
+          items: r.items.map(it => it.data),
+          hasMore: r.items.length === 50,
+        }
+      },
+    })
+  }
+
+  const [albums, singles, appearsOn] = await Promise.all([
+    fetchAlbumList('ARTIST_ALBUMS'),
+    fetchAlbumList('ARTIST_TOP_SINGLES'),
+    withAppearsOn ? fetchAlbumList('ARTIST_APPEARS_ON') : Promise.resolve([]),
   ])
 
-  console.log(`downloading album ${albumRes.title} with ${albumTracks.length} tracks`)
+  // concat and dedupe
+  const seenIds = new Set<number>()
+  const allAlbums: TidalAlbum[] = []
+  for (const album of [...albums, ...singles, ...appearsOn]) {
+    if (seenIds.has(album.id)) continue
+    seenIds.add(album.id)
+    allAlbums.push(album)
+  }
+  console.log('found %d albums', allAlbums.length)
 
-  const outDir = join('assets/tidal-dl', `${getTrackArtistString(albumRes)} - ${sanitizeFilename(albumRes.title)}`)
+  for (const album of allAlbums) {
-  await mkdir(outDir, { recursive: true })
+    await downloadAlbum(album)
-
-  const albumCoverRes = await ffetch(getAlbumCoverUrl(albumRes.cover)).arrayBuffer()
-  await writeFile(join(outDir, 'cover.jpg'), new Uint8Array(albumCoverRes))
-
-  const spinnies = new Spinnies()
-  spinnies.add('download', { text: 'downloading album...' })
-
-  const errors = new Map<number, Error>()
-  await downloadTrackList({
-    tracks: albumTracks,
-    albums: new Map([[albumRes.id, albumRes]]),
-    albumCoverPaths: new Map([[albumRes.id, join(outDir, 'cover.jpg')]]),
-    destination: outDir,
-    includeTrackNumber: true,
-    onDownloadStart(track) {
-      spinnies.add(`${track.id}`, { text: getTrackName(track) })
-    },
-    onDownloadEnd(track, error) {
-      spinnies.remove(`${track.id}`)
-      if (error) {
-        errors.set(track.id, error)
-      }
-      spinnies.remove(`${track.id}`)
-    },
-  })
-
-  spinnies.succeed('download', { text: 'downloaded album' })
-
-  if (errors.size) {
-    console.error('errors:')
-    for (const [id, error] of errors) {
-      console.error(`  ${id}: ${error.message}`)
-    }
   }
 } else {
   todo('unsupported url')

utils/media-metadata.ts

@@ -100,8 +100,10 @@ export function generateFfmpegMetadataFlags(metadata: Partial<Record<string, str
 }
 
 export async function pipeIntoProc(proc: ProcessPromise, stream: ReadableStream) {
-  const pipe = Readable.fromWeb(stream as any).pipe(proc.stdin)
+  const nodeStream = Readable.fromWeb(stream as any)
   await new Promise<void>((resolve, reject) => {
+    nodeStream.on('error', reject)
+    const pipe = nodeStream.pipe(proc.stdin)
     pipe.on('error', reject)
     pipe.on('finish', resolve)
   })