From 25d88cb28bb4f0e55549638c138fb435664d97b7 Mon Sep 17 00:00:00 2001 From: desu-bot Date: Sun, 11 May 2025 22:29:57 +0000 Subject: [PATCH] chore: update public repo --- scripts/media/deezer-dl.ts | 344 ++++++++++++++++++++++++++++++++++--- 1 file changed, 319 insertions(+), 25 deletions(-) diff --git a/scripts/media/deezer-dl.ts b/scripts/media/deezer-dl.ts index c30dee7..9797c3b 100644 --- a/scripts/media/deezer-dl.ts +++ b/scripts/media/deezer-dl.ts @@ -4,8 +4,9 @@ import { dirname, join } from 'node:path' import { Readable } from 'node:stream' import { TransformStream } from 'node:stream/web' import { Bytes, read, write } from '@fuman/io' -import { base64, hex, iter, utf8 } from '@fuman/utils' +import { asNonNull, assert, asyncPool, base64, hex, iter, unknownToError, utf8 } from '@fuman/utils' import { Blowfish } from 'egoroof-blowfish' +import Spinnies from 'spinnies' import { CookieJar } from 'tough-cookie' import { FileCookieStore } from 'tough-cookie-file-store' import { z } from 'zod' @@ -130,7 +131,7 @@ const GwTrack = z.object({ ART_ID: z.string(), ART_NAME: z.string(), ARTIST_IS_DUMMY: z.boolean().optional(), - DIGITAL_RELEASE_DATE: z.string(), + DIGITAL_RELEASE_DATE: z.string().optional(), DISK_NUMBER: z.string(), DURATION: z.string(), EXPLICIT_LYRICS: z.string(), @@ -138,24 +139,18 @@ const GwTrack = z.object({ EXPLICIT_LYRICS_STATUS: z.number(), EXPLICIT_COVER_STATUS: z.number(), }), - GENRE_ID: z.string(), - HIERARCHICAL_TITLE: z.string().optional(), - ISRC: z.string(), + ISRC: z.string().optional(), LYRICS_ID: z.number(), - PHYSICAL_RELEASE_DATE: z.string(), PROVIDER_ID: z.string(), - RANK: z.string().optional(), - SMARTRADIO: z.number(), SNG_CONTRIBUTORS: z.object({ main_artist: z.array(z.string()), author: z.array(z.string()), composer: z.array(z.string()), + featuring: z.array(z.string()), }).partial().optional(), SNG_ID: z.string(), SNG_TITLE: z.string(), - STATUS: z.number(), TRACK_NUMBER: z.string(), - USER_ID: z.number(), VERSION: z.string(), MD5_ORIGIN: z.string(), FILESIZE_AAC_64: z.coerce.number(), @@ -168,7 +163,6 @@ const GwTrack = z.object({ FILESIZE_MP4_RA3: z.coerce.number(), FILESIZE_FLAC: z.coerce.number(), FILESIZE: z.coerce.number(), - GAIN: z.nullable(z.coerce.number()), MEDIA_VERSION: z.string(), TRACK_TOKEN: z.string(), TRACK_TOKEN_EXPIRE: z.number(), @@ -181,6 +175,22 @@ const GwTrack = z.object({ }) type GwTrack = z.infer +const GwAlbum = z.object({ + ALB_ID: z.string(), + ALB_TITLE: z.string(), + ARTISTS: z.array(z.object({ + ART_ID: z.string(), + ART_NAME: z.string(), + })), + COPYRIGHT: z.string(), + PRODUCER_LINE: z.string(), + DIGITAL_RELEASE_DATE: z.string(), + SONGS: z.object({ + total: z.number(), + }).optional(), +}) +type GwAlbum = z.infer + const userData = await gwLightApi({ method: 'deezer.getUserData', result: z.object({ @@ -237,10 +247,10 @@ class BlowfishDecryptTransform implements Transformer { for (let i = 0; i < 16; i++) { bfKey[i] = trackIdMd5[i].charCodeAt(0) - ^ trackIdMd5[i + 16].charCodeAt(0) - ^ (i % 2 ? BLOWFISH_SALT_2 : BLOWFISH_SALT_1)[ - 7 - Math.floor(i / 2) - ] + ^ trackIdMd5[i + 16].charCodeAt(0) + ^ (i % 2 ? BLOWFISH_SALT_2 : BLOWFISH_SALT_1)[ + 7 - Math.floor(i / 2) + ] } this.cipher = new Blowfish(bfKey, Blowfish.MODE.CBC, Blowfish.PADDING.NULL) @@ -288,8 +298,22 @@ class BlowfishDecryptTransform implements Transformer { } } +function getTrackArtistString(track: GwTrack) { + if (track.ARTISTS) return track.ARTISTS.map(it => it.ART_NAME).join(', ').slice(0, 100) + return track.ART_NAME +} + +function getTrackName(track: GwTrack) { + let name = track.SNG_TITLE + if (track.VERSION) { + name += ` ${track.VERSION}` + } + return name +} + async function downloadTrack(track: GwTrack, opts: { destination: string + album?: GwAlbum }) { const albumUrl = `https://cdn-images.dzcdn.net/images/cover/${track.ALB_PICTURE}/1500x1500-000000-80-0-0.jpg` const [getUrlRes, albumAb, lyricsRes] = await Promise.all([ @@ -365,23 +389,40 @@ async function downloadTrack(track: GwTrack, opts: { '-c', 'copy', '-metadata', - `title=${track.SNG_TITLE}`, - '-metadata', - `artist=${track.ART_NAME}`, + `title=${getTrackName(track)}`, '-metadata', `album=${track.ALB_TITLE}`, '-metadata', `year=${track.DIGITAL_RELEASE_DATE}`, '-metadata', `comment=ripped from deezer (id: ${track.SNG_ID})`, + '-metadata', + `track=${track.TRACK_NUMBER}`, + '-metadata', + `disc=${track.DISK_NUMBER}`, filename, ] + if (opts.album) { + params.push('-metadata', `album=${opts.album.ALB_TITLE}`) + } + + if (track.SNG_CONTRIBUTORS?.composer) { + for (const composer of track.SNG_CONTRIBUTORS.composer) { + params.push('-metadata', `composer=${composer}`) + } + } + + if (track.ARTISTS?.length) { + for (const artist of track.ARTISTS) { + params.push('-metadata', `artist=${artist.ART_NAME}`) + } + } else { + params.push('-metadata', `artist=${track.ART_NAME}`) + } + if (lyricsLrc) { - params.push( - '-metadata', - `lyrics=${lyricsLrc}`, - ) + await writeFile(`${opts.destination}.lrc`, lyricsLrc) } const proc = $`ffmpeg ${params}` @@ -408,14 +449,44 @@ async function downloadTrack(track: GwTrack, opts: { const params: string[] = [ '--remove-all-tags', - `--set-tag=TITLE=${track.SNG_TITLE}`, - `--set-tag=ARTIST=${track.ART_NAME}`, + `--set-tag=TITLE=${getTrackName(track)}`, `--set-tag=ALBUM=${track.ALB_TITLE}`, - `--set-tag=DATE=${track.DIGITAL_RELEASE_DATE}`, + `--set-tag=DATE=${track.DIGITAL_RELEASE_DATE ?? asNonNull(opts.album?.DIGITAL_RELEASE_DATE)}`, + `--set-tag=DISCNUMBER=${track.DISK_NUMBER}`, + `--set-tag=TRACKNUMBER=${track.TRACK_NUMBER}`, `--set-tag=COMMENT=ripped from deezer (id: ${track.SNG_ID})`, `--import-picture-from=${albumCoverPath}`, ] + if (track.ARTISTS) { + for (const artist of track.ARTISTS) { + params.push(`--set-tag=ARTIST=${artist.ART_NAME}`) + } + } else { + params.push(`--set-tag=ARTIST=${track.ART_NAME}`) + } + + if (track.SNG_CONTRIBUTORS?.composer) { + for (const composer of track.SNG_CONTRIBUTORS.composer) { + params.push(`--set-tag=COMPOSER=${composer}`) + } + } + + if (track.SNG_CONTRIBUTORS?.main_artist) { + for (const mainArtist of track.SNG_CONTRIBUTORS.main_artist) { + params.push(`--set-tag=MAIN_ARTIST=${mainArtist}`) + } + } + + if (track.ISRC) { + params.push(`--set-tag=ISRC=${track.ISRC}`) + } + + if (opts.album) { + params.push(`--set-tag=PRODUCER=${opts.album.PRODUCER_LINE}`) + params.push(`--set-tag=COPYRIGHT=${opts.album.COPYRIGHT}`) + } + params.push(filename) await $`metaflac ${params}` @@ -424,6 +495,159 @@ async function downloadTrack(track: GwTrack, opts: { await rm(albumCoverPath, { force: true }) } +async function downloadTrackList(tracks: GwTrack[], opts: { + album?: GwAlbum + poolLimit?: number + destination: string + includeTrackNumber?: boolean + onDownloadStart?: (track: GwTrack) => void + onDownloadEnd?: (track: GwTrack, error: Error | null) => void +}) { + await mkdir(opts.destination, { recursive: true }) + + const isMultiDisc = tracks.some(it => it.DISK_NUMBER !== '1') + + const firstTrackArtistString = getTrackArtistString(tracks[0]) + const isVariousArtists = tracks.some(it => getTrackArtistString(it) !== firstTrackArtistString) + + await asyncPool(tracks, async (track) => { + let filename = '' + if (opts.includeTrackNumber) { + if (isMultiDisc) { + filename = `${track.DISK_NUMBER}-` + } + filename = `${track.TRACK_NUMBER.padStart(2, '0')}. ` + } + if (isVariousArtists) { + filename += `${getTrackArtistString(track)} - ` + } + filename += `${getTrackName(track)}` + + const filenamePath = join(opts.destination, sanitizeFilename(filename)) + + opts.onDownloadStart?.(track) + + try { + await downloadTrack(track, { + destination: filenamePath, + album: opts.album, + }) + opts.onDownloadEnd?.(track, null) + } catch (e) { + opts.onDownloadEnd?.(track, unknownToError(e)) + } + }, { limit: opts.poolLimit }) +} + +const GwPageArtist = z.object({ + DATA: z.object({ + ART_NAME: z.string(), + }), + ALBUMS: z.object({ + data: z.array(GwAlbum), + total: z.number(), + }), +}) + +async function downloadArtist(artistId: string) { + const artistInfo = await gwLightApi({ + method: 'deezer.pageArtist', + token: userData.checkForm, + options: { + art_id: artistId, + lang: 'us', + }, + result: z.object({ + DATA: z.object({ + ART_NAME: z.string(), + }), + ALBUMS: z.object({ + data: z.array(GwAlbum), + total: z.number(), + }), + }), + }) + + const albums: GwAlbum[] = artistInfo.ALBUMS.data + + let trackCount = 0 + const spinnies = new Spinnies() + + if (artistInfo.ALBUMS.total > albums.length) { + // fetch the rest + spinnies.add('collect', { text: 'collecting albums...' }) + let offset = albums.length + while (true) { + const res = await gwLightApi({ + method: 'album.getDiscography', + token: userData.checkForm, + options: { + art_id: artistId, + nb: 25, + nb_songs: 0, + start: offset, + }, + result: z.object({ + data: z.array(GwAlbum), + total: z.number(), + }), + }) + + for (const alb of res.data) { + albums.push(alb) + trackCount += asNonNull(alb.SONGS).total + } + + if (res.total <= offset) break + offset += 25 + } + + spinnies.succeed('collect', { text: `collected ${albums.length} albums with a total of ${trackCount} tracks` }) + } + + // fixme: singles should always contain artist name and be saved in artist root dir + // fixme: "featured" albums (i.e. when main artist of the album is not the one we're dling) should have album artist name in its dirname + // todo: automatic musicbrainz matching + + await asyncPool(albums, async (alb) => { + const tracks = await gwLightApi({ + method: 'song.getListByAlbum', + token: userData.checkForm, + options: { + alb_id: alb.ALB_ID, + nb: -1, + }, + result: z.object({ + data: z.array(GwTrack), + total: z.number(), + }), + }) + assert(tracks.total === asNonNull(alb.SONGS).total) + assert(tracks.data.length === asNonNull(alb.SONGS).total) + + await downloadTrackList(tracks.data, { + destination: join( + 'assets/deezer-dl', + sanitizeFilename(artistInfo.DATA.ART_NAME), + sanitizeFilename(alb.ALB_TITLE), + ), + album: alb, + poolLimit: 4, + includeTrackNumber: true, + onDownloadStart(track) { + spinnies.add(`${track.SNG_ID}`, { text: track.SNG_TITLE }) + }, + onDownloadEnd(track, error) { + if (error) { + spinnies.fail(`${track.SNG_ID}`, { text: error.message }) + } else { + spinnies.remove(`${track.SNG_ID}`) + } + }, + }) + }, { limit: 4 }) +} + async function downloadByUri(uri: string) { const [type, id] = uri.split(':') @@ -447,6 +671,76 @@ async function downloadByUri(uri: string) { destination: join('assets/deezer-dl', sanitizeFilename(filename)), }) } + + if (type === 'album') { + const album = await gwLightApi({ + method: 'deezer.pageAlbum', + token: userData.checkForm, + options: { + alb_id: id, + lang: 'us', + }, + result: z.object({ + DATA: GwAlbum, + SONGS: z.object({ + data: z.array(GwTrack), + total: z.number(), + }), + }), + }) + + const tracks = album.SONGS.data + if (tracks.length < album.SONGS.total) { + // fetch the rest + const res = await gwLightApi({ + method: 'song.getListByAlbum', + token: userData.checkForm, + options: { + alb_id: id, + nb: -1, + start: tracks.length, + }, + result: z.object({ + data: z.array(GwTrack), + total: z.number(), + }), + }) + tracks.push(...res.data) + + assert(tracks.length === album.SONGS.total) + } + + const spinnies = new Spinnies() + spinnies.add('download', { text: 'downloading album...' }) + + await downloadTrackList(tracks, { + destination: join( + 'assets/deezer-dl', + sanitizeFilename( + `${album.DATA.ARTISTS.map(it => it.ART_NAME).join(', ').slice(0, 100)} - ${album.DATA.ALB_TITLE}`, + ), + ), + includeTrackNumber: true, + poolLimit: 8, + album: album.DATA, + onDownloadStart(track) { + spinnies.add(`${track.SNG_ID}`, { text: track.SNG_TITLE }) + }, + onDownloadEnd(track, error) { + if (error) { + spinnies.fail(`${track.SNG_ID}`, { text: error.stack }) + } else { + spinnies.remove(`${track.SNG_ID}`) + } + }, + }) + + spinnies.succeed('download', { text: 'downloaded album' }) + } + + if (type === 'artist') { + await downloadArtist(id) + } } console.log('logged in as %s', userData.USER.BLOG_NAME)