diff --git a/scripts/media/deezer-dl.ts b/scripts/media/deezer-dl.ts index 96923db..73ae653 100644 --- a/scripts/media/deezer-dl.ts +++ b/scripts/media/deezer-dl.ts @@ -12,7 +12,8 @@ import { FileCookieStore } from 'tough-cookie-file-store' import { z } from 'zod' import { $, question } from 'zx' import { ffetch as ffetchBase } from '../../utils/fetch.ts' -import { sanitizeFilename } from '../../utils/fs.ts' +import { sanitizeFilename, writeWebStreamToFile } from '../../utils/fs.ts' +import { generateFfmpegMetadataFlags, pipeIntoProc, runMetaflac } from '../../utils/media-metadata.ts' import { getEnv } from '../../utils/misc.ts' const jar = new CookieJar(new FileCookieStore('./assets/deezer-cookies.json')) @@ -151,7 +152,7 @@ const GwTrack = z.object({ SNG_ID: z.string(), SNG_TITLE: z.string(), TRACK_NUMBER: z.string(), - VERSION: z.string(), + VERSION: z.string().optional(), MD5_ORIGIN: z.string(), FILESIZE_AAC_64: z.coerce.number(), FILESIZE_MP3_64: z.coerce.number(), @@ -182,6 +183,8 @@ const GwAlbum = z.object({ ART_ID: z.string(), ART_NAME: z.string(), })), + TYPE: z.string().optional(), // "0" = single, "1" = normal album, "3" = ep + ROLE_ID: z.number().optional(), // 0 = own album, 5 = "featured on" album COPYRIGHT: z.string(), PRODUCER_LINE: z.string(), DIGITAL_RELEASE_DATE: z.string(), @@ -471,112 +474,48 @@ async function downloadTrack(track: GwTrack, opts: { '0:a', '-c', 'copy', - '-metadata', - `title=${getTrackName(track)}`, - '-metadata', - `album=${track.ALB_TITLE}`, - '-metadata', - `year=${track.DIGITAL_RELEASE_DATE}`, - '-metadata', - `comment=ripped from deezer (id: ${track.SNG_ID})`, - '-metadata', - `track=${track.TRACK_NUMBER}`, - '-metadata', - `disc=${track.DISK_NUMBER}`, + ...generateFfmpegMetadataFlags({ + title: getTrackName(track), + album: opts.album?.ALB_TITLE ?? track.ALB_TITLE, + year: track.DIGITAL_RELEASE_DATE, + comment: `ripped from deezer (id: ${track.SNG_ID})`, + track: track.TRACK_NUMBER, + disc: track.DISK_NUMBER, + composer: track.SNG_CONTRIBUTORS?.composer, + artist: track.ARTISTS?.map(it => it.ART_NAME) ?? track.ART_NAME, + }), filename, ] - if (opts.album) { - params.push('-metadata', `album=${opts.album.ALB_TITLE}`) - } - - if (track.SNG_CONTRIBUTORS?.composer) { - for (const composer of track.SNG_CONTRIBUTORS.composer) { - params.push('-metadata', `composer=${composer}`) - } - } - - if (track.ARTISTS?.length) { - for (const artist of track.ARTISTS) { - params.push('-metadata', `artist=${artist.ART_NAME}`) - } - } else { - params.push('-metadata', `artist=${track.ART_NAME}`) - } - if (lyricsLrc) { await writeFile(`${opts.destination}.lrc`, lyricsLrc) } const proc = $`ffmpeg ${params}` - const pipe = Readable.fromWeb(decStream as any).pipe(proc.stdin) - await new Promise((resolve, reject) => { - pipe.on('error', reject) - pipe.on('finish', resolve) - }) + await pipeIntoProc(proc, decStream) await proc } else { - const fd = await open(filename, 'w+') - const writer = fd.createWriteStream() + await writeWebStreamToFile(decStream, filename) - for await (const chunk of decStream as any) { - writer.write(chunk) - } - - writer.end() - - await new Promise((resolve, reject) => { - writer.on('error', reject) - writer.on('finish', resolve) + await runMetaflac({ + path: filename, + tags: { + TITLE: getTrackName(track), + ALBUM: track.ALB_TITLE, + DATE: track.DIGITAL_RELEASE_DATE ?? asNonNull(opts.album?.DIGITAL_RELEASE_DATE), + DISCNUMBER: track.DISK_NUMBER, + TRACKNUMBER: track.TRACK_NUMBER, + COMMENT: `ripped from deezer (id: ${track.SNG_ID})`, + ARTIST: track.ARTISTS?.map(it => it.ART_NAME) ?? track.ART_NAME, + COMPOSER: track.SNG_CONTRIBUTORS?.composer, + MAIN_ARTIST: track.SNG_CONTRIBUTORS?.main_artist, + ISRC: track.ISRC, + PRODUCER: opts.album?.PRODUCER_LINE, + COPYRIGHT: opts.album?.COPYRIGHT, + LYRICS: lyricsLrc, + }, + coverPath: albumCoverPath, }) - - const params: string[] = [ - '--remove-all-tags', - `--set-tag=TITLE=${getTrackName(track)}`, - `--set-tag=ALBUM=${track.ALB_TITLE}`, - `--set-tag=DATE=${track.DIGITAL_RELEASE_DATE ?? asNonNull(opts.album?.DIGITAL_RELEASE_DATE)}`, - `--set-tag=DISCNUMBER=${track.DISK_NUMBER}`, - `--set-tag=TRACKNUMBER=${track.TRACK_NUMBER}`, - `--set-tag=COMMENT=ripped from deezer (id: ${track.SNG_ID})`, - `--import-picture-from=${albumCoverPath}`, - ] - - if (track.ARTISTS) { - for (const artist of track.ARTISTS) { - params.push(`--set-tag=ARTIST=${artist.ART_NAME}`) - } - } else { - params.push(`--set-tag=ARTIST=${track.ART_NAME}`) - } - - if (track.SNG_CONTRIBUTORS?.composer) { - for (const composer of track.SNG_CONTRIBUTORS.composer) { - params.push(`--set-tag=COMPOSER=${composer}`) - } - } - - if (track.SNG_CONTRIBUTORS?.main_artist) { - for (const mainArtist of track.SNG_CONTRIBUTORS.main_artist) { - params.push(`--set-tag=MAIN_ARTIST=${mainArtist}`) - } - } - - if (track.ISRC) { - params.push(`--set-tag=ISRC=${track.ISRC}`) - } - - if (opts.album) { - params.push(`--set-tag=PRODUCER=${opts.album.PRODUCER_LINE}`) - params.push(`--set-tag=COPYRIGHT=${opts.album.COPYRIGHT}`) - } - - if (lyricsLrc) { - params.push(`--set-tag=LYRICS=${lyricsLrc}`) - } - - params.push(filename) - - await $`metaflac ${params}` } await rm(albumCoverPath, { force: true }) @@ -595,7 +534,7 @@ async function downloadTrackList(tracks: GwTrack[], opts: { const isMultiDisc = tracks.some(it => it.DISK_NUMBER !== '1') const firstTrackArtistString = getTrackArtistString(tracks[0]) - const isVariousArtists = tracks.some(it => getTrackArtistString(it) !== firstTrackArtistString) + const isDifferentArtists = tracks.some(it => getTrackArtistString(it) !== firstTrackArtistString) await asyncPool(tracks, async (track) => { let filename = '' @@ -605,7 +544,7 @@ async function downloadTrackList(tracks: GwTrack[], opts: { } filename = `${track.TRACK_NUMBER.padStart(2, '0')}. ` } - if (isVariousArtists) { + if (isDifferentArtists) { filename += `${getTrackArtistString(track)} - ` } filename += `${getTrackName(track)}` @@ -636,7 +575,11 @@ const GwPageArtist = z.object({ }), }) -async function downloadArtist(artistId: string) { +async function downloadArtist(options: { + artistId: string + includeFeaturedAlbums?: boolean +}) { + const { artistId, includeFeaturedAlbums = false } = options const artistInfo = await gwLightApi({ method: 'deezer.pageArtist', token: userData.checkForm, @@ -681,6 +624,8 @@ async function downloadArtist(artistId: string) { }) for (const alb of res.data) { + if (!includeFeaturedAlbums && alb.ROLE_ID === 5) continue + albums.push(alb) trackCount += asNonNull(alb.SONGS).total } @@ -692,7 +637,6 @@ async function downloadArtist(artistId: string) { spinnies.succeed('collect', { text: `collected ${albums.length} albums with a total of ${trackCount} tracks` }) } - // fixme: "featured" albums/tracks (i.e. when main artist of the album is not the one we're dling) should have album artist name in its dirname // fixme: singles should be saved in artist root dir // todo: automatic musicbrainz matching // todo: automatic genius/musixmatch matching for lyrics if unavailable directly from deezer @@ -713,11 +657,16 @@ async function downloadArtist(artistId: string) { assert(tracks.total === asNonNull(alb.SONGS).total) assert(tracks.data.length === asNonNull(alb.SONGS).total) + let folderName = alb.ALB_TITLE + if (alb.ROLE_ID === 5) { + folderName = `${artistInfo.DATA.ART_NAME} - ${folderName}` + } + await downloadTrackList(tracks.data, { destination: join( 'assets/deezer-dl', sanitizeFilename(artistInfo.DATA.ART_NAME), - sanitizeFilename(alb.ALB_TITLE), + sanitizeFilename(folderName), ), album: alb, poolLimit: 4, @@ -827,7 +776,11 @@ async function downloadByUri(uri: string) { } if (type === 'artist') { - await downloadArtist(id) + const includeFeaturedAlbums = await question('include featured albums? (y/N) > ') + await downloadArtist({ + artistId: id, + includeFeaturedAlbums: includeFeaturedAlbums.toLowerCase() === 'y', + }) } } diff --git a/scripts/media/soundcloud-dl.ts b/scripts/media/soundcloud-dl.ts index ac459dc..e931f92 100644 --- a/scripts/media/soundcloud-dl.ts +++ b/scripts/media/soundcloud-dl.ts @@ -10,7 +10,7 @@ import { $, ProcessOutput, question } from 'zx' import { downloadFile, ffetch as ffetchBase } from '../../utils/fetch.ts' import { sanitizeFilename } from '../../utils/fs.ts' import { chunks, getEnv } from '../../utils/misc.ts' -import { generateOpusImageBlob } from '../../utils/opus.ts' +import { generateOpusImageBlob } from '../../utils/media-metadata.ts' const ffetchApi = ffetchBase.extend({ baseUrl: 'https://api-v2.soundcloud.com', diff --git a/scripts/misc/yamusic-token.ts b/scripts/misc/yamusic-token.ts new file mode 100644 index 0000000..d248d26 --- /dev/null +++ b/scripts/misc/yamusic-token.ts @@ -0,0 +1,87 @@ +import { randomBytes } from 'node:crypto' +import { faker } from '@faker-js/faker' +import { question } from 'zx' +import { ffetch } from '../../utils/fetch.ts' + +// log in with your yandex account in the browser, then go to music.yandex.ru and open devtools +// find long ass string in "Cookie" header from the requests to music.yandex.ru, it must contain "Session_id" cookie. +// make sure to copy it completely (on firefox this requires toggling "Raw") +// looks something like: is_gdpr=0; is_gdpr=0; is_gdpr_b=COnCMBCR0wIoAg==; _yasc=ctfv6IPUcb+Lk+jqYr0thW1STKmQC5yB4IJUM5Gn.... +const cookies = await question('music.yandex.ru cookies > ') + +const parsed = new Map(cookies.split('; ').map((cookie) => { + const [name, value] = cookie.split('=') + return [name, value] +})) + +if (!parsed.has('Session_id')) { + throw new Error('Session_id cookie not found') +} + +const deviceId = randomBytes(16).toString('hex') +const uuid = randomBytes(16).toString('hex') +const genRequestId = () => `${uuid}${Math.floor(Date.now())}` +const query = { + manufacturer: 'Google', + model: 'Pixel 9 Pro XL', + app_platform: 'Android 16 (REL)', + am_version_name: '7.46.0(746003972)', + app_id: 'ru.yandex.music', + app_version_name: '2025.09.2 #114gpr', + am_app: 'ru.yandex.music 2025.09.2 #114gpr', + deviceid: deviceId, + device_id: deviceId, + uuid, +} + +const res = await ffetch('https://mobileproxy.passport.yandex.net/1/bundle/oauth/token_by_sessionid', { + query: { + ...query, + request_id: genRequestId(), + }, + form: { + client_id: 'c0ebe342af7d48fbbbfcf2d2eedb8f9e', + client_secret: 'ad0a908f0aa341a182a37ecd75bc319e', + grant_type: 'sessionid', + host: 'yandex.ru', + }, + headers: { + 'Accept': '*/*', + 'User-Agent': 'com.yandex.mobile.auth.sdk/7.46.0.746003972 (Google Pixel 9 Pro XL; Android 16) PassportSDK/7.46.0.746003972', + 'Accept-Language': 'en-RU;q=1, ru-RU;q=0.9', + 'Ya-Client-Host': 'passport.yandex.ru', + 'Ya-Client-Cookie': cookies, + }, +}).json() as any + +if (res.status !== 'ok') { + console.error('Unexpected response:', res) + process.exit(1) +} + +console.log('res', res) + +const res2 = await ffetch('https://mobileproxy.passport.yandex.net/1/token', { + query: { + ...query, + request_id: genRequestId(), + }, + form: { + access_token: res.access_token, + client_id: '23cabbbdc6cd418abb4b39c32c41195d', + client_secret: '53bc75238f0c4d08a118e51fe9203300', + grant_type: 'x-token', + }, +}).json() as any + +if (!res2.access_token) { + console.error('Unexpected response:', res2) + process.exit(1) +} + +console.log('res2', res2) + +console.log('') +console.log('Your auth token is:') +console.log(res2.access_token) +console.log('Expires at:', new Date(Date.now() + res.expires_in * 1000).toLocaleString('ru-RU')) diff --git a/utils/fs.ts b/utils/fs.ts index 11ab17f..b2d9bcc 100644 --- a/utils/fs.ts +++ b/utils/fs.ts @@ -21,3 +21,19 @@ export async function directoryExists(path: string): Promise { export function sanitizeFilename(filename: string) { return filename.replace(/[/\\?%*:|"<>]/g, '_') } + +export async function writeWebStreamToFile(stream: ReadableStream, path: string) { + const fd = await fsp.open(path, 'w+') + const writer = fd.createWriteStream() + + for await (const chunk of stream as any) { + writer.write(chunk) + } + + writer.end() + + await new Promise((resolve, reject) => { + writer.on('error', reject) + writer.on('finish', resolve) + }) +} diff --git a/utils/media-metadata.ts b/utils/media-metadata.ts new file mode 100644 index 0000000..3b6ad13 --- /dev/null +++ b/utils/media-metadata.ts @@ -0,0 +1,102 @@ +import type { ProcessPromise } from 'zx' +import { Readable } from 'node:stream' +import { Bytes, write } from '@fuman/io' +import { $ } from 'zx' + +export async function generateOpusImageBlob(image: Uint8Array) { + // todo we should probably not use ffprobe here but whatever lol + const proc = $`ffprobe -of json -v error -show_entries stream=codec_name,width,height pipe:0` + proc.stdin.write(image) + proc.stdin.end() + const json = await proc.json() + + const img = json.streams[0] + // https://www.rfc-editor.org/rfc/rfc9639.html#section-8.8 + const mime = img.codec_name === 'mjpeg' ? 'image/jpeg' : 'image/png' + const description = 'Cover Artwork' + + const res = Bytes.alloc(image.length + 128) + write.uint32be(res, 3) // picture type = album cover + write.uint32be(res, mime.length) + write.rawString(res, mime) + write.uint32be(res, description.length) + write.rawString(res, description) + write.uint32be(res, img.width) + write.uint32be(res, img.height) + write.uint32be(res, 0) // color depth + write.uint32be(res, 0) // color index (unused, for gifs) + write.uint32be(res, image.length) + write.bytes(res, image) + + return res.result() +} + +export async function runMetaflac(options: { + path: string + tags: Partial + > + coverPath?: string +}) { + const params: string[] = [ + '--remove-all-tags', + ] + + for (const [key, value] of Object.entries(options.tags)) { + if (value == null) continue + if (Array.isArray(value)) { + for (const v of value) { + params.push(`--set-tag=${key}=${v}`) + } + } else { + params.push(`--set-tag=${key}=${value}`) + } + } + + if (options.coverPath) { + params.push(`--import-picture-from=${options.coverPath}`) + } + + params.push(options.path) + + await $`metaflac ${params}` +} + +export function generateFfmpegMetadataFlags(metadata: Partial>) { + const res: string[] = [] + + for (const [key, value] of Object.entries(metadata)) { + if (value == null) continue + if (Array.isArray(value)) { + for (const v of value) { + res.push('-metadata', `${key}=${v}`) + } + } else { + res.push('-metadata', `${key}=${value}`) + } + } + + return res +} + +export async function pipeIntoProc(proc: ProcessPromise, stream: ReadableStream) { + const pipe = Readable.fromWeb(stream as any).pipe(proc.stdin) + await new Promise((resolve, reject) => { + pipe.on('error', reject) + pipe.on('finish', resolve) + }) +} diff --git a/utils/opus.ts b/utils/opus.ts deleted file mode 100644 index 06ff13b..0000000 --- a/utils/opus.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { Bytes, write } from '@fuman/io' -import { $ } from 'zx' - -export async function generateOpusImageBlob(image: Uint8Array) { - // todo we should probably not use ffprobe here but whatever lol - const proc = $`ffprobe -of json -v error -show_entries stream=codec_name,width,height pipe:0` - proc.stdin.write(image) - proc.stdin.end() - const json = await proc.json() - - const img = json.streams[0] - // https://www.rfc-editor.org/rfc/rfc9639.html#section-8.8 - const mime = img.codec_name === 'mjpeg' ? 'image/jpeg' : 'image/png' - const description = 'Cover Artwork' - - const res = Bytes.alloc(image.length + 128) - write.uint32be(res, 3) // picture type = album cover - write.uint32be(res, mime.length) - write.rawString(res, mime) - write.uint32be(res, description.length) - write.rawString(res, description) - write.uint32be(res, img.width) - write.uint32be(res, img.height) - write.uint32be(res, 0) // color depth - write.uint32be(res, 0) // color index (unused, for gifs) - write.uint32be(res, image.length) - write.bytes(res, image) - - return res.result() -}