diff --git a/scripts/media/deezer-dl.ts b/scripts/media/deezer-dl.ts index 73ae653..96923db 100644 --- a/scripts/media/deezer-dl.ts +++ b/scripts/media/deezer-dl.ts @@ -12,8 +12,7 @@ import { FileCookieStore } from 'tough-cookie-file-store' import { z } from 'zod' import { $, question } from 'zx' import { ffetch as ffetchBase } from '../../utils/fetch.ts' -import { sanitizeFilename, writeWebStreamToFile } from '../../utils/fs.ts' -import { generateFfmpegMetadataFlags, pipeIntoProc, runMetaflac } from '../../utils/media-metadata.ts' +import { sanitizeFilename } from '../../utils/fs.ts' import { getEnv } from '../../utils/misc.ts' const jar = new CookieJar(new FileCookieStore('./assets/deezer-cookies.json')) @@ -152,7 +151,7 @@ const GwTrack = z.object({ SNG_ID: z.string(), SNG_TITLE: z.string(), TRACK_NUMBER: z.string(), - VERSION: z.string().optional(), + VERSION: z.string(), MD5_ORIGIN: z.string(), FILESIZE_AAC_64: z.coerce.number(), FILESIZE_MP3_64: z.coerce.number(), @@ -183,8 +182,6 @@ const GwAlbum = z.object({ ART_ID: z.string(), ART_NAME: z.string(), })), - TYPE: z.string().optional(), // "0" = single, "1" = normal album, "3" = ep - ROLE_ID: z.number().optional(), // 0 = own album, 5 = "featured on" album COPYRIGHT: z.string(), PRODUCER_LINE: z.string(), DIGITAL_RELEASE_DATE: z.string(), @@ -474,48 +471,112 @@ async function downloadTrack(track: GwTrack, opts: { '0:a', '-c', 'copy', - ...generateFfmpegMetadataFlags({ - title: getTrackName(track), - album: opts.album?.ALB_TITLE ?? track.ALB_TITLE, - year: track.DIGITAL_RELEASE_DATE, - comment: `ripped from deezer (id: ${track.SNG_ID})`, - track: track.TRACK_NUMBER, - disc: track.DISK_NUMBER, - composer: track.SNG_CONTRIBUTORS?.composer, - artist: track.ARTISTS?.map(it => it.ART_NAME) ?? track.ART_NAME, - }), + '-metadata', + `title=${getTrackName(track)}`, + '-metadata', + `album=${track.ALB_TITLE}`, + '-metadata', + `year=${track.DIGITAL_RELEASE_DATE}`, + '-metadata', + `comment=ripped from deezer (id: ${track.SNG_ID})`, + '-metadata', + `track=${track.TRACK_NUMBER}`, + '-metadata', + `disc=${track.DISK_NUMBER}`, filename, ] + if (opts.album) { + params.push('-metadata', `album=${opts.album.ALB_TITLE}`) + } + + if (track.SNG_CONTRIBUTORS?.composer) { + for (const composer of track.SNG_CONTRIBUTORS.composer) { + params.push('-metadata', `composer=${composer}`) + } + } + + if (track.ARTISTS?.length) { + for (const artist of track.ARTISTS) { + params.push('-metadata', `artist=${artist.ART_NAME}`) + } + } else { + params.push('-metadata', `artist=${track.ART_NAME}`) + } + if (lyricsLrc) { await writeFile(`${opts.destination}.lrc`, lyricsLrc) } const proc = $`ffmpeg ${params}` - await pipeIntoProc(proc, decStream) + const pipe = Readable.fromWeb(decStream as any).pipe(proc.stdin) + await new Promise((resolve, reject) => { + pipe.on('error', reject) + pipe.on('finish', resolve) + }) await proc } else { - await writeWebStreamToFile(decStream, filename) + const fd = await open(filename, 'w+') + const writer = fd.createWriteStream() - await runMetaflac({ - path: filename, - tags: { - TITLE: getTrackName(track), - ALBUM: track.ALB_TITLE, - DATE: track.DIGITAL_RELEASE_DATE ?? asNonNull(opts.album?.DIGITAL_RELEASE_DATE), - DISCNUMBER: track.DISK_NUMBER, - TRACKNUMBER: track.TRACK_NUMBER, - COMMENT: `ripped from deezer (id: ${track.SNG_ID})`, - ARTIST: track.ARTISTS?.map(it => it.ART_NAME) ?? track.ART_NAME, - COMPOSER: track.SNG_CONTRIBUTORS?.composer, - MAIN_ARTIST: track.SNG_CONTRIBUTORS?.main_artist, - ISRC: track.ISRC, - PRODUCER: opts.album?.PRODUCER_LINE, - COPYRIGHT: opts.album?.COPYRIGHT, - LYRICS: lyricsLrc, - }, - coverPath: albumCoverPath, + for await (const chunk of decStream as any) { + writer.write(chunk) + } + + writer.end() + + await new Promise((resolve, reject) => { + writer.on('error', reject) + writer.on('finish', resolve) }) + + const params: string[] = [ + '--remove-all-tags', + `--set-tag=TITLE=${getTrackName(track)}`, + `--set-tag=ALBUM=${track.ALB_TITLE}`, + `--set-tag=DATE=${track.DIGITAL_RELEASE_DATE ?? asNonNull(opts.album?.DIGITAL_RELEASE_DATE)}`, + `--set-tag=DISCNUMBER=${track.DISK_NUMBER}`, + `--set-tag=TRACKNUMBER=${track.TRACK_NUMBER}`, + `--set-tag=COMMENT=ripped from deezer (id: ${track.SNG_ID})`, + `--import-picture-from=${albumCoverPath}`, + ] + + if (track.ARTISTS) { + for (const artist of track.ARTISTS) { + params.push(`--set-tag=ARTIST=${artist.ART_NAME}`) + } + } else { + params.push(`--set-tag=ARTIST=${track.ART_NAME}`) + } + + if (track.SNG_CONTRIBUTORS?.composer) { + for (const composer of track.SNG_CONTRIBUTORS.composer) { + params.push(`--set-tag=COMPOSER=${composer}`) + } + } + + if (track.SNG_CONTRIBUTORS?.main_artist) { + for (const mainArtist of track.SNG_CONTRIBUTORS.main_artist) { + params.push(`--set-tag=MAIN_ARTIST=${mainArtist}`) + } + } + + if (track.ISRC) { + params.push(`--set-tag=ISRC=${track.ISRC}`) + } + + if (opts.album) { + params.push(`--set-tag=PRODUCER=${opts.album.PRODUCER_LINE}`) + params.push(`--set-tag=COPYRIGHT=${opts.album.COPYRIGHT}`) + } + + if (lyricsLrc) { + params.push(`--set-tag=LYRICS=${lyricsLrc}`) + } + + params.push(filename) + + await $`metaflac ${params}` } await rm(albumCoverPath, { force: true }) @@ -534,7 +595,7 @@ async function downloadTrackList(tracks: GwTrack[], opts: { const isMultiDisc = tracks.some(it => it.DISK_NUMBER !== '1') const firstTrackArtistString = getTrackArtistString(tracks[0]) - const isDifferentArtists = tracks.some(it => getTrackArtistString(it) !== firstTrackArtistString) + const isVariousArtists = tracks.some(it => getTrackArtistString(it) !== firstTrackArtistString) await asyncPool(tracks, async (track) => { let filename = '' @@ -544,7 +605,7 @@ async function downloadTrackList(tracks: GwTrack[], opts: { } filename = `${track.TRACK_NUMBER.padStart(2, '0')}. ` } - if (isDifferentArtists) { + if (isVariousArtists) { filename += `${getTrackArtistString(track)} - ` } filename += `${getTrackName(track)}` @@ -575,11 +636,7 @@ const GwPageArtist = z.object({ }), }) -async function downloadArtist(options: { - artistId: string - includeFeaturedAlbums?: boolean -}) { - const { artistId, includeFeaturedAlbums = false } = options +async function downloadArtist(artistId: string) { const artistInfo = await gwLightApi({ method: 'deezer.pageArtist', token: userData.checkForm, @@ -624,8 +681,6 @@ async function downloadArtist(options: { }) for (const alb of res.data) { - if (!includeFeaturedAlbums && alb.ROLE_ID === 5) continue - albums.push(alb) trackCount += asNonNull(alb.SONGS).total } @@ -637,6 +692,7 @@ async function downloadArtist(options: { spinnies.succeed('collect', { text: `collected ${albums.length} albums with a total of ${trackCount} tracks` }) } + // fixme: "featured" albums/tracks (i.e. when main artist of the album is not the one we're dling) should have album artist name in its dirname // fixme: singles should be saved in artist root dir // todo: automatic musicbrainz matching // todo: automatic genius/musixmatch matching for lyrics if unavailable directly from deezer @@ -657,16 +713,11 @@ async function downloadArtist(options: { assert(tracks.total === asNonNull(alb.SONGS).total) assert(tracks.data.length === asNonNull(alb.SONGS).total) - let folderName = alb.ALB_TITLE - if (alb.ROLE_ID === 5) { - folderName = `${artistInfo.DATA.ART_NAME} - ${folderName}` - } - await downloadTrackList(tracks.data, { destination: join( 'assets/deezer-dl', sanitizeFilename(artistInfo.DATA.ART_NAME), - sanitizeFilename(folderName), + sanitizeFilename(alb.ALB_TITLE), ), album: alb, poolLimit: 4, @@ -776,11 +827,7 @@ async function downloadByUri(uri: string) { } if (type === 'artist') { - const includeFeaturedAlbums = await question('include featured albums? (y/N) > ') - await downloadArtist({ - artistId: id, - includeFeaturedAlbums: includeFeaturedAlbums.toLowerCase() === 'y', - }) + await downloadArtist(id) } } diff --git a/scripts/media/soundcloud-dl.ts b/scripts/media/soundcloud-dl.ts index e931f92..ac459dc 100644 --- a/scripts/media/soundcloud-dl.ts +++ b/scripts/media/soundcloud-dl.ts @@ -10,7 +10,7 @@ import { $, ProcessOutput, question } from 'zx' import { downloadFile, ffetch as ffetchBase } from '../../utils/fetch.ts' import { sanitizeFilename } from '../../utils/fs.ts' import { chunks, getEnv } from '../../utils/misc.ts' -import { generateOpusImageBlob } from '../../utils/media-metadata.ts' +import { generateOpusImageBlob } from '../../utils/opus.ts' const ffetchApi = ffetchBase.extend({ baseUrl: 'https://api-v2.soundcloud.com', diff --git a/scripts/misc/spotify-albums-weekday-stats.ts b/scripts/misc/spotify-albums-weekday-stats.ts deleted file mode 100644 index 6b537f5..0000000 --- a/scripts/misc/spotify-albums-weekday-stats.ts +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env tsx - -import { ffetch } from '../../utils/fetch.ts' -import { getEnv } from '../../utils/misc.ts' - -// context: had a discussion in a group chat about which day of the week albums are usually released on, needed a way to find out -// the script is mostly vibe-coded but i have no intentions to run it more than once so who cares - -interface SpotifyTrack { - track: { - id: string - name: string - album: { - id: string - name: string - release_date: string - release_date_precision: 'year' | 'month' | 'day' - } - artists: Array<{ - name: string - }> - } -} - -interface SpotifyAlbum { - id: string - name: string - release_date: string - release_date_precision: 'year' | 'month' | 'day' - artists: Array<{ - name: string - }> -} - -interface SpotifyResponse { - items: T[] - next: string | null - total: number -} - -class SpotifyClient { - private accessToken: string - private baseUrl = 'https://api.spotify.com/v1' - - constructor(accessToken: string) { - this.accessToken = accessToken - } - - private async makeRequest(endpoint: string): Promise { - const response = await ffetch(endpoint, { - baseUrl: this.baseUrl, - headers: { - 'Authorization': `Bearer ${this.accessToken}`, - 'Content-Type': 'application/json', - }, - }) - - if (!response.ok) { - throw new Error(`Spotify API error: ${response.status} ${response.statusText}: ${await response.text()}`) - } - - return response.json() - } - - async getLikedTracks(): Promise { - const allTracks: SpotifyTrack[] = [] - let url = '/me/tracks?limit=50' - - while (url) { - const response = await this.makeRequest>(url) - allTracks.push(...response.items) - console.log(`Fetched ${allTracks.length} out of ${response.total} tracks`) - url = response.next ? response.next.replace(this.baseUrl, '') : '' - } - - return allTracks - } - - async getAlbum(albumId: string): Promise { - return this.makeRequest(`/albums/${albumId}`) - } -} - -interface DayStats { - [key: string]: { - count: number - albums: Array<{ - name: string - artist: string - releaseDate: string - }> - } -} - -function getDayOfWeek(dateString: string, precision: string): string { - if (precision === 'year') { - return 'Unknown (Year only)' - } - - if (precision === 'month') { - return 'Unknown (Month only)' - } - - try { - const date = new Date(dateString) - if (Number.isNaN(date.getTime())) { - return 'Unknown (Invalid date)' - } - - const days = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'] - return days[date.getDay()] - } catch (error) { - return 'Unknown (Parse error)' - } -} - -async function main() { - const accessToken = getEnv('SPOTIFY_API_TOKEN') - - if (!accessToken) { - console.error('Error: SPOTIFY_API_TOKEN environment variable is required') - process.exit(1) - } - - console.log('šŸŽµ Fetching your liked tracks from Spotify...') - - const spotify = new SpotifyClient(accessToken) - - try { - const likedTracks = await spotify.getLikedTracks() - console.log(`Found ${likedTracks.length} liked tracks`) - - const processedAlbums = new Set() - const dayStats: DayStats = {} - - // First, count unique albums from tracks - const uniqueAlbumIds = new Set() - for (const track of likedTracks) { - uniqueAlbumIds.add(track.track.album.id) - } - - console.log(`šŸ“Š Analyzing ${uniqueAlbumIds.size} unique album release dates...`) - - let processedCount = 0 - let skippedCount = 0 - - for (const track of likedTracks) { - const albumId = track.track.album.id - - // Skip if we've already processed this album - if (processedAlbums.has(albumId)) { - skippedCount++ - continue - } - - processedAlbums.add(albumId) - processedCount++ - - try { - // Get detailed album info - const album = await spotify.getAlbum(albumId) - const dayOfWeek = getDayOfWeek(album.release_date, album.release_date_precision) - - if (!dayStats[dayOfWeek]) { - dayStats[dayOfWeek] = { - count: 0, - albums: [], - } - } - - dayStats[dayOfWeek].count++ - dayStats[dayOfWeek].albums.push({ - name: album.name, - artist: album.artists.map(a => a.name).join(', '), - releaseDate: album.release_date, - }) - - // Progress reporting - if (processedCount % 10 === 0 || processedCount === uniqueAlbumIds.size) { - console.log(`Progress: ${processedCount}/${uniqueAlbumIds.size} albums processed (${skippedCount} skipped)`) - } - - // Add a small delay to avoid rate limiting - await new Promise(resolve => setTimeout(resolve, 100)) - } catch (error) { - console.warn(`Failed to fetch album info for ${track.track.album.name}: ${error}`) - } - } - - console.log('\nšŸ“ˆ Album Release Day Statistics') - console.log('='.repeat(50)) - - // Sort by count (descending) - const sortedStats = Object.entries(dayStats) - .sort(([,a], [,b]) => b.count - a.count) - - for (const [day, stats] of sortedStats) { - console.log(`\n${day}: ${stats.count} albums`) - console.log('-'.repeat(30)) - - // Show top 5 albums for this day - const topAlbums = stats.albums.slice(0, 5) - for (const album of topAlbums) { - console.log(` ā€¢ ${album.name} by ${album.artist} (${album.releaseDate})`) - } - - if (stats.albums.length > 5) { - console.log(` ... and ${stats.albums.length - 5} more`) - } - } - - console.log('\nšŸ“Š Summary:') - console.log(`Total unique albums found: ${uniqueAlbumIds.size}`) - console.log(`Total unique albums analyzed: ${processedAlbums.size}`) - console.log(`Albums skipped (duplicates): ${skippedCount}`) - console.log(`Total liked tracks: ${likedTracks.length}`) - } catch (error) { - console.error('Error:', error) - process.exit(1) - } -} - -await main() diff --git a/scripts/misc/yamusic-token.ts b/scripts/misc/yamusic-token.ts deleted file mode 100644 index d248d26..0000000 --- a/scripts/misc/yamusic-token.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { randomBytes } from 'node:crypto' -import { faker } from '@faker-js/faker' -import { question } from 'zx' -import { ffetch } from '../../utils/fetch.ts' - -// log in with your yandex account in the browser, then go to music.yandex.ru and open devtools -// find long ass string in "Cookie" header from the requests to music.yandex.ru, it must contain "Session_id" cookie. -// make sure to copy it completely (on firefox this requires toggling "Raw") -// looks something like: is_gdpr=0; is_gdpr=0; is_gdpr_b=COnCMBCR0wIoAg==; _yasc=ctfv6IPUcb+Lk+jqYr0thW1STKmQC5yB4IJUM5Gn.... -const cookies = await question('music.yandex.ru cookies > ') - -const parsed = new Map(cookies.split('; ').map((cookie) => { - const [name, value] = cookie.split('=') - return [name, value] -})) - -if (!parsed.has('Session_id')) { - throw new Error('Session_id cookie not found') -} - -const deviceId = randomBytes(16).toString('hex') -const uuid = randomBytes(16).toString('hex') -const genRequestId = () => `${uuid}${Math.floor(Date.now())}` -const query = { - manufacturer: 'Google', - model: 'Pixel 9 Pro XL', - app_platform: 'Android 16 (REL)', - am_version_name: '7.46.0(746003972)', - app_id: 'ru.yandex.music', - app_version_name: '2025.09.2 #114gpr', - am_app: 'ru.yandex.music 2025.09.2 #114gpr', - deviceid: deviceId, - device_id: deviceId, - uuid, -} - -const res = await ffetch('https://mobileproxy.passport.yandex.net/1/bundle/oauth/token_by_sessionid', { - query: { - ...query, - request_id: genRequestId(), - }, - form: { - client_id: 'c0ebe342af7d48fbbbfcf2d2eedb8f9e', - client_secret: 'ad0a908f0aa341a182a37ecd75bc319e', - grant_type: 'sessionid', - host: 'yandex.ru', - }, - headers: { - 'Accept': '*/*', - 'User-Agent': 'com.yandex.mobile.auth.sdk/7.46.0.746003972 (Google Pixel 9 Pro XL; Android 16) PassportSDK/7.46.0.746003972', - 'Accept-Language': 'en-RU;q=1, ru-RU;q=0.9', - 'Ya-Client-Host': 'passport.yandex.ru', - 'Ya-Client-Cookie': cookies, - }, -}).json() as any - -if (res.status !== 'ok') { - console.error('Unexpected response:', res) - process.exit(1) -} - -console.log('res', res) - -const res2 = await ffetch('https://mobileproxy.passport.yandex.net/1/token', { - query: { - ...query, - request_id: genRequestId(), - }, - form: { - access_token: res.access_token, - client_id: '23cabbbdc6cd418abb4b39c32c41195d', - client_secret: '53bc75238f0c4d08a118e51fe9203300', - grant_type: 'x-token', - }, -}).json() as any - -if (!res2.access_token) { - console.error('Unexpected response:', res2) - process.exit(1) -} - -console.log('res2', res2) - -console.log('') -console.log('Your auth token is:') -console.log(res2.access_token) -console.log('Expires at:', new Date(Date.now() + res.expires_in * 1000).toLocaleString('ru-RU')) diff --git a/utils/fs.ts b/utils/fs.ts index b2d9bcc..11ab17f 100644 --- a/utils/fs.ts +++ b/utils/fs.ts @@ -21,19 +21,3 @@ export async function directoryExists(path: string): Promise { export function sanitizeFilename(filename: string) { return filename.replace(/[/\\?%*:|"<>]/g, '_') } - -export async function writeWebStreamToFile(stream: ReadableStream, path: string) { - const fd = await fsp.open(path, 'w+') - const writer = fd.createWriteStream() - - for await (const chunk of stream as any) { - writer.write(chunk) - } - - writer.end() - - await new Promise((resolve, reject) => { - writer.on('error', reject) - writer.on('finish', resolve) - }) -} diff --git a/utils/media-metadata.ts b/utils/media-metadata.ts deleted file mode 100644 index 3b6ad13..0000000 --- a/utils/media-metadata.ts +++ /dev/null @@ -1,102 +0,0 @@ -import type { ProcessPromise } from 'zx' -import { Readable } from 'node:stream' -import { Bytes, write } from '@fuman/io' -import { $ } from 'zx' - -export async function generateOpusImageBlob(image: Uint8Array) { - // todo we should probably not use ffprobe here but whatever lol - const proc = $`ffprobe -of json -v error -show_entries stream=codec_name,width,height pipe:0` - proc.stdin.write(image) - proc.stdin.end() - const json = await proc.json() - - const img = json.streams[0] - // https://www.rfc-editor.org/rfc/rfc9639.html#section-8.8 - const mime = img.codec_name === 'mjpeg' ? 'image/jpeg' : 'image/png' - const description = 'Cover Artwork' - - const res = Bytes.alloc(image.length + 128) - write.uint32be(res, 3) // picture type = album cover - write.uint32be(res, mime.length) - write.rawString(res, mime) - write.uint32be(res, description.length) - write.rawString(res, description) - write.uint32be(res, img.width) - write.uint32be(res, img.height) - write.uint32be(res, 0) // color depth - write.uint32be(res, 0) // color index (unused, for gifs) - write.uint32be(res, image.length) - write.bytes(res, image) - - return res.result() -} - -export async function runMetaflac(options: { - path: string - tags: Partial - > - coverPath?: string -}) { - const params: string[] = [ - '--remove-all-tags', - ] - - for (const [key, value] of Object.entries(options.tags)) { - if (value == null) continue - if (Array.isArray(value)) { - for (const v of value) { - params.push(`--set-tag=${key}=${v}`) - } - } else { - params.push(`--set-tag=${key}=${value}`) - } - } - - if (options.coverPath) { - params.push(`--import-picture-from=${options.coverPath}`) - } - - params.push(options.path) - - await $`metaflac ${params}` -} - -export function generateFfmpegMetadataFlags(metadata: Partial>) { - const res: string[] = [] - - for (const [key, value] of Object.entries(metadata)) { - if (value == null) continue - if (Array.isArray(value)) { - for (const v of value) { - res.push('-metadata', `${key}=${v}`) - } - } else { - res.push('-metadata', `${key}=${value}`) - } - } - - return res -} - -export async function pipeIntoProc(proc: ProcessPromise, stream: ReadableStream) { - const pipe = Readable.fromWeb(stream as any).pipe(proc.stdin) - await new Promise((resolve, reject) => { - pipe.on('error', reject) - pipe.on('finish', resolve) - }) -} diff --git a/utils/opus.ts b/utils/opus.ts new file mode 100644 index 0000000..06ff13b --- /dev/null +++ b/utils/opus.ts @@ -0,0 +1,30 @@ +import { Bytes, write } from '@fuman/io' +import { $ } from 'zx' + +export async function generateOpusImageBlob(image: Uint8Array) { + // todo we should probably not use ffprobe here but whatever lol + const proc = $`ffprobe -of json -v error -show_entries stream=codec_name,width,height pipe:0` + proc.stdin.write(image) + proc.stdin.end() + const json = await proc.json() + + const img = json.streams[0] + // https://www.rfc-editor.org/rfc/rfc9639.html#section-8.8 + const mime = img.codec_name === 'mjpeg' ? 'image/jpeg' : 'image/png' + const description = 'Cover Artwork' + + const res = Bytes.alloc(image.length + 128) + write.uint32be(res, 3) // picture type = album cover + write.uint32be(res, mime.length) + write.rawString(res, mime) + write.uint32be(res, description.length) + write.rawString(res, description) + write.uint32be(res, img.width) + write.uint32be(res, img.height) + write.uint32be(res, 0) // color depth + write.uint32be(res, 0) // color index (unused, for gifs) + write.uint32be(res, image.length) + write.bytes(res, image) + + return res.result() +}