Compare commits

...

2 commits

Author SHA1 Message Date
desu-bot
261c7eefa0
chore: update public repo 2025-09-21 19:26:47 +00:00
desu-bot
96ca247fcb
chore: update public repo 2025-09-14 21:52:13 +00:00
7 changed files with 484 additions and 133 deletions

View file

@ -12,7 +12,8 @@ import { FileCookieStore } from 'tough-cookie-file-store'
import { z } from 'zod'
import { $, question } from 'zx'
import { ffetch as ffetchBase } from '../../utils/fetch.ts'
import { sanitizeFilename } from '../../utils/fs.ts'
import { sanitizeFilename, writeWebStreamToFile } from '../../utils/fs.ts'
import { generateFfmpegMetadataFlags, pipeIntoProc, runMetaflac } from '../../utils/media-metadata.ts'
import { getEnv } from '../../utils/misc.ts'
const jar = new CookieJar(new FileCookieStore('./assets/deezer-cookies.json'))
@ -151,7 +152,7 @@ const GwTrack = z.object({
SNG_ID: z.string(),
SNG_TITLE: z.string(),
TRACK_NUMBER: z.string(),
VERSION: z.string(),
VERSION: z.string().optional(),
MD5_ORIGIN: z.string(),
FILESIZE_AAC_64: z.coerce.number(),
FILESIZE_MP3_64: z.coerce.number(),
@ -182,6 +183,8 @@ const GwAlbum = z.object({
ART_ID: z.string(),
ART_NAME: z.string(),
})),
TYPE: z.string().optional(), // "0" = single, "1" = normal album, "3" = ep
ROLE_ID: z.number().optional(), // 0 = own album, 5 = "featured on" album
COPYRIGHT: z.string(),
PRODUCER_LINE: z.string(),
DIGITAL_RELEASE_DATE: z.string(),
@ -471,112 +474,48 @@ async function downloadTrack(track: GwTrack, opts: {
'0:a',
'-c',
'copy',
'-metadata',
`title=${getTrackName(track)}`,
'-metadata',
`album=${track.ALB_TITLE}`,
'-metadata',
`year=${track.DIGITAL_RELEASE_DATE}`,
'-metadata',
`comment=ripped from deezer (id: ${track.SNG_ID})`,
'-metadata',
`track=${track.TRACK_NUMBER}`,
'-metadata',
`disc=${track.DISK_NUMBER}`,
...generateFfmpegMetadataFlags({
title: getTrackName(track),
album: opts.album?.ALB_TITLE ?? track.ALB_TITLE,
year: track.DIGITAL_RELEASE_DATE,
comment: `ripped from deezer (id: ${track.SNG_ID})`,
track: track.TRACK_NUMBER,
disc: track.DISK_NUMBER,
composer: track.SNG_CONTRIBUTORS?.composer,
artist: track.ARTISTS?.map(it => it.ART_NAME) ?? track.ART_NAME,
}),
filename,
]
if (opts.album) {
params.push('-metadata', `album=${opts.album.ALB_TITLE}`)
}
if (track.SNG_CONTRIBUTORS?.composer) {
for (const composer of track.SNG_CONTRIBUTORS.composer) {
params.push('-metadata', `composer=${composer}`)
}
}
if (track.ARTISTS?.length) {
for (const artist of track.ARTISTS) {
params.push('-metadata', `artist=${artist.ART_NAME}`)
}
} else {
params.push('-metadata', `artist=${track.ART_NAME}`)
}
if (lyricsLrc) {
await writeFile(`${opts.destination}.lrc`, lyricsLrc)
}
const proc = $`ffmpeg ${params}`
const pipe = Readable.fromWeb(decStream as any).pipe(proc.stdin)
await new Promise<void>((resolve, reject) => {
pipe.on('error', reject)
pipe.on('finish', resolve)
})
await pipeIntoProc(proc, decStream)
await proc
} else {
const fd = await open(filename, 'w+')
const writer = fd.createWriteStream()
await writeWebStreamToFile(decStream, filename)
for await (const chunk of decStream as any) {
writer.write(chunk)
}
writer.end()
await new Promise<void>((resolve, reject) => {
writer.on('error', reject)
writer.on('finish', resolve)
await runMetaflac({
path: filename,
tags: {
TITLE: getTrackName(track),
ALBUM: track.ALB_TITLE,
DATE: track.DIGITAL_RELEASE_DATE ?? asNonNull(opts.album?.DIGITAL_RELEASE_DATE),
DISCNUMBER: track.DISK_NUMBER,
TRACKNUMBER: track.TRACK_NUMBER,
COMMENT: `ripped from deezer (id: ${track.SNG_ID})`,
ARTIST: track.ARTISTS?.map(it => it.ART_NAME) ?? track.ART_NAME,
COMPOSER: track.SNG_CONTRIBUTORS?.composer,
MAIN_ARTIST: track.SNG_CONTRIBUTORS?.main_artist,
ISRC: track.ISRC,
PRODUCER: opts.album?.PRODUCER_LINE,
COPYRIGHT: opts.album?.COPYRIGHT,
LYRICS: lyricsLrc,
},
coverPath: albumCoverPath,
})
const params: string[] = [
'--remove-all-tags',
`--set-tag=TITLE=${getTrackName(track)}`,
`--set-tag=ALBUM=${track.ALB_TITLE}`,
`--set-tag=DATE=${track.DIGITAL_RELEASE_DATE ?? asNonNull(opts.album?.DIGITAL_RELEASE_DATE)}`,
`--set-tag=DISCNUMBER=${track.DISK_NUMBER}`,
`--set-tag=TRACKNUMBER=${track.TRACK_NUMBER}`,
`--set-tag=COMMENT=ripped from deezer (id: ${track.SNG_ID})`,
`--import-picture-from=${albumCoverPath}`,
]
if (track.ARTISTS) {
for (const artist of track.ARTISTS) {
params.push(`--set-tag=ARTIST=${artist.ART_NAME}`)
}
} else {
params.push(`--set-tag=ARTIST=${track.ART_NAME}`)
}
if (track.SNG_CONTRIBUTORS?.composer) {
for (const composer of track.SNG_CONTRIBUTORS.composer) {
params.push(`--set-tag=COMPOSER=${composer}`)
}
}
if (track.SNG_CONTRIBUTORS?.main_artist) {
for (const mainArtist of track.SNG_CONTRIBUTORS.main_artist) {
params.push(`--set-tag=MAIN_ARTIST=${mainArtist}`)
}
}
if (track.ISRC) {
params.push(`--set-tag=ISRC=${track.ISRC}`)
}
if (opts.album) {
params.push(`--set-tag=PRODUCER=${opts.album.PRODUCER_LINE}`)
params.push(`--set-tag=COPYRIGHT=${opts.album.COPYRIGHT}`)
}
if (lyricsLrc) {
params.push(`--set-tag=LYRICS=${lyricsLrc}`)
}
params.push(filename)
await $`metaflac ${params}`
}
await rm(albumCoverPath, { force: true })
@ -595,7 +534,7 @@ async function downloadTrackList(tracks: GwTrack[], opts: {
const isMultiDisc = tracks.some(it => it.DISK_NUMBER !== '1')
const firstTrackArtistString = getTrackArtistString(tracks[0])
const isVariousArtists = tracks.some(it => getTrackArtistString(it) !== firstTrackArtistString)
const isDifferentArtists = tracks.some(it => getTrackArtistString(it) !== firstTrackArtistString)
await asyncPool(tracks, async (track) => {
let filename = ''
@ -605,7 +544,7 @@ async function downloadTrackList(tracks: GwTrack[], opts: {
}
filename = `${track.TRACK_NUMBER.padStart(2, '0')}. `
}
if (isVariousArtists) {
if (isDifferentArtists) {
filename += `${getTrackArtistString(track)} - `
}
filename += `${getTrackName(track)}`
@ -636,7 +575,11 @@ const GwPageArtist = z.object({
}),
})
async function downloadArtist(artistId: string) {
async function downloadArtist(options: {
artistId: string
includeFeaturedAlbums?: boolean
}) {
const { artistId, includeFeaturedAlbums = false } = options
const artistInfo = await gwLightApi({
method: 'deezer.pageArtist',
token: userData.checkForm,
@ -681,6 +624,8 @@ async function downloadArtist(artistId: string) {
})
for (const alb of res.data) {
if (!includeFeaturedAlbums && alb.ROLE_ID === 5) continue
albums.push(alb)
trackCount += asNonNull(alb.SONGS).total
}
@ -692,7 +637,6 @@ async function downloadArtist(artistId: string) {
spinnies.succeed('collect', { text: `collected ${albums.length} albums with a total of ${trackCount} tracks` })
}
// fixme: "featured" albums/tracks (i.e. when main artist of the album is not the one we're dling) should have album artist name in its dirname
// fixme: singles should be saved in artist root dir
// todo: automatic musicbrainz matching
// todo: automatic genius/musixmatch matching for lyrics if unavailable directly from deezer
@ -713,11 +657,16 @@ async function downloadArtist(artistId: string) {
assert(tracks.total === asNonNull(alb.SONGS).total)
assert(tracks.data.length === asNonNull(alb.SONGS).total)
let folderName = alb.ALB_TITLE
if (alb.ROLE_ID === 5) {
folderName = `${artistInfo.DATA.ART_NAME} - ${folderName}`
}
await downloadTrackList(tracks.data, {
destination: join(
'assets/deezer-dl',
sanitizeFilename(artistInfo.DATA.ART_NAME),
sanitizeFilename(alb.ALB_TITLE),
sanitizeFilename(folderName),
),
album: alb,
poolLimit: 4,
@ -827,7 +776,11 @@ async function downloadByUri(uri: string) {
}
if (type === 'artist') {
await downloadArtist(id)
const includeFeaturedAlbums = await question('include featured albums? (y/N) > ')
await downloadArtist({
artistId: id,
includeFeaturedAlbums: includeFeaturedAlbums.toLowerCase() === 'y',
})
}
}

View file

@ -10,7 +10,7 @@ import { $, ProcessOutput, question } from 'zx'
import { downloadFile, ffetch as ffetchBase } from '../../utils/fetch.ts'
import { sanitizeFilename } from '../../utils/fs.ts'
import { chunks, getEnv } from '../../utils/misc.ts'
import { generateOpusImageBlob } from '../../utils/opus.ts'
import { generateOpusImageBlob } from '../../utils/media-metadata.ts'
const ffetchApi = ffetchBase.extend({
baseUrl: 'https://api-v2.soundcloud.com',

View file

@ -0,0 +1,223 @@
#!/usr/bin/env tsx
import { ffetch } from '../../utils/fetch.ts'
import { getEnv } from '../../utils/misc.ts'
// context: had a discussion in a group chat about which day of the week albums are usually released on, needed a way to find out
// the script is mostly vibe-coded but i have no intentions to run it more than once so who cares
interface SpotifyTrack {
track: {
id: string
name: string
album: {
id: string
name: string
release_date: string
release_date_precision: 'year' | 'month' | 'day'
}
artists: Array<{
name: string
}>
}
}
interface SpotifyAlbum {
id: string
name: string
release_date: string
release_date_precision: 'year' | 'month' | 'day'
artists: Array<{
name: string
}>
}
interface SpotifyResponse<T> {
items: T[]
next: string | null
total: number
}
class SpotifyClient {
private accessToken: string
private baseUrl = 'https://api.spotify.com/v1'
constructor(accessToken: string) {
this.accessToken = accessToken
}
private async makeRequest<T>(endpoint: string): Promise<T> {
const response = await ffetch(endpoint, {
baseUrl: this.baseUrl,
headers: {
'Authorization': `Bearer ${this.accessToken}`,
'Content-Type': 'application/json',
},
})
if (!response.ok) {
throw new Error(`Spotify API error: ${response.status} ${response.statusText}: ${await response.text()}`)
}
return response.json()
}
async getLikedTracks(): Promise<SpotifyTrack[]> {
const allTracks: SpotifyTrack[] = []
let url = '/me/tracks?limit=50'
while (url) {
const response = await this.makeRequest<SpotifyResponse<SpotifyTrack>>(url)
allTracks.push(...response.items)
console.log(`Fetched ${allTracks.length} out of ${response.total} tracks`)
url = response.next ? response.next.replace(this.baseUrl, '') : ''
}
return allTracks
}
async getAlbum(albumId: string): Promise<SpotifyAlbum> {
return this.makeRequest<SpotifyAlbum>(`/albums/${albumId}`)
}
}
interface DayStats {
[key: string]: {
count: number
albums: Array<{
name: string
artist: string
releaseDate: string
}>
}
}
function getDayOfWeek(dateString: string, precision: string): string {
if (precision === 'year') {
return 'Unknown (Year only)'
}
if (precision === 'month') {
return 'Unknown (Month only)'
}
try {
const date = new Date(dateString)
if (Number.isNaN(date.getTime())) {
return 'Unknown (Invalid date)'
}
const days = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']
return days[date.getDay()]
} catch (error) {
return 'Unknown (Parse error)'
}
}
async function main() {
const accessToken = getEnv('SPOTIFY_API_TOKEN')
if (!accessToken) {
console.error('Error: SPOTIFY_API_TOKEN environment variable is required')
process.exit(1)
}
console.log('🎵 Fetching your liked tracks from Spotify...')
const spotify = new SpotifyClient(accessToken)
try {
const likedTracks = await spotify.getLikedTracks()
console.log(`Found ${likedTracks.length} liked tracks`)
const processedAlbums = new Set<string>()
const dayStats: DayStats = {}
// First, count unique albums from tracks
const uniqueAlbumIds = new Set<string>()
for (const track of likedTracks) {
uniqueAlbumIds.add(track.track.album.id)
}
console.log(`📊 Analyzing ${uniqueAlbumIds.size} unique album release dates...`)
let processedCount = 0
let skippedCount = 0
for (const track of likedTracks) {
const albumId = track.track.album.id
// Skip if we've already processed this album
if (processedAlbums.has(albumId)) {
skippedCount++
continue
}
processedAlbums.add(albumId)
processedCount++
try {
// Get detailed album info
const album = await spotify.getAlbum(albumId)
const dayOfWeek = getDayOfWeek(album.release_date, album.release_date_precision)
if (!dayStats[dayOfWeek]) {
dayStats[dayOfWeek] = {
count: 0,
albums: [],
}
}
dayStats[dayOfWeek].count++
dayStats[dayOfWeek].albums.push({
name: album.name,
artist: album.artists.map(a => a.name).join(', '),
releaseDate: album.release_date,
})
// Progress reporting
if (processedCount % 10 === 0 || processedCount === uniqueAlbumIds.size) {
console.log(`Progress: ${processedCount}/${uniqueAlbumIds.size} albums processed (${skippedCount} skipped)`)
}
// Add a small delay to avoid rate limiting
await new Promise(resolve => setTimeout(resolve, 100))
} catch (error) {
console.warn(`Failed to fetch album info for ${track.track.album.name}: ${error}`)
}
}
console.log('\n📈 Album Release Day Statistics')
console.log('='.repeat(50))
// Sort by count (descending)
const sortedStats = Object.entries(dayStats)
.sort(([,a], [,b]) => b.count - a.count)
for (const [day, stats] of sortedStats) {
console.log(`\n${day}: ${stats.count} albums`)
console.log('-'.repeat(30))
// Show top 5 albums for this day
const topAlbums = stats.albums.slice(0, 5)
for (const album of topAlbums) {
console.log(`${album.name} by ${album.artist} (${album.releaseDate})`)
}
if (stats.albums.length > 5) {
console.log(` ... and ${stats.albums.length - 5} more`)
}
}
console.log('\n📊 Summary:')
console.log(`Total unique albums found: ${uniqueAlbumIds.size}`)
console.log(`Total unique albums analyzed: ${processedAlbums.size}`)
console.log(`Albums skipped (duplicates): ${skippedCount}`)
console.log(`Total liked tracks: ${likedTracks.length}`)
} catch (error) {
console.error('Error:', error)
process.exit(1)
}
}
await main()

View file

@ -0,0 +1,87 @@
import { randomBytes } from 'node:crypto'
import { faker } from '@faker-js/faker'
import { question } from 'zx'
import { ffetch } from '../../utils/fetch.ts'
// log in with your yandex account in the browser, then go to music.yandex.ru and open devtools
// find long ass string in "Cookie" header from the requests to music.yandex.ru, it must contain "Session_id" cookie.
// make sure to copy it completely (on firefox this requires toggling "Raw")
// looks something like: is_gdpr=0; is_gdpr=0; is_gdpr_b=COnCMBCR0wIoAg==; _yasc=ctfv6IPUcb+Lk+jqYr0thW1STKmQC5yB4IJUM5Gn....
const cookies = await question('music.yandex.ru cookies > ')
const parsed = new Map(cookies.split('; ').map((cookie) => {
const [name, value] = cookie.split('=')
return [name, value]
}))
if (!parsed.has('Session_id')) {
throw new Error('Session_id cookie not found')
}
const deviceId = randomBytes(16).toString('hex')
const uuid = randomBytes(16).toString('hex')
const genRequestId = () => `${uuid}${Math.floor(Date.now())}`
const query = {
manufacturer: 'Google',
model: 'Pixel 9 Pro XL',
app_platform: 'Android 16 (REL)',
am_version_name: '7.46.0(746003972)',
app_id: 'ru.yandex.music',
app_version_name: '2025.09.2 #114gpr',
am_app: 'ru.yandex.music 2025.09.2 #114gpr',
deviceid: deviceId,
device_id: deviceId,
uuid,
}
const res = await ffetch('https://mobileproxy.passport.yandex.net/1/bundle/oauth/token_by_sessionid', {
query: {
...query,
request_id: genRequestId(),
},
form: {
client_id: 'c0ebe342af7d48fbbbfcf2d2eedb8f9e',
client_secret: 'ad0a908f0aa341a182a37ecd75bc319e',
grant_type: 'sessionid',
host: 'yandex.ru',
},
headers: {
'Accept': '*/*',
'User-Agent': 'com.yandex.mobile.auth.sdk/7.46.0.746003972 (Google Pixel 9 Pro XL; Android 16) PassportSDK/7.46.0.746003972',
'Accept-Language': 'en-RU;q=1, ru-RU;q=0.9',
'Ya-Client-Host': 'passport.yandex.ru',
'Ya-Client-Cookie': cookies,
},
}).json() as any
if (res.status !== 'ok') {
console.error('Unexpected response:', res)
process.exit(1)
}
console.log('res', res)
const res2 = await ffetch('https://mobileproxy.passport.yandex.net/1/token', {
query: {
...query,
request_id: genRequestId(),
},
form: {
access_token: res.access_token,
client_id: '23cabbbdc6cd418abb4b39c32c41195d',
client_secret: '53bc75238f0c4d08a118e51fe9203300',
grant_type: 'x-token',
},
}).json() as any
if (!res2.access_token) {
console.error('Unexpected response:', res2)
process.exit(1)
}
console.log('res2', res2)
console.log('')
console.log('Your auth token is:')
console.log(res2.access_token)
console.log('Expires at:', new Date(Date.now() + res.expires_in * 1000).toLocaleString('ru-RU'))

View file

@ -21,3 +21,19 @@ export async function directoryExists(path: string): Promise<boolean> {
export function sanitizeFilename(filename: string) {
return filename.replace(/[/\\?%*:|"<>]/g, '_')
}
export async function writeWebStreamToFile(stream: ReadableStream<unknown>, path: string) {
const fd = await fsp.open(path, 'w+')
const writer = fd.createWriteStream()
for await (const chunk of stream as any) {
writer.write(chunk)
}
writer.end()
await new Promise<void>((resolve, reject) => {
writer.on('error', reject)
writer.on('finish', resolve)
})
}

102
utils/media-metadata.ts Normal file
View file

@ -0,0 +1,102 @@
import type { ProcessPromise } from 'zx'
import { Readable } from 'node:stream'
import { Bytes, write } from '@fuman/io'
import { $ } from 'zx'
export async function generateOpusImageBlob(image: Uint8Array) {
// todo we should probably not use ffprobe here but whatever lol
const proc = $`ffprobe -of json -v error -show_entries stream=codec_name,width,height pipe:0`
proc.stdin.write(image)
proc.stdin.end()
const json = await proc.json()
const img = json.streams[0]
// https://www.rfc-editor.org/rfc/rfc9639.html#section-8.8
const mime = img.codec_name === 'mjpeg' ? 'image/jpeg' : 'image/png'
const description = 'Cover Artwork'
const res = Bytes.alloc(image.length + 128)
write.uint32be(res, 3) // picture type = album cover
write.uint32be(res, mime.length)
write.rawString(res, mime)
write.uint32be(res, description.length)
write.rawString(res, description)
write.uint32be(res, img.width)
write.uint32be(res, img.height)
write.uint32be(res, 0) // color depth
write.uint32be(res, 0) // color index (unused, for gifs)
write.uint32be(res, image.length)
write.bytes(res, image)
return res.result()
}
export async function runMetaflac(options: {
path: string
tags: Partial<Record<
| 'TITLE'
| 'ARTIST'
| 'COMPOSER'
| 'ALBUM'
| 'DATE'
| 'DISCNUMBER'
| 'TRACKNUMBER'
| 'COMMENT'
| 'PRODUCER'
| 'COPYRIGHT'
| 'ISRC'
| 'LYRICS'
| 'MAIN_ARTIST',
string | number | string[] | null
>
>
coverPath?: string
}) {
const params: string[] = [
'--remove-all-tags',
]
for (const [key, value] of Object.entries(options.tags)) {
if (value == null) continue
if (Array.isArray(value)) {
for (const v of value) {
params.push(`--set-tag=${key}=${v}`)
}
} else {
params.push(`--set-tag=${key}=${value}`)
}
}
if (options.coverPath) {
params.push(`--import-picture-from=${options.coverPath}`)
}
params.push(options.path)
await $`metaflac ${params}`
}
export function generateFfmpegMetadataFlags(metadata: Partial<Record<string, string | string[]>>) {
const res: string[] = []
for (const [key, value] of Object.entries(metadata)) {
if (value == null) continue
if (Array.isArray(value)) {
for (const v of value) {
res.push('-metadata', `${key}=${v}`)
}
} else {
res.push('-metadata', `${key}=${value}`)
}
}
return res
}
export async function pipeIntoProc(proc: ProcessPromise, stream: ReadableStream) {
const pipe = Readable.fromWeb(stream as any).pipe(proc.stdin)
await new Promise<void>((resolve, reject) => {
pipe.on('error', reject)
pipe.on('finish', resolve)
})
}

View file

@ -1,30 +0,0 @@
import { Bytes, write } from '@fuman/io'
import { $ } from 'zx'
export async function generateOpusImageBlob(image: Uint8Array) {
// todo we should probably not use ffprobe here but whatever lol
const proc = $`ffprobe -of json -v error -show_entries stream=codec_name,width,height pipe:0`
proc.stdin.write(image)
proc.stdin.end()
const json = await proc.json()
const img = json.streams[0]
// https://www.rfc-editor.org/rfc/rfc9639.html#section-8.8
const mime = img.codec_name === 'mjpeg' ? 'image/jpeg' : 'image/png'
const description = 'Cover Artwork'
const res = Bytes.alloc(image.length + 128)
write.uint32be(res, 3) // picture type = album cover
write.uint32be(res, mime.length)
write.rawString(res, mime)
write.uint32be(res, description.length)
write.rawString(res, description)
write.uint32be(res, img.width)
write.uint32be(res, img.height)
write.uint32be(res, 0) // color depth
write.uint32be(res, 0) // color index (unused, for gifs)
write.uint32be(res, image.length)
write.bytes(res, image)
return res.result()
}