aria/teidesu-scripts
1
0
Fork 0
mirror of https://git.stupid.fish/teidesu/scripts.git synced 2025-07-28 02:32:11 +10:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: update public repo

Browse source
This commit is contained in:
desu-bot 2025-04-09 04:20:08 +00:00
parent 874e1952e3
commit ce9f435ef2
No known key found for this signature in database
3 changed files with 70 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

31
scripts/infra/navidrome-find-duplicates.ts
View file

@ -5,7 +5,7 @@ import { join } from 'node:path'
import kuromoji from 'kuromoji'
import { isKana, toRomaji } from 'wanakana'
import { fetchSongs, navidromeFfetch as ffetch } from '../../utils/navidrome.ts'
import { fetchSongs, fetchSongsIter } from '../../utils/navidrome.ts'
const WHITELIST_KEYS = new Set([
// actual different tracks with the same title
@ -53,8 +53,6 @@ function clean(s: string) {
return str
}
const CHUNK_SIZE = 1000
function getSongKey(song: NavidromeSong) {
return JSON.stringify([
clean(song.artist),
@ -64,23 +62,20 @@ function getSongKey(song: NavidromeSong) {
const seen = new Map<string, NavidromeSong[]>()
for (let offset = 0; ; offset += CHUNK_SIZE) {
const songs = await fetchSongs(offset, CHUNK_SIZE)
if (songs.length === 0) break
for (const song of songs) {
const key = getSongKey(song)
if (WHITELIST_KEYS.has(key)) continue
let arr = seen.get(key)
if (!arr) {
arr = []
seen.set(key, arr)
}
arr.push(song)
for await (const song of fetchSongsIter({
onChunkProcessed: (page, items) => {
console.log('⌛ fetched chunk %d (%d items)', page, items)
},
})) {
const key = getSongKey(song)
if (WHITELIST_KEYS.has(key)) continue
let arr = seen.get(key)
if (!arr) {
arr = []
seen.set(key, arr)
}
console.log('⌛ fetched chunk %d (%d items)', Math.floor(offset / CHUNK_SIZE), songs.length)
arr.push(song)
}
const keysSorted = Array.from(seen.keys()).sort()