teidesu-scripts/scripts/misc/shikimori/clubs.ts

import { asyncPool } from '@fuman/utils'
import Database from 'better-sqlite3'
import { counterIter, ffetchShiki } from './utils.ts'

const db = new Database('assets/shikimori.db')
db.pragma('journal_mode = WAL')
db.exec(`
  create table if not exists clubs (
    id integer primary key,
    data text not null
  );
`)

const insertQuery = db.prepare('insert into clubs (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')

// collect clubs ids

const ids: Set<number> = new Set()

const pageCounter = counterIter(1)

await asyncPool(pageCounter.iter, async (page) => {
  const data = await ffetchShiki('/api/clubs', {
    query: { page, limit: 50 },
    validateResponse: false,
  }).json<any>()
  if (!data.length) {
    pageCounter.end()
    return
  }

  for (const club of data) {
    ids.add(club.id)
  }
}, { limit: 16 })

console.log('collected %d clubs', ids.size)

await asyncPool(ids, async (id, idx) => {
  if (idx % 100 === 0) {
    console.log('currently at %d', idx)
  }

  const clubData = await ffetchShiki(`/api/clubs/${id}`).json<any>()
  if (clubData.code === 404) {
    return
  }
  insertQuery.run(id, JSON.stringify(clubData))
}, { limit: 64 })