teidesu-scripts/scripts/misc/shikimori/people.ts

import { asyncPool } from '@fuman/utils'
import Database from 'better-sqlite3'
import { counterIter, ffetchShiki } from './utils.ts'

const db = new Database('assets/shikimori.db')
db.pragma('journal_mode = WAL')
db.exec(`
  create table if not exists people (
    id integer primary key,
    data text not null
  );
`)

const insertQuery = db.prepare('insert into people (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')

// find maxId with binary search
let maxIdPage = 20000
let maxIdPageStart = 1
let maxId = 0
while (true) {
  const midPage = Math.floor((maxIdPageStart + maxIdPage) / 2)
  console.log('trying page %d', midPage)
  const res = await ffetchShiki.post('/api/graphql', {
    json: {
      query: `{people(page: ${midPage}, limit: 50) { id }}`,
    },
  }).json<any>()
  const items = res.data.people
  if (!items.length) {
    maxIdPage = midPage - 1
    continue
  }

  if (maxIdPageStart === midPage) {
    maxId = Math.max(...items.map(item => item.id))
    break
  } else {
    maxIdPageStart = midPage
  }
}

console.log('max id: %d', maxId)

const counter = counterIter(1, maxId)

await asyncPool(counter.iter, async (id) => {
  if (id % 1000 === 0) {
    console.log('currently at %d', id)
  }
  const data = await ffetchShiki(`/api/people/${id}`, {
    validateResponse: false,
  }).json<any>()

  if (data.code === 404) {
    return
  }

  insertQuery.run(id, JSON.stringify(data))
}, { limit: 64 })