teidesu-scripts/scripts/misc/shikimori/animes.ts

import { asyncPool } from '@fuman/utils'
import Database from 'better-sqlite3'
import { counterIter, ffetchShiki } from './utils.ts'

const isManga = process.argv[2] === 'manga'
const isRanobe = process.argv[2] === 'ranobe'

const collection = isManga ? 'mangas' : isRanobe ? 'ranobe' : 'animes'

const db = new Database('assets/shikimori.db')
db.exec(`
  create table if not exists ${collection} (
    id integer primary key,
    data text not null
  );
  create table if not exists ${collection}_related (
    id integer primary key,
    data text not null
  );
`)

const insertQuery = db.prepare(`insert into ${collection} (id, data) values (?, ?) on conflict (id) do update set data = excluded.data`)
const insertRelatedQuery = db.prepare(`insert into ${collection}_related (id, data) values (?, ?) on conflict (id) do update set data = excluded.data`)

const maxId = await ffetchShiki(`/api/${collection}?order=id_desc`).json<any>().then(res => res[0].id)
console.log('max id: %d', maxId)

const counter = counterIter(1, maxId)

await asyncPool(counter.iter, async (id) => {
  if (id % 1000 === 0) {
    console.log('currently at %d', id)
  }
  // const data = await ffetchShiki(`/api/${collection}/${id}`, {
  //   validateResponse: false,
  // }).json<any>()

  // if (data.code === 404) {
  //   return
  // }

  // insertQuery.run(id, JSON.stringify(data))

  const data = await ffetchShiki(`/api/${collection}/${id}/related`, {
    validateResponse: false,
  }).json<any>()

  if (data.code === 404) {
    return
  }

  insertRelatedQuery.run(id, JSON.stringify(data))
}, { limit: 64 })