teidesu-scripts/scripts/misc/shikimori/comments.ts

import { asyncPool } from '@fuman/utils'
import Database from 'better-sqlite3'
import { counterIter, ffetchShiki } from './utils.ts'

const db = new Database('assets/shikimori.db')
db.pragma('journal_mode = WAL')
db.exec(`
  create table if not exists comments (
    id integer primary key,
    data text not null
  );
`)

const insertQuery = db.prepare('insert into comments (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')

const counter = counterIter(11312000)
let consequent404 = 0
await asyncPool(counter.iter, async (id) => {
  if (id % 1000 === 0) {
    console.log('currently at %d', id)
  }
  const data = await ffetchShiki(`/api/comments/${id}`, {
    validateResponse: false,
  }).json<any>()

  if (data.code === 404) {
    consequent404++
    if (consequent404 > 10_000) {
      counter.end()
      console.log('10k consequent 404-s, stopping')
    }
    return
  }

  consequent404 = 0
  insertQuery.run(id, JSON.stringify(data))
}, { limit: 64 })