chore: update public repo

scripts/misc/shikimori/comments.ts

@@ -0,0 +1,37 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.pragma('journal_mode = WAL')
+db.exec(`
+  create table if not exists comments (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into comments (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+const counter = counterIter(11312000)
+let consequent404 = 0
+await asyncPool(counter.iter, async (id) => {
+  if (id % 1000 === 0) {
+    console.log('currently at %d', id)
+  }
+  const data = await ffetchShiki(`/api/comments/${id}`, {
+    validateResponse: false,
+  }).json<any>()
+
+  if (data.code === 404) {
+    consequent404++
+    if (consequent404 > 10_000) {
+      counter.end()
+      console.log('10k consequent 404-s, stopping')
+    }
+    return
+  }
+
+  consequent404 = 0
+  insertQuery.run(id, JSON.stringify(data))
+}, { limit: 64 })