chore: update public repo

scripts/misc/shikimori/characters.ts

@@ -0,0 +1,59 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.pragma('journal_mode = WAL')
+db.exec(`
+  create table if not exists characters (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into characters (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+// find maxId with binary search
+let maxIdPage = 20000
+let maxIdPageStart = 1
+let maxId = 0
+while (true) {
+  const midPage = Math.floor((maxIdPageStart + maxIdPage) / 2)
+  console.log('trying page %d', midPage)
+  const res = await ffetchShiki.post('/api/graphql', {
+    json: {
+      query: `{characters(page: ${midPage}, limit: 50) { id }}`,
+    },
+  }).json<any>()
+  const items = res.data.characters
+  if (!items.length) {
+    maxIdPage = midPage - 1
+    continue
+  }
+
+  if (maxIdPageStart === midPage) {
+    maxId = Math.max(...items.map(item => item.id))
+    break
+  } else {
+    maxIdPageStart = midPage
+  }
+}
+
+console.log('max id: %d', maxId)
+
+const counter = counterIter(1, maxId)
+
+await asyncPool(counter.iter, async (id) => {
+  if (id % 1000 === 0) {
+    console.log('currently at %d', id)
+  }
+  const data = await ffetchShiki(`/api/characters/${id}`, {
+    validateResponse: false,
+  }).json<any>()
+
+  if (data.code === 404) {
+    return
+  }
+
+  insertQuery.run(id, JSON.stringify(data))
+}, { limit: 64 })