chore: update public repo

scripts/misc/shikimori/clubs.ts

@@ -0,0 +1,49 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.pragma('journal_mode = WAL')
+db.exec(`
+  create table if not exists clubs (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into clubs (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+// collect clubs ids
+
+const ids: Set<number> = new Set()
+
+const pageCounter = counterIter(1)
+
+await asyncPool(pageCounter.iter, async (page) => {
+  const data = await ffetchShiki('/api/clubs', {
+    query: { page, limit: 50 },
+    validateResponse: false,
+  }).json<any>()
+  if (!data.length) {
+    pageCounter.end()
+    return
+  }
+
+  for (const club of data) {
+    ids.add(club.id)
+  }
+}, { limit: 16 })
+
+console.log('collected %d clubs', ids.size)
+
+await asyncPool(ids, async (id, idx) => {
+  if (idx % 100 === 0) {
+    console.log('currently at %d', idx)
+  }
+
+  const clubData = await ffetchShiki(`/api/clubs/${id}`).json<any>()
+  if (clubData.code === 404) {
+    return
+  }
+  insertQuery.run(id, JSON.stringify(clubData))
+}, { limit: 64 })