chore: update public repo

scripts/misc/shikimori/users.ts

@@ -0,0 +1,129 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.pragma('journal_mode = WAL')
+db.exec(`
+  create table if not exists users (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into users (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+async function fetchUserFriends(userId: number) {
+  const list: any[] = []
+  for (let page = 1; ; page++) {
+    const data = await ffetchShiki(`/api/users/${userId}/friends`, {
+      query: { page, limit: 100 },
+      validateResponse: false,
+    }).json<any>()
+    if (!data.length) {
+      break
+    }
+
+    list.push(...data)
+  }
+
+  return list
+}
+
+async function fetchUserRates(userId: number, kind: 'anime' | 'manga') {
+  const list: any[] = []
+
+  for (let page = 1; ; page++) {
+    const data = await ffetchShiki(`/api/users/${userId}/${kind}_rates`, {
+      query: { page, limit: 1000 },
+      validateResponse: false,
+    }).json<any>()
+    if (data === null || !data.length) {
+      break
+    }
+
+    for (const item of data) {
+      // clean up unnecessary data before inserting
+      delete item.user
+      if (item[kind]) {
+        item[`${kind}_id`] = item[kind].id
+        delete item[kind]
+      }
+
+      list.push(item)
+    }
+  }
+
+  return list
+}
+
+async function fetchUserHistory(userId: number) {
+  const list: any[] = []
+  for (let page = 0; ; page++) {
+    const data = await ffetchShiki(`/api/users/${userId}/history`, {
+      query: { page, limit: 100 },
+      validateResponse: false,
+    }).json<any>()
+    if (!data.length) {
+      break
+    }
+
+    for (const item of data) {
+      if (item.target) {
+        item.target_type = item.target.url.startsWith('/animes/') ? 'anime' : 'manga'
+        item.target_id = item.target.id
+        delete item.target
+      }
+      list.push(item)
+    }
+  }
+
+  return list
+}
+
+const counter = counterIter(467800)
+let consequent404 = 0
+await asyncPool(counter.iter, async (id) => {
+  if (id % 100 === 0) {
+    console.log('currently at %d', id)
+  }
+  const data = await ffetchShiki(`/api/users/${id}`, {
+    validateResponse: false,
+  }).json<any>()
+
+  if (data.code === 404) {
+    consequent404++
+    if (consequent404 > 1_000) {
+      counter.end()
+      console.log('1k consequent 404-s, stopping')
+    }
+    return
+  }
+
+  consequent404 = 0
+
+  // fetch extra data
+  const [
+    favsData,
+    friends,
+    animeRates,
+    mangaRates,
+    history,
+  ] = await Promise.all([
+    ffetchShiki(`/api/users/${id}/favourites`).json<any>(),
+    fetchUserFriends(id),
+    fetchUserRates(id, 'anime'),
+    fetchUserRates(id, 'manga'),
+    fetchUserHistory(id),
+  ])
+
+  data._extra = {
+    favs: favsData,
+    friends,
+    animeRates,
+    mangaRates,
+    history,
+  }
+
+  insertQuery.run(id, JSON.stringify(data))
+}, { limit: 32 })