teidesu-scripts/scripts/misc/shikimori/users.ts

import { asyncPool } from '@fuman/utils'
import Database from 'better-sqlite3'
import { counterIter, ffetchShiki } from './utils.ts'

const db = new Database('assets/shikimori.db')
db.pragma('journal_mode = WAL')
db.exec(`
  create table if not exists users (
    id integer primary key,
    data text not null
  );
`)

const insertQuery = db.prepare('insert into users (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')

async function fetchUserFriends(userId: number) {
  const list: any[] = []
  for (let page = 1; ; page++) {
    const data = await ffetchShiki(`/api/users/${userId}/friends`, {
      query: { page, limit: 100 },
      validateResponse: false,
    }).json<any>()
    if (!data.length) {
      break
    }

    list.push(...data)
  }

  return list
}

async function fetchUserRates(userId: number, kind: 'anime' | 'manga') {
  const list: any[] = []

  for (let page = 1; ; page++) {
    const data = await ffetchShiki(`/api/users/${userId}/${kind}_rates`, {
      query: { page, limit: 1000 },
      validateResponse: false,
    }).json<any>()
    if (data === null || !data.length) {
      break
    }

    for (const item of data) {
      // clean up unnecessary data before inserting
      delete item.user
      if (item[kind]) {
        item[`${kind}_id`] = item[kind].id
        delete item[kind]
      }

      list.push(item)
    }
  }

  return list
}

async function fetchUserHistory(userId: number) {
  const list: any[] = []
  for (let page = 0; ; page++) {
    const data = await ffetchShiki(`/api/users/${userId}/history`, {
      query: { page, limit: 100 },
      validateResponse: false,
    }).json<any>()
    if (!data.length) {
      break
    }

    for (const item of data) {
      if (item.target) {
        item.target_type = item.target.url.startsWith('/animes/') ? 'anime' : 'manga'
        item.target_id = item.target.id
        delete item.target
      }
      list.push(item)
    }
  }

  return list
}

const counter = counterIter(467800)
let consequent404 = 0
await asyncPool(counter.iter, async (id) => {
  if (id % 100 === 0) {
    console.log('currently at %d', id)
  }
  const data = await ffetchShiki(`/api/users/${id}`, {
    validateResponse: false,
  }).json<any>()

  if (data.code === 404) {
    consequent404++
    if (consequent404 > 1_000) {
      counter.end()
      console.log('1k consequent 404-s, stopping')
    }
    return
  }

  consequent404 = 0

  // fetch extra data
  const [
    favsData,
    friends,
    animeRates,
    mangaRates,
    history,
  ] = await Promise.all([
    ffetchShiki(`/api/users/${id}/favourites`).json<any>(),
    fetchUserFriends(id),
    fetchUserRates(id, 'anime'),
    fetchUserRates(id, 'manga'),
    fetchUserHistory(id),
  ])

  data._extra = {
    favs: favsData,
    friends,
    animeRates,
    mangaRates,
    history,
  }

  insertQuery.run(id, JSON.stringify(data))
}, { limit: 32 })