chore: update public repo

package.json

@@ -11,8 +11,11 @@
     "@fuman/net": "^0.0.9",
     "@fuman/node": "^0.0.4",
     "@mtcute/node": "^0.19.1",
+    "@types/better-sqlite3": "^7.6.12",
     "@types/plist": "^3.0.5",
     "@types/spinnies": "^0.5.3",
+    "better-sqlite3": "^11.8.1",
+    "canvas": "^3.1.0",
     "cheerio": "^1.0.0",
     "es-main": "^1.3.0",
     "filesize": "^10.1.6",
@@ -38,5 +41,11 @@
     "htmlparser2": "^10.0.0",
     "zod": "3.23.8",
     "zx": "8.2.2"
+  },
+  "pnpm": {
+    "onlyBuiltDependencies": [
+      "better-sqlite3",
+      "canvas"
+    ]
   }
 }

pnpm-lock.yaml

@@ -23,12 +23,21 @@ importers:
       '@mtcute/node':
         specifier: ^0.19.1
         version: 0.19.1
+      '@types/better-sqlite3':
+        specifier: ^7.6.12
+        version: 7.6.12
       '@types/plist':
         specifier: ^3.0.5
         version: 3.0.5
       '@types/spinnies':
         specifier: ^0.5.3
         version: 0.5.3
+      better-sqlite3:
+        specifier: ^11.8.1
+        version: 11.8.1
+      canvas:
+        specifier: ^3.1.0
+        version: 3.1.0
       cheerio:
         specifier: ^1.0.0
         version: 1.0.0
@@ -507,6 +516,9 @@ packages:
     peerDependencies:
       eslint: '>=8.40.0'
 
+  '@types/better-sqlite3@7.6.12':
+    resolution: {integrity: sha512-fnQmj8lELIj7BSrZQAdBMHEHX8OZLYIHXqAKT1O7tDfLxaINzf00PMjw22r3N/xXh0w/sGHlO6SVaCQ2mj78lg==}
+
   '@types/debug@4.1.12':
     resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==}
 
@@ -688,6 +700,9 @@ packages:
   better-sqlite3@11.3.0:
     resolution: {integrity: sha512-iHt9j8NPYF3oKCNOO5ZI4JwThjt3Z6J6XrcwG85VNMVzv1ByqrHWv5VILEbCMFWDsoHhXvQ7oC8vgRXFAKgl9w==}
 
+  better-sqlite3@11.8.1:
+    resolution: {integrity: sha512-9BxNaBkblMjhJW8sMRZxnxVTRgbRmssZW0Oxc1MPBTfiR+WW21e2Mk4qu8CzrcZb1LwPCnFsfDEzq+SNcBU8eg==}
+
   bindings@1.5.0:
     resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
 
@@ -726,6 +741,10 @@ packages:
   caniuse-lite@1.0.30001684:
     resolution: {integrity: sha512-G1LRwLIQjBQoyq0ZJGqGIJUXzJ8irpbjHLpVRXDvBEScFJ9b17sgK6vlx0GAJFE21okD7zXl08rRRUfq6HdoEQ==}
 
+  canvas@3.1.0:
+    resolution: {integrity: sha512-tTj3CqqukVJ9NgSahykNwtGda7V33VLObwrHfzT0vqJXu7J4d4C/7kQQW3fOEGDfZZoILPut5H00gOjyttPGyg==}
+    engines: {node: ^18.12.0 || >= 20.9.0}
+
   ccount@2.0.1:
     resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==}
 
@@ -1579,6 +1598,9 @@ packages:
     resolution: {integrity: sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==}
     engines: {node: '>=10'}
 
+  node-addon-api@7.1.1:
+    resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==}
+
   node-releases@2.0.18:
     resolution: {integrity: sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==}
 
@@ -2462,6 +2484,10 @@ snapshots:
       - supports-color
       - typescript
 
+  '@types/better-sqlite3@7.6.12':
+    dependencies:
+      '@types/node': 22.10.0
+
   '@types/debug@4.1.12':
     dependencies:
       '@types/ms': 0.7.34
@@ -2669,6 +2695,11 @@ snapshots:
       bindings: 1.5.0
       prebuild-install: 7.1.2
 
+  better-sqlite3@11.8.1:
+    dependencies:
+      bindings: 1.5.0
+      prebuild-install: 7.1.2
+
   bindings@1.5.0:
     dependencies:
       file-uri-to-path: 1.0.0
@@ -2712,6 +2743,11 @@ snapshots:
 
   caniuse-lite@1.0.30001684: {}
 
+  canvas@3.1.0:
+    dependencies:
+      node-addon-api: 7.1.1
+      prebuild-install: 7.1.2
+
   ccount@2.0.1: {}
 
   chalk@2.4.2:
@@ -3808,6 +3844,8 @@ snapshots:
     dependencies:
       semver: 7.6.3
 
+  node-addon-api@7.1.1: {}
+
   node-releases@2.0.18: {}
 
   normalize-package-data@2.5.0:

scripts/misc/shikimori/.gitignore

@@ -0,0 +1 @@
+/_very-secret-ratelimit-bypass.ts

scripts/misc/shikimori/animes.ts

@@ -0,0 +1,53 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const isManga = process.argv[2] === 'manga'
+const isRanobe = process.argv[2] === 'ranobe'
+
+const collection = isManga ? 'mangas' : isRanobe ? 'ranobe' : 'animes'
+
+const db = new Database('assets/shikimori.db')
+db.exec(`
+  create table if not exists ${collection} (
+    id integer primary key,
+    data text not null
+  );
+  create table if not exists ${collection}_related (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare(`insert into ${collection} (id, data) values (?, ?) on conflict (id) do update set data = excluded.data`)
+const insertRelatedQuery = db.prepare(`insert into ${collection}_related (id, data) values (?, ?) on conflict (id) do update set data = excluded.data`)
+
+const maxId = await ffetchShiki(`/api/${collection}?order=id_desc`).json<any>().then(res => res[0].id)
+console.log('max id: %d', maxId)
+
+const counter = counterIter(1, maxId)
+
+await asyncPool(counter.iter, async (id) => {
+  if (id % 1000 === 0) {
+    console.log('currently at %d', id)
+  }
+  // const data = await ffetchShiki(`/api/${collection}/${id}`, {
+  //   validateResponse: false,
+  // }).json<any>()
+
+  // if (data.code === 404) {
+  //   return
+  // }
+
+  // insertQuery.run(id, JSON.stringify(data))
+
+  const data = await ffetchShiki(`/api/${collection}/${id}/related`, {
+    validateResponse: false,
+  }).json<any>()
+
+  if (data.code === 404) {
+    return
+  }
+
+  insertRelatedQuery.run(id, JSON.stringify(data))
+}, { limit: 64 })

scripts/misc/shikimori/bans.ts

@@ -0,0 +1,30 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.exec(`
+  create table if not exists bans (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into bans (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+const counter = counterIter(1)
+
+await asyncPool(counter.iter, async (page) => {
+  if (page % 100 === 0) {
+    console.log('currently at page %d', page)
+  }
+  const data = await ffetchShiki(`/api/bans?page=${page}`).json<any>()
+  if (!data.length) {
+    counter.end()
+    return
+  }
+
+  for (const ban of data) {
+    insertQuery.run(ban.id, JSON.stringify(ban))
+  }
+}, { limit: 64 })

scripts/misc/shikimori/characters.ts

@@ -0,0 +1,59 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.pragma('journal_mode = WAL')
+db.exec(`
+  create table if not exists characters (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into characters (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+// find maxId with binary search
+let maxIdPage = 20000
+let maxIdPageStart = 1
+let maxId = 0
+while (true) {
+  const midPage = Math.floor((maxIdPageStart + maxIdPage) / 2)
+  console.log('trying page %d', midPage)
+  const res = await ffetchShiki.post('/api/graphql', {
+    json: {
+      query: `{characters(page: ${midPage}, limit: 50) { id }}`,
+    },
+  }).json<any>()
+  const items = res.data.characters
+  if (!items.length) {
+    maxIdPage = midPage - 1
+    continue
+  }
+
+  if (maxIdPageStart === midPage) {
+    maxId = Math.max(...items.map(item => item.id))
+    break
+  } else {
+    maxIdPageStart = midPage
+  }
+}
+
+console.log('max id: %d', maxId)
+
+const counter = counterIter(1, maxId)
+
+await asyncPool(counter.iter, async (id) => {
+  if (id % 1000 === 0) {
+    console.log('currently at %d', id)
+  }
+  const data = await ffetchShiki(`/api/characters/${id}`, {
+    validateResponse: false,
+  }).json<any>()
+
+  if (data.code === 404) {
+    return
+  }
+
+  insertQuery.run(id, JSON.stringify(data))
+}, { limit: 64 })

scripts/misc/shikimori/clubs.ts

@@ -0,0 +1,49 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.pragma('journal_mode = WAL')
+db.exec(`
+  create table if not exists clubs (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into clubs (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+// collect clubs ids
+
+const ids: Set<number> = new Set()
+
+const pageCounter = counterIter(1)
+
+await asyncPool(pageCounter.iter, async (page) => {
+  const data = await ffetchShiki('/api/clubs', {
+    query: { page, limit: 50 },
+    validateResponse: false,
+  }).json<any>()
+  if (!data.length) {
+    pageCounter.end()
+    return
+  }
+
+  for (const club of data) {
+    ids.add(club.id)
+  }
+}, { limit: 16 })
+
+console.log('collected %d clubs', ids.size)
+
+await asyncPool(ids, async (id, idx) => {
+  if (idx % 100 === 0) {
+    console.log('currently at %d', idx)
+  }
+
+  const clubData = await ffetchShiki(`/api/clubs/${id}`).json<any>()
+  if (clubData.code === 404) {
+    return
+  }
+  insertQuery.run(id, JSON.stringify(clubData))
+}, { limit: 64 })

scripts/misc/shikimori/comments.ts

@@ -0,0 +1,37 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.pragma('journal_mode = WAL')
+db.exec(`
+  create table if not exists comments (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into comments (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+const counter = counterIter(11312000)
+let consequent404 = 0
+await asyncPool(counter.iter, async (id) => {
+  if (id % 1000 === 0) {
+    console.log('currently at %d', id)
+  }
+  const data = await ffetchShiki(`/api/comments/${id}`, {
+    validateResponse: false,
+  }).json<any>()
+
+  if (data.code === 404) {
+    consequent404++
+    if (consequent404 > 10_000) {
+      counter.end()
+      console.log('10k consequent 404-s, stopping')
+    }
+    return
+  }
+
+  consequent404 = 0
+  insertQuery.run(id, JSON.stringify(data))
+}, { limit: 64 })

scripts/misc/shikimori/people.ts

@@ -0,0 +1,59 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.pragma('journal_mode = WAL')
+db.exec(`
+  create table if not exists people (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into people (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+// find maxId with binary search
+let maxIdPage = 20000
+let maxIdPageStart = 1
+let maxId = 0
+while (true) {
+  const midPage = Math.floor((maxIdPageStart + maxIdPage) / 2)
+  console.log('trying page %d', midPage)
+  const res = await ffetchShiki.post('/api/graphql', {
+    json: {
+      query: `{people(page: ${midPage}, limit: 50) { id }}`,
+    },
+  }).json<any>()
+  const items = res.data.people
+  if (!items.length) {
+    maxIdPage = midPage - 1
+    continue
+  }
+
+  if (maxIdPageStart === midPage) {
+    maxId = Math.max(...items.map(item => item.id))
+    break
+  } else {
+    maxIdPageStart = midPage
+  }
+}
+
+console.log('max id: %d', maxId)
+
+const counter = counterIter(1, maxId)
+
+await asyncPool(counter.iter, async (id) => {
+  if (id % 1000 === 0) {
+    console.log('currently at %d', id)
+  }
+  const data = await ffetchShiki(`/api/people/${id}`, {
+    validateResponse: false,
+  }).json<any>()
+
+  if (data.code === 404) {
+    return
+  }
+
+  insertQuery.run(id, JSON.stringify(data))
+}, { limit: 64 })

scripts/misc/shikimori/users.ts

@@ -0,0 +1,129 @@
+import { asyncPool } from '@fuman/utils'
+import Database from 'better-sqlite3'
+import { counterIter, ffetchShiki } from './utils.ts'
+
+const db = new Database('assets/shikimori.db')
+db.pragma('journal_mode = WAL')
+db.exec(`
+  create table if not exists users (
+    id integer primary key,
+    data text not null
+  );
+`)
+
+const insertQuery = db.prepare('insert into users (id, data) values (?, ?) on conflict (id) do update set data = excluded.data')
+
+async function fetchUserFriends(userId: number) {
+  const list: any[] = []
+  for (let page = 1; ; page++) {
+    const data = await ffetchShiki(`/api/users/${userId}/friends`, {
+      query: { page, limit: 100 },
+      validateResponse: false,
+    }).json<any>()
+    if (!data.length) {
+      break
+    }
+
+    list.push(...data)
+  }
+
+  return list
+}
+
+async function fetchUserRates(userId: number, kind: 'anime' | 'manga') {
+  const list: any[] = []
+
+  for (let page = 1; ; page++) {
+    const data = await ffetchShiki(`/api/users/${userId}/${kind}_rates`, {
+      query: { page, limit: 1000 },
+      validateResponse: false,
+    }).json<any>()
+    if (data === null || !data.length) {
+      break
+    }
+
+    for (const item of data) {
+      // clean up unnecessary data before inserting
+      delete item.user
+      if (item[kind]) {
+        item[`${kind}_id`] = item[kind].id
+        delete item[kind]
+      }
+
+      list.push(item)
+    }
+  }
+
+  return list
+}
+
+async function fetchUserHistory(userId: number) {
+  const list: any[] = []
+  for (let page = 0; ; page++) {
+    const data = await ffetchShiki(`/api/users/${userId}/history`, {
+      query: { page, limit: 100 },
+      validateResponse: false,
+    }).json<any>()
+    if (!data.length) {
+      break
+    }
+
+    for (const item of data) {
+      if (item.target) {
+        item.target_type = item.target.url.startsWith('/animes/') ? 'anime' : 'manga'
+        item.target_id = item.target.id
+        delete item.target
+      }
+      list.push(item)
+    }
+  }
+
+  return list
+}
+
+const counter = counterIter(467800)
+let consequent404 = 0
+await asyncPool(counter.iter, async (id) => {
+  if (id % 100 === 0) {
+    console.log('currently at %d', id)
+  }
+  const data = await ffetchShiki(`/api/users/${id}`, {
+    validateResponse: false,
+  }).json<any>()
+
+  if (data.code === 404) {
+    consequent404++
+    if (consequent404 > 1_000) {
+      counter.end()
+      console.log('1k consequent 404-s, stopping')
+    }
+    return
+  }
+
+  consequent404 = 0
+
+  // fetch extra data
+  const [
+    favsData,
+    friends,
+    animeRates,
+    mangaRates,
+    history,
+  ] = await Promise.all([
+    ffetchShiki(`/api/users/${id}/favourites`).json<any>(),
+    fetchUserFriends(id),
+    fetchUserRates(id, 'anime'),
+    fetchUserRates(id, 'manga'),
+    fetchUserHistory(id),
+  ])
+
+  data._extra = {
+    favs: favsData,
+    friends,
+    animeRates,
+    mangaRates,
+    history,
+  }
+
+  insertQuery.run(id, JSON.stringify(data))
+}, { limit: 32 })

scripts/misc/shikimori/utils.ts

@@ -0,0 +1,39 @@
+import { ffetch as ffetchBase } from '../../../utils/fetch.ts'
+import { rateLimitBypass } from './_very-secret-ratelimit-bypass.ts'
+
+export const ffetchShiki = ffetchBase.extend({
+  baseUrl: 'https://shikimori.one',
+  headers: {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Accept-Encoding': 'gzip, deflate, br',
+  },
+  retry: {},
+  ...(rateLimitBypass as any),
+})
+
+export function counterIter(start = 0, end = Infinity) {
+  let i = start
+  let ended = false
+
+  const iter: IterableIterator<number> = {
+    [Symbol.iterator]: () => iter,
+    next() {
+      if (ended) {
+        return { value: undefined, done: true }
+      }
+
+      if (i > end) {
+        return { value: undefined, done: true }
+      }
+
+      return { value: i++, done: false }
+    },
+  }
+  return {
+    iter,
+    end: () => {
+      ended = true
+    },
+  }
+}

scripts/misc/update-forkgram.ts

@@ -2,7 +2,7 @@ import { readFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import plist from 'plist'
 import { z } from 'zod'
-import { $ } from 'zx'
+import { $, sleep } from 'zx'
 import { ffetch } from '../../utils/fetch.ts'
 
 const latestVerInfo = await ffetch('https://api.github.com/repos/forkgram/tdesktop/releases/latest').parsedJson(
@@ -40,12 +40,17 @@ if (!arm64Asset) {
 console.log('installing new version...')
 await $`curl -L ${arm64Asset.browser_download_url} -o /tmp/forkgram.zip`
 await $`unzip -o /tmp/forkgram.zip -d /tmp/forkgram`
-await $`kill -9 $(pgrep -f /Applications/Forkgram.app/Contents/MacOS/Telegram)`
+const pid = await $`/usr/bin/pgrep -f /Applications/Forkgram.app/Contents/MacOS/Telegram`.text().catch(() => null)
+if (pid) {
+  await $`kill -9 ${pid.trim()}`
+}
 await $`rm -rf ${INSTALL_PATH}`
 await $`mv /tmp/forkgram/Telegram.app ${INSTALL_PATH}`
 await $`rm -rf /tmp/forkgram`
 await $`xattr -cr ${INSTALL_PATH}`
 
+await sleep(1000)
+
 await $`open ${INSTALL_PATH}`
 
 console.log('✅ done')