From f02ccb6029e36b23ce5869b952e8b6ffa742ddbe Mon Sep 17 00:00:00 2001 From: desu-bot Date: Wed, 19 Feb 2025 02:30:26 +0000 Subject: [PATCH] chore: update public repo --- package.json | 9 ++ pnpm-lock.yaml | 38 ++++++++ scripts/misc/shikimori/.gitignore | 1 + scripts/misc/shikimori/animes.ts | 53 +++++++++++ scripts/misc/shikimori/bans.ts | 30 +++++++ scripts/misc/shikimori/characters.ts | 59 ++++++++++++ scripts/misc/shikimori/clubs.ts | 49 ++++++++++ scripts/misc/shikimori/comments.ts | 37 ++++++++ scripts/misc/shikimori/people.ts | 59 ++++++++++++ scripts/misc/shikimori/users.ts | 129 +++++++++++++++++++++++++++ scripts/misc/shikimori/utils.ts | 39 ++++++++ scripts/misc/update-forkgram.ts | 9 +- 12 files changed, 510 insertions(+), 2 deletions(-) create mode 100644 scripts/misc/shikimori/.gitignore create mode 100644 scripts/misc/shikimori/animes.ts create mode 100644 scripts/misc/shikimori/bans.ts create mode 100644 scripts/misc/shikimori/characters.ts create mode 100644 scripts/misc/shikimori/clubs.ts create mode 100644 scripts/misc/shikimori/comments.ts create mode 100644 scripts/misc/shikimori/people.ts create mode 100644 scripts/misc/shikimori/users.ts create mode 100644 scripts/misc/shikimori/utils.ts diff --git a/package.json b/package.json index bccf840..1ecf24e 100644 --- a/package.json +++ b/package.json @@ -11,8 +11,11 @@ "@fuman/net": "^0.0.9", "@fuman/node": "^0.0.4", "@mtcute/node": "^0.19.1", + "@types/better-sqlite3": "^7.6.12", "@types/plist": "^3.0.5", "@types/spinnies": "^0.5.3", + "better-sqlite3": "^11.8.1", + "canvas": "^3.1.0", "cheerio": "^1.0.0", "es-main": "^1.3.0", "filesize": "^10.1.6", @@ -38,5 +41,11 @@ "htmlparser2": "^10.0.0", "zod": "3.23.8", "zx": "8.2.2" + }, + "pnpm": { + "onlyBuiltDependencies": [ + "better-sqlite3", + "canvas" + ] } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0185865..ef97f05 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,12 +23,21 @@ importers: '@mtcute/node': specifier: ^0.19.1 version: 0.19.1 + '@types/better-sqlite3': + specifier: ^7.6.12 + version: 7.6.12 '@types/plist': specifier: ^3.0.5 version: 3.0.5 '@types/spinnies': specifier: ^0.5.3 version: 0.5.3 + better-sqlite3: + specifier: ^11.8.1 + version: 11.8.1 + canvas: + specifier: ^3.1.0 + version: 3.1.0 cheerio: specifier: ^1.0.0 version: 1.0.0 @@ -507,6 +516,9 @@ packages: peerDependencies: eslint: '>=8.40.0' + '@types/better-sqlite3@7.6.12': + resolution: {integrity: sha512-fnQmj8lELIj7BSrZQAdBMHEHX8OZLYIHXqAKT1O7tDfLxaINzf00PMjw22r3N/xXh0w/sGHlO6SVaCQ2mj78lg==} + '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} @@ -688,6 +700,9 @@ packages: better-sqlite3@11.3.0: resolution: {integrity: sha512-iHt9j8NPYF3oKCNOO5ZI4JwThjt3Z6J6XrcwG85VNMVzv1ByqrHWv5VILEbCMFWDsoHhXvQ7oC8vgRXFAKgl9w==} + better-sqlite3@11.8.1: + resolution: {integrity: sha512-9BxNaBkblMjhJW8sMRZxnxVTRgbRmssZW0Oxc1MPBTfiR+WW21e2Mk4qu8CzrcZb1LwPCnFsfDEzq+SNcBU8eg==} + bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -726,6 +741,10 @@ packages: caniuse-lite@1.0.30001684: resolution: {integrity: sha512-G1LRwLIQjBQoyq0ZJGqGIJUXzJ8irpbjHLpVRXDvBEScFJ9b17sgK6vlx0GAJFE21okD7zXl08rRRUfq6HdoEQ==} + canvas@3.1.0: + resolution: {integrity: sha512-tTj3CqqukVJ9NgSahykNwtGda7V33VLObwrHfzT0vqJXu7J4d4C/7kQQW3fOEGDfZZoILPut5H00gOjyttPGyg==} + engines: {node: ^18.12.0 || >= 20.9.0} + ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -1579,6 +1598,9 @@ packages: resolution: {integrity: sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==} engines: {node: '>=10'} + node-addon-api@7.1.1: + resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + node-releases@2.0.18: resolution: {integrity: sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==} @@ -2462,6 +2484,10 @@ snapshots: - supports-color - typescript + '@types/better-sqlite3@7.6.12': + dependencies: + '@types/node': 22.10.0 + '@types/debug@4.1.12': dependencies: '@types/ms': 0.7.34 @@ -2669,6 +2695,11 @@ snapshots: bindings: 1.5.0 prebuild-install: 7.1.2 + better-sqlite3@11.8.1: + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.2 + bindings@1.5.0: dependencies: file-uri-to-path: 1.0.0 @@ -2712,6 +2743,11 @@ snapshots: caniuse-lite@1.0.30001684: {} + canvas@3.1.0: + dependencies: + node-addon-api: 7.1.1 + prebuild-install: 7.1.2 + ccount@2.0.1: {} chalk@2.4.2: @@ -3808,6 +3844,8 @@ snapshots: dependencies: semver: 7.6.3 + node-addon-api@7.1.1: {} + node-releases@2.0.18: {} normalize-package-data@2.5.0: diff --git a/scripts/misc/shikimori/.gitignore b/scripts/misc/shikimori/.gitignore new file mode 100644 index 0000000..56d5f49 --- /dev/null +++ b/scripts/misc/shikimori/.gitignore @@ -0,0 +1 @@ +/_very-secret-ratelimit-bypass.ts \ No newline at end of file diff --git a/scripts/misc/shikimori/animes.ts b/scripts/misc/shikimori/animes.ts new file mode 100644 index 0000000..b681ac0 --- /dev/null +++ b/scripts/misc/shikimori/animes.ts @@ -0,0 +1,53 @@ +import { asyncPool } from '@fuman/utils' +import Database from 'better-sqlite3' +import { counterIter, ffetchShiki } from './utils.ts' + +const isManga = process.argv[2] === 'manga' +const isRanobe = process.argv[2] === 'ranobe' + +const collection = isManga ? 'mangas' : isRanobe ? 'ranobe' : 'animes' + +const db = new Database('assets/shikimori.db') +db.exec(` + create table if not exists ${collection} ( + id integer primary key, + data text not null + ); + create table if not exists ${collection}_related ( + id integer primary key, + data text not null + ); +`) + +const insertQuery = db.prepare(`insert into ${collection} (id, data) values (?, ?) on conflict (id) do update set data = excluded.data`) +const insertRelatedQuery = db.prepare(`insert into ${collection}_related (id, data) values (?, ?) on conflict (id) do update set data = excluded.data`) + +const maxId = await ffetchShiki(`/api/${collection}?order=id_desc`).json().then(res => res[0].id) +console.log('max id: %d', maxId) + +const counter = counterIter(1, maxId) + +await asyncPool(counter.iter, async (id) => { + if (id % 1000 === 0) { + console.log('currently at %d', id) + } + // const data = await ffetchShiki(`/api/${collection}/${id}`, { + // validateResponse: false, + // }).json() + + // if (data.code === 404) { + // return + // } + + // insertQuery.run(id, JSON.stringify(data)) + + const data = await ffetchShiki(`/api/${collection}/${id}/related`, { + validateResponse: false, + }).json() + + if (data.code === 404) { + return + } + + insertRelatedQuery.run(id, JSON.stringify(data)) +}, { limit: 64 }) diff --git a/scripts/misc/shikimori/bans.ts b/scripts/misc/shikimori/bans.ts new file mode 100644 index 0000000..085d954 --- /dev/null +++ b/scripts/misc/shikimori/bans.ts @@ -0,0 +1,30 @@ +import { asyncPool } from '@fuman/utils' +import Database from 'better-sqlite3' +import { counterIter, ffetchShiki } from './utils.ts' + +const db = new Database('assets/shikimori.db') +db.exec(` + create table if not exists bans ( + id integer primary key, + data text not null + ); +`) + +const insertQuery = db.prepare('insert into bans (id, data) values (?, ?) on conflict (id) do update set data = excluded.data') + +const counter = counterIter(1) + +await asyncPool(counter.iter, async (page) => { + if (page % 100 === 0) { + console.log('currently at page %d', page) + } + const data = await ffetchShiki(`/api/bans?page=${page}`).json() + if (!data.length) { + counter.end() + return + } + + for (const ban of data) { + insertQuery.run(ban.id, JSON.stringify(ban)) + } +}, { limit: 64 }) diff --git a/scripts/misc/shikimori/characters.ts b/scripts/misc/shikimori/characters.ts new file mode 100644 index 0000000..29fe2f1 --- /dev/null +++ b/scripts/misc/shikimori/characters.ts @@ -0,0 +1,59 @@ +import { asyncPool } from '@fuman/utils' +import Database from 'better-sqlite3' +import { counterIter, ffetchShiki } from './utils.ts' + +const db = new Database('assets/shikimori.db') +db.pragma('journal_mode = WAL') +db.exec(` + create table if not exists characters ( + id integer primary key, + data text not null + ); +`) + +const insertQuery = db.prepare('insert into characters (id, data) values (?, ?) on conflict (id) do update set data = excluded.data') + +// find maxId with binary search +let maxIdPage = 20000 +let maxIdPageStart = 1 +let maxId = 0 +while (true) { + const midPage = Math.floor((maxIdPageStart + maxIdPage) / 2) + console.log('trying page %d', midPage) + const res = await ffetchShiki.post('/api/graphql', { + json: { + query: `{characters(page: ${midPage}, limit: 50) { id }}`, + }, + }).json() + const items = res.data.characters + if (!items.length) { + maxIdPage = midPage - 1 + continue + } + + if (maxIdPageStart === midPage) { + maxId = Math.max(...items.map(item => item.id)) + break + } else { + maxIdPageStart = midPage + } +} + +console.log('max id: %d', maxId) + +const counter = counterIter(1, maxId) + +await asyncPool(counter.iter, async (id) => { + if (id % 1000 === 0) { + console.log('currently at %d', id) + } + const data = await ffetchShiki(`/api/characters/${id}`, { + validateResponse: false, + }).json() + + if (data.code === 404) { + return + } + + insertQuery.run(id, JSON.stringify(data)) +}, { limit: 64 }) diff --git a/scripts/misc/shikimori/clubs.ts b/scripts/misc/shikimori/clubs.ts new file mode 100644 index 0000000..714f093 --- /dev/null +++ b/scripts/misc/shikimori/clubs.ts @@ -0,0 +1,49 @@ +import { asyncPool } from '@fuman/utils' +import Database from 'better-sqlite3' +import { counterIter, ffetchShiki } from './utils.ts' + +const db = new Database('assets/shikimori.db') +db.pragma('journal_mode = WAL') +db.exec(` + create table if not exists clubs ( + id integer primary key, + data text not null + ); +`) + +const insertQuery = db.prepare('insert into clubs (id, data) values (?, ?) on conflict (id) do update set data = excluded.data') + +// collect clubs ids + +const ids: Set = new Set() + +const pageCounter = counterIter(1) + +await asyncPool(pageCounter.iter, async (page) => { + const data = await ffetchShiki('/api/clubs', { + query: { page, limit: 50 }, + validateResponse: false, + }).json() + if (!data.length) { + pageCounter.end() + return + } + + for (const club of data) { + ids.add(club.id) + } +}, { limit: 16 }) + +console.log('collected %d clubs', ids.size) + +await asyncPool(ids, async (id, idx) => { + if (idx % 100 === 0) { + console.log('currently at %d', idx) + } + + const clubData = await ffetchShiki(`/api/clubs/${id}`).json() + if (clubData.code === 404) { + return + } + insertQuery.run(id, JSON.stringify(clubData)) +}, { limit: 64 }) diff --git a/scripts/misc/shikimori/comments.ts b/scripts/misc/shikimori/comments.ts new file mode 100644 index 0000000..80a5bf3 --- /dev/null +++ b/scripts/misc/shikimori/comments.ts @@ -0,0 +1,37 @@ +import { asyncPool } from '@fuman/utils' +import Database from 'better-sqlite3' +import { counterIter, ffetchShiki } from './utils.ts' + +const db = new Database('assets/shikimori.db') +db.pragma('journal_mode = WAL') +db.exec(` + create table if not exists comments ( + id integer primary key, + data text not null + ); +`) + +const insertQuery = db.prepare('insert into comments (id, data) values (?, ?) on conflict (id) do update set data = excluded.data') + +const counter = counterIter(11312000) +let consequent404 = 0 +await asyncPool(counter.iter, async (id) => { + if (id % 1000 === 0) { + console.log('currently at %d', id) + } + const data = await ffetchShiki(`/api/comments/${id}`, { + validateResponse: false, + }).json() + + if (data.code === 404) { + consequent404++ + if (consequent404 > 10_000) { + counter.end() + console.log('10k consequent 404-s, stopping') + } + return + } + + consequent404 = 0 + insertQuery.run(id, JSON.stringify(data)) +}, { limit: 64 }) diff --git a/scripts/misc/shikimori/people.ts b/scripts/misc/shikimori/people.ts new file mode 100644 index 0000000..50073bf --- /dev/null +++ b/scripts/misc/shikimori/people.ts @@ -0,0 +1,59 @@ +import { asyncPool } from '@fuman/utils' +import Database from 'better-sqlite3' +import { counterIter, ffetchShiki } from './utils.ts' + +const db = new Database('assets/shikimori.db') +db.pragma('journal_mode = WAL') +db.exec(` + create table if not exists people ( + id integer primary key, + data text not null + ); +`) + +const insertQuery = db.prepare('insert into people (id, data) values (?, ?) on conflict (id) do update set data = excluded.data') + +// find maxId with binary search +let maxIdPage = 20000 +let maxIdPageStart = 1 +let maxId = 0 +while (true) { + const midPage = Math.floor((maxIdPageStart + maxIdPage) / 2) + console.log('trying page %d', midPage) + const res = await ffetchShiki.post('/api/graphql', { + json: { + query: `{people(page: ${midPage}, limit: 50) { id }}`, + }, + }).json() + const items = res.data.people + if (!items.length) { + maxIdPage = midPage - 1 + continue + } + + if (maxIdPageStart === midPage) { + maxId = Math.max(...items.map(item => item.id)) + break + } else { + maxIdPageStart = midPage + } +} + +console.log('max id: %d', maxId) + +const counter = counterIter(1, maxId) + +await asyncPool(counter.iter, async (id) => { + if (id % 1000 === 0) { + console.log('currently at %d', id) + } + const data = await ffetchShiki(`/api/people/${id}`, { + validateResponse: false, + }).json() + + if (data.code === 404) { + return + } + + insertQuery.run(id, JSON.stringify(data)) +}, { limit: 64 }) diff --git a/scripts/misc/shikimori/users.ts b/scripts/misc/shikimori/users.ts new file mode 100644 index 0000000..73411b8 --- /dev/null +++ b/scripts/misc/shikimori/users.ts @@ -0,0 +1,129 @@ +import { asyncPool } from '@fuman/utils' +import Database from 'better-sqlite3' +import { counterIter, ffetchShiki } from './utils.ts' + +const db = new Database('assets/shikimori.db') +db.pragma('journal_mode = WAL') +db.exec(` + create table if not exists users ( + id integer primary key, + data text not null + ); +`) + +const insertQuery = db.prepare('insert into users (id, data) values (?, ?) on conflict (id) do update set data = excluded.data') + +async function fetchUserFriends(userId: number) { + const list: any[] = [] + for (let page = 1; ; page++) { + const data = await ffetchShiki(`/api/users/${userId}/friends`, { + query: { page, limit: 100 }, + validateResponse: false, + }).json() + if (!data.length) { + break + } + + list.push(...data) + } + + return list +} + +async function fetchUserRates(userId: number, kind: 'anime' | 'manga') { + const list: any[] = [] + + for (let page = 1; ; page++) { + const data = await ffetchShiki(`/api/users/${userId}/${kind}_rates`, { + query: { page, limit: 1000 }, + validateResponse: false, + }).json() + if (data === null || !data.length) { + break + } + + for (const item of data) { + // clean up unnecessary data before inserting + delete item.user + if (item[kind]) { + item[`${kind}_id`] = item[kind].id + delete item[kind] + } + + list.push(item) + } + } + + return list +} + +async function fetchUserHistory(userId: number) { + const list: any[] = [] + for (let page = 0; ; page++) { + const data = await ffetchShiki(`/api/users/${userId}/history`, { + query: { page, limit: 100 }, + validateResponse: false, + }).json() + if (!data.length) { + break + } + + for (const item of data) { + if (item.target) { + item.target_type = item.target.url.startsWith('/animes/') ? 'anime' : 'manga' + item.target_id = item.target.id + delete item.target + } + list.push(item) + } + } + + return list +} + +const counter = counterIter(467800) +let consequent404 = 0 +await asyncPool(counter.iter, async (id) => { + if (id % 100 === 0) { + console.log('currently at %d', id) + } + const data = await ffetchShiki(`/api/users/${id}`, { + validateResponse: false, + }).json() + + if (data.code === 404) { + consequent404++ + if (consequent404 > 1_000) { + counter.end() + console.log('1k consequent 404-s, stopping') + } + return + } + + consequent404 = 0 + + // fetch extra data + const [ + favsData, + friends, + animeRates, + mangaRates, + history, + ] = await Promise.all([ + ffetchShiki(`/api/users/${id}/favourites`).json(), + fetchUserFriends(id), + fetchUserRates(id, 'anime'), + fetchUserRates(id, 'manga'), + fetchUserHistory(id), + ]) + + data._extra = { + favs: favsData, + friends, + animeRates, + mangaRates, + history, + } + + insertQuery.run(id, JSON.stringify(data)) +}, { limit: 32 }) diff --git a/scripts/misc/shikimori/utils.ts b/scripts/misc/shikimori/utils.ts new file mode 100644 index 0000000..e25dc28 --- /dev/null +++ b/scripts/misc/shikimori/utils.ts @@ -0,0 +1,39 @@ +import { ffetch as ffetchBase } from '../../../utils/fetch.ts' +import { rateLimitBypass } from './_very-secret-ratelimit-bypass.ts' + +export const ffetchShiki = ffetchBase.extend({ + baseUrl: 'https://shikimori.one', + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + }, + retry: {}, + ...(rateLimitBypass as any), +}) + +export function counterIter(start = 0, end = Infinity) { + let i = start + let ended = false + + const iter: IterableIterator = { + [Symbol.iterator]: () => iter, + next() { + if (ended) { + return { value: undefined, done: true } + } + + if (i > end) { + return { value: undefined, done: true } + } + + return { value: i++, done: false } + }, + } + return { + iter, + end: () => { + ended = true + }, + } +} diff --git a/scripts/misc/update-forkgram.ts b/scripts/misc/update-forkgram.ts index 878def9..700beab 100644 --- a/scripts/misc/update-forkgram.ts +++ b/scripts/misc/update-forkgram.ts @@ -2,7 +2,7 @@ import { readFile } from 'node:fs/promises' import { join } from 'node:path' import plist from 'plist' import { z } from 'zod' -import { $ } from 'zx' +import { $, sleep } from 'zx' import { ffetch } from '../../utils/fetch.ts' const latestVerInfo = await ffetch('https://api.github.com/repos/forkgram/tdesktop/releases/latest').parsedJson( @@ -40,12 +40,17 @@ if (!arm64Asset) { console.log('installing new version...') await $`curl -L ${arm64Asset.browser_download_url} -o /tmp/forkgram.zip` await $`unzip -o /tmp/forkgram.zip -d /tmp/forkgram` -await $`kill -9 $(pgrep -f /Applications/Forkgram.app/Contents/MacOS/Telegram)` +const pid = await $`/usr/bin/pgrep -f /Applications/Forkgram.app/Contents/MacOS/Telegram`.text().catch(() => null) +if (pid) { + await $`kill -9 ${pid.trim()}` +} await $`rm -rf ${INSTALL_PATH}` await $`mv /tmp/forkgram/Telegram.app ${INSTALL_PATH}` await $`rm -rf /tmp/forkgram` await $`xattr -cr ${INSTALL_PATH}` +await sleep(1000) + await $`open ${INSTALL_PATH}` console.log('✅ done')