From 4b71c3d8d03e3f4f0ecff3b7b81db8a4e5a8495a Mon Sep 17 00:00:00 2001 From: Battlesquid <25509915+Battlesquid@users.noreply.github.com> Date: Sat, 20 Jun 2026 23:22:19 -0700 Subject: [PATCH 1/3] fix: update storage script --- services/updater/scripts/update_storage.ts | 24 +++++++++++++++++----- services/updater/src/update_storage.ts | 24 ++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/services/updater/scripts/update_storage.ts b/services/updater/scripts/update_storage.ts index 3e1e151..7334913 100644 --- a/services/updater/scripts/update_storage.ts +++ b/services/updater/scripts/update_storage.ts @@ -1,8 +1,11 @@ -import { disconnectPgClient, testConnection } from "@qnaplus/store"; +import { disconnectPgClient, getAllQuestions, getMetadata, testConnection } from "@qnaplus/store"; import pino from "pino"; -import { updateStorage } from "../src/update_storage"; +import { uploadAllToStorage } from "../src/update_storage"; +import { initializeEnv } from "@qnaplus/dotenv"; (async () => { + await initializeEnv(); + const logger = pino({ errorKey: "error" }); logger.info("Running storage update script."); logger.info("Testing database connection..."); @@ -11,8 +14,19 @@ import { updateStorage } from "../src/update_storage"; logger.error({ error }, "Unable to connect to database, exiting."); return; } - logger.info("Starting storage update."); - await updateStorage(logger); - logger.info("Storage update completed."); + logger.info("Fetching metadata and all questions from database."); + const [metadataError, metadata] = await getMetadata(); + if (metadataError || !metadata) { + logger.error({ error: metadataError }, "Unable to fetch metadata from database, exiting."); + return; + } + const [questionsError, questions] = await getAllQuestions(); + if (questionsError) { + logger.error({ error: questionsError }, "Unable to fetch questions from database, exiting."); + return; + } + logger.info(`Fetched ${questions.length} questions. Starting storage upload.`); + await uploadAllToStorage(questions, metadata, logger); + logger.info("Storage upload completed."); await disconnectPgClient(); })(); diff --git a/services/updater/src/update_storage.ts b/services/updater/src/update_storage.ts index 296a08c..5cfe7e8 100644 --- a/services/updater/src/update_storage.ts +++ b/services/updater/src/update_storage.ts @@ -28,6 +28,30 @@ const get = async (key: string, logger: Logger): Promise => { return parsedData; }; +export const uploadAllToStorage = async ( + questions: Question[], + { currentSeason }: Metadata, + _logger: Logger, +) => { + const logger = _logger.child({ label: "upload_all_to_storage" }); + logger.info("Starting full storage upload."); + + const ALL_QUESTIONS_KEY = `questions-${getenv("NODE_ENV")}.json`; + const SEASON_QUESTIONS_KEY = `questions-${getenv("NODE_ENV")}-${currentSeason}.json`; + + const seasonQuestions = questions + .filter((q) => q.season === currentSeason) + .toSorted((a, b) => Number.parseInt(b.id) - Number.parseInt(a.id)); + await update(logger, seasonQuestions, SEASON_QUESTIONS_KEY); + + const sortedAll = questions.toSorted( + (a, b) => Number.parseInt(b.id) - Number.parseInt(a.id), + ); + await update(logger, sortedAll, ALL_QUESTIONS_KEY); + + logger.info("Completed full storage upload."); +}; + export const updateStorage = async ( updates: Question[], { currentSeason }: Metadata, From aa2f3e6ccaa3528c2414b1a6fd8176753808664b Mon Sep 17 00:00:00 2001 From: Battlesquid <25509915+Battlesquid@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:54:50 -0700 Subject: [PATCH 2/3] fix: upgrade scraper --- pnpm-lock.yaml | 36 ++++++++++++++++++------------------ pnpm-workspace.yaml | 8 ++++---- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bd2b00a..40bf3a7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,11 +8,11 @@ settings: catalogs: default: '@qnaplus/scraper': - specifier: ^3.1.6 - version: 3.1.6 + specifier: ^3.1.7 + version: 3.1.7 '@qnaplus/scraper-strategies': - specifier: ^3.1.6 - version: 3.1.6 + specifier: ^3.1.7 + version: 3.1.7 croner: specifier: ^10.0.1 version: 10.0.1 @@ -79,7 +79,7 @@ importers: version: link:../dotenv '@qnaplus/scraper': specifier: 'catalog:' - version: 3.1.6 + version: 3.1.7 '@qnaplus/utils': specifier: workspace:* version: link:../utils @@ -95,7 +95,7 @@ importers: devDependencies: '@qnaplus/scraper-strategies': specifier: 'catalog:' - version: 3.1.6 + version: 3.1.7 '@qnaplus/typescript-config': specifier: workspace:* version: link:../typescript-config @@ -158,7 +158,7 @@ importers: devDependencies: '@qnaplus/scraper': specifier: 'catalog:' - version: 3.1.6 + version: 3.1.7 '@qnaplus/typescript-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -179,10 +179,10 @@ importers: version: link:../../packages/logger '@qnaplus/scraper': specifier: 'catalog:' - version: 3.1.6 + version: 3.1.7 '@qnaplus/scraper-strategies': specifier: 'catalog:' - version: 3.1.6 + version: 3.1.7 '@qnaplus/store': specifier: workspace:* version: link:../../packages/store @@ -446,11 +446,11 @@ packages: '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} @@ -1228,11 +1228,11 @@ packages: '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} - '@qnaplus/scraper-strategies@3.1.6': - resolution: {integrity: sha512-Kkw4LEcsFPONHbC+iFcaBP2Zq1iRmbhQxf8/tUI8FwHOddwvNKG473gTEQc8L78GgrAXucheI9A4kpYzwwh2+g==} + '@qnaplus/scraper-strategies@3.1.7': + resolution: {integrity: sha512-gcTkI96NfFTcNX33Yn7R9dvlRH8s4ZAMsQtbs1tq82Nl6A0Tsn2la/3weUPiQ03wkunLP22FvppQJiP/7PWqxw==} - '@qnaplus/scraper@3.1.6': - resolution: {integrity: sha512-5gOC6cBJHvBpw1t7as6TVnVWd+dnKezVawblSgp4Ry4zNLrRR8uXRmH/mL1NR0CZ9u7MIDD/6+upSc8CUacyJQ==} + '@qnaplus/scraper@3.1.7': + resolution: {integrity: sha512-LaHTgufwpC3lPNvXSksSjXf5fbqbMlCNLWE79Be4X+kb5rm+akySpb1ZZehCW6FC4KJaZ7k0lBkLb7sIkFLleg==} '@sapphire/async-queue@1.5.5': resolution: {integrity: sha512-cvGzxbba6sav2zZkH8GPf2oGk9yYoD5qrNWdu9fRehifgnFZJMV+nuy2nON2roRO4yQQ+v7MK/Pktl/HgfsUXg==} @@ -3289,9 +3289,9 @@ snapshots: '@pinojs/redact@0.4.0': {} - '@qnaplus/scraper-strategies@3.1.6': + '@qnaplus/scraper-strategies@3.1.7': dependencies: - '@qnaplus/scraper': 3.1.6 + '@qnaplus/scraper': 3.1.7 cycletls: 2.0.5 header-generator: 2.1.82 pino: 10.3.1 @@ -3301,7 +3301,7 @@ snapshots: - supports-color - utf-8-validate - '@qnaplus/scraper@3.1.6': + '@qnaplus/scraper@3.1.7': dependencies: '@crawlee/core': 3.16.0 cheerio: 1.2.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c0c25fa..95446fe 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -9,10 +9,10 @@ injectWorkspacePackages: true catalog: pino: ^10.3.1 pino-pretty: ^13.1.3 - '@qnaplus/scraper': ^3.1.6 - '@qnaplus/scraper-strategies': ^3.1.6 + '@qnaplus/scraper': ^3.1.7 + '@qnaplus/scraper-strategies': ^3.1.7 croner: ^10.0.1 minimumReleaseAgeExclude: - - '@qnaplus/scraper-strategies@3.1.6' - - '@qnaplus/scraper@3.1.6' + - '@qnaplus/scraper-strategies@3.1.7' + - '@qnaplus/scraper@3.1.7' - ts-curl-impersonate@2.0.0-rc.0 From c2f2c43712bd61dfe7daba47a80f3bf73f3fa482 Mon Sep 17 00:00:00 2001 From: Battlesquid <25509915+Battlesquid@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:57:06 -0700 Subject: [PATCH 3/3] chore: neat --- scripts/build_images.sh | 6 +++--- scripts/push_images.sh | 4 ++-- services/updater/src/update_storage.ts | 4 +--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/build_images.sh b/scripts/build_images.sh index a3fb03f..7bba909 100755 --- a/scripts/build_images.sh +++ b/scripts/build_images.sh @@ -1,3 +1,3 @@ -#!/bin/sh -docker build --rm -t battlesquid0101/qnaplus_bot -f ./services/bot/Dockerfile . -docker build --rm -t battlesquid0101/qnaplus_updater -f ./services/updater/Dockerfile . +#!/bin/sh +docker build --rm -t battlesquid0101/qnaplus_bot -f ./services/bot/Dockerfile . +docker build --rm -t battlesquid0101/qnaplus_updater -f ./services/updater/Dockerfile . diff --git a/scripts/push_images.sh b/scripts/push_images.sh index 68219b7..837d6d7 100755 --- a/scripts/push_images.sh +++ b/scripts/push_images.sh @@ -1,3 +1,3 @@ -#!/bin/sh -docker push battlesquid0101/qnaplus_updater:latest +#!/bin/sh +docker push battlesquid0101/qnaplus_updater:latest docker push battlesquid0101/qnaplus_bot:latest \ No newline at end of file diff --git a/services/updater/src/update_storage.ts b/services/updater/src/update_storage.ts index 5cfe7e8..8d8659f 100644 --- a/services/updater/src/update_storage.ts +++ b/services/updater/src/update_storage.ts @@ -44,9 +44,7 @@ export const uploadAllToStorage = async ( .toSorted((a, b) => Number.parseInt(b.id) - Number.parseInt(a.id)); await update(logger, seasonQuestions, SEASON_QUESTIONS_KEY); - const sortedAll = questions.toSorted( - (a, b) => Number.parseInt(b.id) - Number.parseInt(a.id), - ); + const sortedAll = questions.toSorted((a, b) => Number.parseInt(b.id) - Number.parseInt(a.id)); await update(logger, sortedAll, ALL_QUESTIONS_KEY); logger.info("Completed full storage upload.");