fix: update title selector

.github/workflows/pull-request.yml

@@ -26,4 +26,3 @@ jobs:
                     fi
                   done
             - run: pnpm neat
-            - run: pnpm test

package.json

@@ -6,13 +6,13 @@
 	"author": "battlesqui_d",
 	"license": "GPL-3.0",
 	"devDependencies": {
-		"oxfmt": "^0.51.0",
-		"oxlint": "^1.66.0",
-		"typescript": "6.0.3"
+		"oxfmt": "^0.68.0",
+		"oxlint": "^1.83.0",
+		"typescript": "7.0.2"
 	},
 	"scripts": {
 		"build": "tsc --build",
-		"clean": "pnpm -r clean",
+		"clean:dist": "pnpm -r clean:dist",
 		"reset": "pnpm -r reset && rm -rf node_modules",
 		"test": "pnpm -r test",
 		"lint": "pnpm -r lint",

packages/scraper-strategies/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@qnaplus/scraper-strategies",
-	"version": "3.1.6",
+	"version": "3.1.7",
 	"description": "Request stategies for @qnaplus/scraper",
 	"main": "dist/index.js",
 	"repository": {
@@ -10,9 +10,9 @@
 	"license": "GPL-3.0",
 	"scripts": {
 		"build": "tsc",
-		"build:clean": "pnpm clean && pnpm build",
-		"clean": "rm -rf dist tsconfig.tsbuildinfo",
-		"reset": "pnpm clean && rm -rf node_modules",
+		"build:clean": "pnpm clean:dist && pnpm build",
+		"clean:dist": "rm -rf dist tsconfig.tsbuildinfo",
+		"reset": "pnpm clean:dist && rm -rf node_modules",
 		"lint": "oxlint src",
 		"lint:fix": "oxlint --fix src",
 		"fmt": "oxfmt --check src",

packages/scraper/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@qnaplus/scraper",
-	"version": "3.1.6",
+	"version": "3.1.7",
 	"description": "Utility package to scrape questions from the VEX Robotics Q&A.",
 	"main": "dist/index.js",
 	"repository": {
@@ -10,10 +10,10 @@
 	"license": "GPL-3.0",
 	"scripts": {
 		"build": "tsc",
-		"build:clean": "pnpm clean && pnpm build",
+		"build:clean": "pnpm clean:dist && pnpm build",
 		"test": "vitest --run",
-		"clean": "rm -rf dist tsconfig.tsbuildinfo",
-		"reset": "pnpm clean && rm -rf node_modules",
+		"clean:dist": "rm -rf dist tsconfig.tsbuildinfo",
+		"reset": "pnpm clean:dist && rm -rf node_modules",
 		"lint": "oxlint src",
 		"lint:fix": "oxlint --fix src",
 		"fmt": "oxfmt --check src",

packages/scraper/src/modules/extractors.ts

@@ -1,5 +1,5 @@
 import * as cheerio from "cheerio";
-import SELECTORS from "../selectors";
+import SELECTORS, { ScrapingSelectors } from "../selectors";
 import type { Question } from "../types";
 import { type QnaHomeUrl, type QnaIdUrl, type QnaPageUrl, parseQnaUrlWithId } from "./parsing";
 
@@ -27,20 +27,23 @@ function selectHtml(
 	selectorKey: keyof typeof SELECTORS,
 	required: true,
 	raw?: boolean,
+	selectorOverrides?: Partial<ScrapingSelectors>,
 ): string;
 function selectHtml(
 	$: cheerio.CheerioAPI,
 	selectorKey: keyof typeof SELECTORS,
 	required: false,
 	raw?: boolean,
+	selectorOverrides?: Partial<ScrapingSelectors>,
 ): string | null;
 function selectHtml(
 	$: cheerio.CheerioAPI,
 	selectorKey: keyof typeof SELECTORS,
 	required: boolean,
 	raw?: boolean,
+	selectorOverrides?: Partial<ScrapingSelectors>,
 ): string | null {
-	const selector = SELECTORS[selectorKey];
+	const selector = selectorOverrides?.[selectorKey] ?? SELECTORS[selectorKey];
 	const text = raw ? unleak(unformat($(selector).html())) : unleak(unformat($(selector).text()));
 	const isEmptyString = text.trim() === "";
 	if (required && isEmptyString) {
@@ -49,37 +52,46 @@ function selectHtml(
 	return isEmptyString ? null : text;
 }
 
-export const extractPageQuestions = ({ html }: ScrapedPage<QnaPageUrl>): QnaIdUrl[] => {
+export const extractPageQuestions = (
+	{ html }: ScrapedPage<QnaPageUrl>,
+	selectorOverrides?: Partial<ScrapingSelectors>,
+): QnaIdUrl[] => {
 	const $ = cheerio.load(html);
-	return $(SELECTORS.URLS)
+	return $(selectorOverrides?.URLS ?? SELECTORS.URLS)
 		.toArray()
 		.map((el) => $(el).attr("href"))
 		.filter((s): s is QnaIdUrl => s !== undefined);
 };
 
-export const extractPageCount = ({ html }: ScrapedPage<QnaHomeUrl>): number => {
+export const extractPageCount = (
+	{ html }: ScrapedPage<QnaHomeUrl>,
+	selectorOverrides?: Partial<ScrapingSelectors>,
+): number => {
 	const $ = cheerio.load(html);
-	const el = $(SELECTORS.PAGE_COUNT);
+	const el = $(selectorOverrides?.PAGE_COUNT ?? SELECTORS.PAGE_COUNT);
 	return Number.isNaN(Number.parseInt(el.text())) ? 1 : Number.parseInt(el.text());
 };
 
-export const extractQuestion = ({ html, url }: ScrapedPage<QnaIdUrl>): Question => {
+export const extractQuestion = (
+	{ html, url }: ScrapedPage<QnaIdUrl>,
+	selectorOverrides?: Partial<ScrapingSelectors>,
+): Question => {
 	const $ = cheerio.load(html);
 
 	const { id, program, season } = parseQnaUrlWithId(url);
-	const author = selectHtml($, "AUTHOR", true);
-	const title = selectHtml($, "TITLE", true);
-	const question = selectHtml($, "QUESTION", true);
-	const questionRaw = selectHtml($, "QUESTION", true, true);
-	const answer = selectHtml($, "ANSWER", false);
-	const answerRaw = selectHtml($, "ANSWER", false, true);
-	const askedTimestamp = selectHtml($, "ASKED_TIMESTAMP", true);
+	const author = selectHtml($, "AUTHOR", true, false, selectorOverrides);
+	const title = selectHtml($, "TITLE", true, false, selectorOverrides);
+	const question = selectHtml($, "QUESTION", true, false, selectorOverrides);
+	const questionRaw = selectHtml($, "QUESTION", true, true, selectorOverrides);
+	const answer = selectHtml($, "ANSWER", false, false, selectorOverrides);
+	const answerRaw = selectHtml($, "ANSWER", false, true, selectorOverrides);
+	const askedTimestamp = selectHtml($, "ASKED_TIMESTAMP", true, false, selectorOverrides);
 	const askedTimestampMs = new Date(askedTimestamp).getTime();
-	const answeredTimestamp = selectHtml($, "ANSWERED_TIMESTAMP", false);
+	const answeredTimestamp = selectHtml($, "ANSWERED_TIMESTAMP", false, false, selectorOverrides);
 	const answeredTimestampMs =
 		answeredTimestamp !== null ? new Date(answeredTimestamp).getTime() : null;
 	const answered = answer !== null;
-	const tags = $(SELECTORS.TAGS)
+	const tags = $(selectorOverrides?.TAGS ?? SELECTORS.TAGS)
 		.map((_i, el) => unleak($(el).text().trim()))
 		.get();
 
@@ -103,7 +115,10 @@ export const extractQuestion = ({ html, url }: ScrapedPage<QnaIdUrl>): Question
 	};
 };
 
-export const extractReadOnly = ({ html }: ScrapedPage<QnaHomeUrl>): boolean => {
+export const extractReadOnly = (
+	{ html }: ScrapedPage<QnaHomeUrl>,
+	selectorOverrides?: Partial<ScrapingSelectors>,
+): boolean => {
 	const $ = cheerio.load(html);
-	return selectHtml($, "READONLY", false) !== null;
+	return selectHtml($, "READONLY", false, false, selectorOverrides) !== null;
 };

packages/scraper/src/selectors.ts

@@ -1,12 +1,25 @@
+export interface ScrapingSelectors {
+	URLS: string;
+	AUTHOR: string;
+	TITLE: string;
+	QUESTION: string;
+	ANSWER: string;
+	ASKED_TIMESTAMP: string;
+	ANSWERED_TIMESTAMP: string;
+	TAGS: string;
+	PAGE_COUNT: string;
+	READONLY: string;
+}
+
 export default {
 	URLS: "div.card-body h4.title > a",
 	AUTHOR: "div.author",
-	TITLE: "div.question > h4",
+	TITLE: "div.question > h2",
 	QUESTION: "div.content-body:nth-child(4)",
 	ANSWER: "div.answer.approved .content-body",
 	ASKED_TIMESTAMP: "div.details:nth-child(3) > div:nth-child(2)",
 	ANSWERED_TIMESTAMP: "div.pull-right",
 	TAGS: "div.tags a",
 	PAGE_COUNT: "nav ul.pagination li:nth-last-child(2)",
 	READONLY: "div.alert-warning",
-};
+} satisfies ScrapingSelectors;