feat: improve content extraction and reading experience

apps/server/src/lib/create-app.test.ts

@@ -0,0 +1,48 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { envMock, rateLimitMock } = vi.hoisted(() => ({
+  envMock: {
+    BODY_SIZE_LIMIT: 4_194_304,
+    CORS_ORIGINS: 'http://localhost:3001',
+    isDevelopment: true,
+    isProduction: false
+  },
+  rateLimitMock: vi.fn(async (_context: unknown, next: () => Promise<void>) => next())
+}));
+
+vi.mock('./env-config.js', () => ({ env: envMock }));
+vi.mock('../middlewares/rate-limit.js', () => ({ rateLimit: rateLimitMock }));
+
+import createApp from './create-app.js';
+
+async function requestProbe() {
+  const app = createApp();
+  app.get('/probe', (c) => c.text('ok'));
+
+  return app.request('/probe');
+}
+
+describe('createApp global rate limiting', () => {
+  beforeEach(() => {
+    envMock.isDevelopment = true;
+    envMock.isProduction = false;
+    rateLimitMock.mockClear();
+  });
+
+  it('skips the global rate limiter in development', async () => {
+    const response = await requestProbe();
+
+    expect(response.status).toBe(200);
+    expect(rateLimitMock).not.toHaveBeenCalled();
+  });
+
+  it('applies the global rate limiter in production', async () => {
+    envMock.isDevelopment = false;
+    envMock.isProduction = true;
+
+    const response = await requestProbe();
+
+    expect(response.status).toBe(200);
+    expect(rateLimitMock).toHaveBeenCalledOnce();
+  });
+});

apps/server/src/lib/create-app.ts

@@ -69,7 +69,7 @@ export default function createApp() {
       )
     );
 
-  if (!env.isTest) {
+  if (env.isProduction) {
     app.use('*', async (c, next) => {
       if (c.req.path === '/health') {
         return next();

apps/server/src/services/browser.service.test.ts

@@ -0,0 +1,81 @@
+import type { BrowserContext, Page, Response } from 'playwright-core';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('../index.js', () => ({ getIsShuttingDown: () => false }));
+
+import { browserService } from './browser.service.js';
+
+type Deferred = {
+  promise: Promise<void>;
+  resolve: () => void;
+};
+
+function deferred(): Deferred {
+  let resolve = () => {};
+  const promise = new Promise<void>((resolvePromise) => {
+    resolve = resolvePromise;
+  });
+
+  return { promise, resolve };
+}
+
+function challengeResponse(): Response {
+  return {
+    headers: () => ({ 'x-amzn-waf-action': 'challenge' })
+  } as unknown as Response;
+}
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+describe('browserService.crawlPage', () => {
+  it('waits for challenged article content before extracting the page', async () => {
+    const ready = deferred();
+    const content = vi.fn().mockResolvedValue('<article>Ready</article>');
+    const waitForFunction = vi.fn(() => ready.promise);
+    const page = {
+      content,
+      goto: vi.fn().mockResolvedValue(challengeResponse()),
+      waitForFunction,
+      waitForLoadState: vi.fn()
+    } as unknown as Page;
+    const context = { newPage: vi.fn().mockResolvedValue(page) } as unknown as BrowserContext;
+
+    vi.spyOn(browserService, 'acquireContext').mockResolvedValue(context);
+    vi.spyOn(browserService, 'releaseContext').mockResolvedValue();
+
+    const crawl = browserService.crawlPage('https://arstechnica.com/example');
+    await vi.waitFor(() => expect(waitForFunction).toHaveBeenCalledOnce());
+
+    expect(content).not.toHaveBeenCalled();
+
+    ready.resolve();
+    await expect(crawl).resolves.toEqual({ html: '<article>Ready</article>', isPaywalled: false });
+    expect(content).toHaveBeenCalledOnce();
+    expect(waitForFunction).toHaveBeenCalledWith(expect.any(Function), undefined, {
+      polling: 250,
+      timeout: 15_000
+    });
+  });
+
+  it('releases the browser context when challenge readiness times out', async () => {
+    const challengeError = new Error('Challenge readiness timed out');
+    const page = {
+      content: vi.fn(),
+      goto: vi.fn().mockResolvedValue(challengeResponse()),
+      waitForFunction: vi.fn().mockRejectedValue(challengeError),
+      waitForLoadState: vi.fn()
+    } as unknown as Page;
+    const context = { newPage: vi.fn().mockResolvedValue(page) } as unknown as BrowserContext;
+
+    vi.spyOn(browserService, 'acquireContext').mockResolvedValue(context);
+    const releaseContext = vi.spyOn(browserService, 'releaseContext').mockResolvedValue();
+
+    await expect(browserService.crawlPage('https://arstechnica.com/example')).rejects.toThrow(
+      challengeError
+    );
+    expect(releaseContext).toHaveBeenCalledWith(context);
+    expect(page.content).not.toHaveBeenCalled();
+  });
+});

apps/server/src/services/browser.service.ts

@@ -96,19 +96,32 @@ class BrowserService {
 
       logger.info(`[Crawler] Navigating to ${url}...`);
 
-      await page.goto(url, {
+      const response = await page.goto(url, {
         timeout: 30_000,
         waitUntil: 'domcontentloaded'
       });
 
-      logger.info(`[Crawler] Successfully navigated to ${url}. Waiting for page to load...`);
-
-      await Promise.race([
-        page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {}),
-        new Promise((resolve) => setTimeout(resolve, 5000))
-      ]);
-
-      logger.info(`[Crawler] Successfully loaded ${url}. Extracting content...`);
+      const wafAction = response?.headers()['x-amzn-waf-action'];
+
+      if (wafAction === 'challenge') {
+        await page.waitForFunction(
+          () => {
+            const articleText = document.querySelector('article')?.textContent?.trim() ?? '';
+            const bodyText = document.body?.innerText.trim() ?? '';
+
+            return (
+              document.title.length > 0 && Math.max(articleText.length, bodyText.length) >= 500
+            );
+          },
+          undefined,
+          {
+            polling: 250,
+            timeout: 15_000
+          }
+        );
+      } else {
+        await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});
+      }
 
       const html = await page.content();
 

apps/server/src/services/content-extraction-bookmark.integration.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from 'vitest';
+
+import { contentExtractionService } from './content-extraction.service.js';
+import { markdownService } from './markdown.service.js';
+
+describe('Ghost bookmark extraction pipeline', () => {
+  it('preserves bookmark metadata through Readability and Markdown conversion', async () => {
+    const sourceUrl = 'https://publisher.example/articles/story';
+    const html = `
+      <!doctype html>
+      <html>
+        <head><title>Example article</title></head>
+        <body>
+          <article>
+            <p>Opening article content.</p>
+            <figure class="kg-card kg-bookmark-card">
+              <a class="kg-bookmark-container" href="https://example.com/bookmark-target">
+                <div class="kg-bookmark-content">
+                  <div class="kg-bookmark-title">Example bookmark</div>
+                  <div class="kg-bookmark-description">Bookmark description.</div>
+                </div>
+                <div class="kg-bookmark-thumbnail">
+                  <img src="https://cdn.example.com/bookmark-thumbnail.png" alt="" />
+                </div>
+              </a>
+            </figure>
+          </article>
+        </body>
+      </html>
+    `;
+
+    const readable = await contentExtractionService.extractReadableContent(html, sourceUrl);
+
+    expect(readable.content).toContain('kg-bookmark-card');
+
+    const markdown = markdownService.convertToMarkdown(readable.content ?? '', {
+      baseUrl: sourceUrl,
+      title: readable.title
+    });
+
+    expect(markdown).toContain('[**Example bookmark**](https://example.com/bookmark-target)');
+    expect(markdown).toContain('Bookmark description.');
+  });
+});

apps/server/src/services/content-extraction.service.ts

@@ -3,6 +3,14 @@ import { parseHTML } from 'linkedom';
 
 import { logger } from '@/lib/logger.js';
 
+const GHOST_BOOKMARK_CLASSES = [
+  'kg-bookmark-card',
+  'kg-bookmark-content',
+  'kg-bookmark-title',
+  'kg-bookmark-description',
+  'kg-bookmark-thumbnail'
+];
+
 class ContentExtraction {
   async extractReadableContent(htmlContent: string, url: string) {
     const { document } = parseHTML(htmlContent);
@@ -21,6 +29,7 @@ class ContentExtraction {
 
     try {
       const article = new Readability(document, {
+        classesToPreserve: GHOST_BOOKMARK_CLASSES,
         // @ts-expect-error: missing type definition
         linkDensityModifier: 0.1
       }).parse();