From 4ffa6f6f732bc4ea90b4c4c2f78efbac6f1cf242 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Tue, 6 Oct 2026 21:28:06 +0800 Subject: [PATCH 1/3] fix(platform): skip pages a sitemap dates as unchanged on a rescan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crawler read `` out of a sitemap and nothing else, so every scan requested every page again — probe, browser render and the assets a render loads — and compared content hashes afterwards. A site that dated all 700 of its pages answered some 8,000 requests and two gigabytes a scan, every six hours, for pages its sitemap marked unchanged for months. A scheduled rescan now leaves a page alone when its stored text was read after the entry's ``. Every doubt still requests the page: an entry without a readable date, a date that moved, a date the sitemap dropped, a failed last read, text stored but not indexed yet. A date alone covers its whole day in any time zone, a read within the hour after the date does not count, and a page is read again once its last read is older than seven days, whatever its date says. "Scan now" requests every page; "Resume scanning" and the scans that follow a saved embedding model stay ordinary ones. The date lives in `public_web.website_urls.sitemap_lastmod` (knowledge-db migration 13), and each discovery replaces what the one before recorded. --- docs/de/platform/knowledge/crawling.md | 8 +- docs/en/platform/knowledge/crawling.md | 8 +- docs/fr/platform/knowledge/crawling.md | 8 +- ...0000000013_website_url_sitemap_lastmod.sql | 34 +++ .../backend/core/knowledge/crawl.test.ts | 78 ++++++ .../platform/backend/core/knowledge/crawl.ts | 47 ++++ .../knowledge/crawl_action.lastmod.test.ts | 255 ++++++++++++++++++ .../backend/core/knowledge/crawl_action.ts | 120 ++++++++- .../backend/domains/knowledge/spec.md | 18 ++ .../websites/service.embedding-follow.test.ts | 5 + .../domains/websites/service.scan-now.test.ts | 58 +++- .../backend/domains/websites/service.ts | 32 ++- .../platform/backend/integration-check.ts | 194 +++++++++++++ services/platform/backend/jobs/task-list.ts | 1 + services/platform/backend/jobs/tasks.ts | 6 +- .../lib/knowledge/crawl-parse.test.ts | 156 +++++++++++ .../platform/lib/knowledge/crawl-parse.ts | 141 ++++++++++ .../tests/manual/reference/automation.md | 1 + 18 files changed, 1145 insertions(+), 25 deletions(-) create mode 100644 services/db/migrations/knowledge-db/public_web/00000000000013_website_url_sitemap_lastmod.sql create mode 100644 services/platform/backend/core/knowledge/crawl_action.lastmod.test.ts diff --git a/docs/de/platform/knowledge/crawling.md b/docs/de/platform/knowledge/crawling.md index 07c6d4194d..c7faabb1d1 100644 --- a/docs/de/platform/knowledge/crawling.md +++ b/docs/de/platform/knowledge/crawling.md @@ -42,6 +42,8 @@ Bei einer ganzen Website nutzt der Crawler Startseite und veröffentlichte Sitem Scans arbeiten schrittweise: Unveränderte Inhalte werden übersprungen, geänderte erneut indexiert, neue Seiten hinzugefügt und entfernte aus dem Index genommen — ebenso Seiten, die die `robots.txt` inzwischen verbietet. Eine Seite, die auf eine andere Adresse derselben Website weiterleitet, wird nur einmal indexiert: unter der Adresse, auf der sie landet. Die Seitenzähler der Zeile folgen dem Scan, während Seiten landen, nach dem Entdecken und nach jedem gespeicherten Batch, die Tabelle bewegt sich also, während ein Scan läuft. Eine URL-Liste aktualisiert ihre feste Auswahl nach demselben Zeitplan. Nach erfolgreicher Indexierung ist keine gesonderte Veröffentlichung nötig. +Ein Scan ruft nicht jedes Mal jede Seite ab. Trägt der Sitemap-Eintrag einer Seite ein ``-Datum und hat der Crawler die Seite nach diesem Datum gelesen, lässt er sie in Ruhe, bis sich das Datum ändert. Ein Datum ohne Uhrzeit gilt für den ganzen Tag: Eine heute geänderte Seite wird deshalb bis zum Folgetag bei jedem Scan erneut gelesen. Auch eine Seite mit Datum wird wieder gelesen, sobald ihr letzter Abruf mehr als sieben Tage zurückliegt, denn eine Sitemap kann ihr Datum behalten, während sich die Seite ändert, etwa bei einer Übersichtsseite mit den neuesten Artikeln. Eine Seite, deren Eintrag kein lesbares Datum hat oder die in keiner Sitemap steht, wird bei jedem Scan abgerufen. Eine Website, die in ihrer Sitemap zutreffende ``-Daten veröffentlicht, erhält bei einem wiederholten Scan daher nur wenige Anfragen. + Der Crawler besucht die Seiten ohne Anmeldung. Eine URL macht private Inhalte nicht zugänglich. Bei jeder Anfrage stellt er sich als `TaleBot/ (+https://docs.tale.dev/platform/knowledge/crawling)` vor, sodass eine `robots.txt`-Gruppe ihn beim Namen nennen kann — `User-agent: TaleBot` —, um allein ihn zu erlauben, zu drosseln oder abzuweisen. Der Crawler hält sich an die `Disallow`-Regeln der `robots.txt` für den Agenten `*` auf jedem Weg, über den eine URL hereinkommen kann — die Sitemaps, der Linklauf und die Links, die eine gerenderte JavaScript-Seite preisgibt — und noch einmal vor jedem Abruf: Eine Seite, die eine Regel abdeckt, wird nie geholt, und eine Seite, die eine später hinzugekommene Regel abdeckt, verlässt den Index beim nächsten Scan. Ausdrücklich angegebene URLs filtern die Regeln nicht: Eine gelistete Adresse ist deine Anweisung. Liefert ein Abruf den HTTP-Header `X-Robots-Tag: noindex` oder `none` oder trägt die Seite ein HTML-Tag ``, wird der Inhalt nicht indexiert — auch bei einer URL-Liste —, und was ein früherer Scan von der Seite gespeichert hat, fällt weg. Diese Regeln sind Höflichkeit, kein Zugriffsschutz: Wenn du die Quellwebsite verwaltest, verlass dich nicht auf den Crawler als Zugangskontrolle. @@ -62,7 +64,7 @@ Du kannst weder eine eigene Seitenobergrenze noch Pfadfilter festlegen oder eine ## Die indexierten Inhalte prüfen -Die Tabelle zeigt **Status**, die Seitenzahl unter **Indexiert**, **Gescannt** und **Intervall**. Öffne die Quellzeile, um die Seitenliste, Wort- und Chunk-Anzahl sowie den letzten Abruf zu prüfen. Klappe eine Seite auf, um die gespeicherten Textabschnitte zu lesen. Bei einem fehlgeschlagenen Abruf stehen dort Ursache und Anzahl aufeinanderfolgender Fehler. Eine Seite, die der Crawler bewusst auslässt — die Quelle wünscht keine Indexierung, der Inhaltstyp liefert keinen lesbaren Text, oder eine Weiterleitung führt von der Website weg — steht als **Übersprungen** mit ihrer Ursache in der Liste und zählt nicht als fehlgeschlagen. +Die Tabelle zeigt **Status**, die Seitenzahl unter **Indexiert**, **Gescannt** und **Intervall**. Öffne die Quellzeile, um die Seitenliste, Wort- und Chunk-Anzahl sowie den letzten Abruf zu prüfen. Eine Seite, die ein Scan wegen ihres Sitemap-Datums ausgelassen hat, behält die Zeit ihres letzten Abrufs. Klappe eine Seite auf, um die gespeicherten Textabschnitte zu lesen. Bei einem fehlgeschlagenen Abruf stehen dort Ursache und Anzahl aufeinanderfolgender Fehler. Eine Seite, die der Crawler bewusst auslässt — die Quelle wünscht keine Indexierung, der Inhaltstyp liefert keinen lesbaren Text, oder eine Weiterleitung führt von der Website weg — steht als **Übersprungen** mit ihrer Ursache in der Liste und zählt nicht als fehlgeschlagen. | Status | Bedeutung | | --- | --- | @@ -73,7 +75,7 @@ Die Tabelle zeigt **Status**, die Seitenzahl unter **Indexiert**, **Gescannt** u Ein Scan, der gerade lief, als Tale neu gestartet, aktualisiert oder ohne Vorwarnung beendet wurde, läuft innerhalb weniger Minuten von selbst weiter, und zwar mit den Seiten, die er noch nicht erreicht hatte. Seiten, die den Browser brauchen, können bis zu einer Viertelstunde später folgen. Die Quelle zeigt währenddessen **Wird gescannt**. -Um außerhalb des Intervalls zu scannen, öffne das Zeilenmenü der Quelle oder ihre Details und wähle **Jetzt scannen**. Das hilft, wenn sich die Website geändert hat oder ein Scan fehlgeschlagen ist; ein fehlgeschlagener Scan wird sonst innerhalb von zwei Stunden von selbst wiederholt. Die Aktion steht zur Verfügung, solange die Quelle weder gescannt noch gelöscht wird. +Um außerhalb des Intervalls zu scannen, öffne das Zeilenmenü der Quelle oder ihre Details und wähle **Jetzt scannen**. Dabei wird jede Seite erneut abgerufen, unabhängig von den Daten der Sitemap. Das hilft, wenn sich die Website geändert hat oder ein Scan fehlgeschlagen ist; ein fehlgeschlagener Scan wird sonst innerhalb von zwei Stunden von selbst wiederholt. Die Aktion steht zur Verfügung, solange die Quelle weder gescannt noch gelöscht wird. Die Seitenansicht bietet auch eine Suche im indexierten Inhalt. Suche nach einer auffälligen Formulierung der Seite, bevor du dich im Chat darauf verlässt. Stelle anschließend eine konkrete Frage und prüfe den Quellenbeleg. @@ -94,6 +96,6 @@ Prüfe zuerst Adresse, Quelltyp und letzte Scan-Zeit. Öffne danach die Quelle, | **Fehler** mit „Das Embedding-Modell konnte die Seiten nicht verarbeiten.“ | Der Embedding-Anbieter hat die Anfrage abgelehnt oder konnte sie nicht bedienen: abgelehnte Zugangsdaten, aufgebrauchtes Guthaben, ein Ratenlimit, das über alle Wiederholungen anhielt, oder ein Ausfall. Die Details der Quelle sagen, was davon zutrifft, mit der Antwort des Anbieters darunter. Ein Administrator repariert das Modell unter **Einstellungen > Datenresidenz** oder die Zugangsdaten unter **Einstellungen > KI-Anbieter**; beim Speichern werden die betroffenen Quellen erneut gescannt. Nach einem Ausfall beim Anbieter wählst du **Jetzt scannen**, sobald er wieder erreichbar ist. | | **Fehler** mit „Der Scan ist nicht gelaufen.“ | Der Browser des Crawlers konnte nicht starten, deshalb wurde keine Seite dargestellt; sein Bericht steht in den Details der Quelle unter **Technische Details**. An der Adresse liegt es nicht: Lass deinen Betreiber den Sandbox-Dienst prüfen und wähle danach **Jetzt scannen**. | -Ein späterer erfolgreicher Abruf entfernt den vorherigen Fehler. Nach einer fehlgeschlagenen Aktualisierung kann die früher indexierte Fassung weiterhin verfügbar sein: **Aktiv** und die Anzahl indexierter Seiten belegen nicht, dass jede Seite aktuell ist. Vergleiche gespeicherte Textabschnitte und Abrufdatum mit dem Original, bevor du dich auf eine kürzliche Änderung verlässt. +Ein späterer erfolgreicher Abruf entfernt den vorherigen Fehler. Nach einer fehlgeschlagenen Aktualisierung kann die früher indexierte Fassung weiterhin verfügbar sein: **Aktiv** und die Anzahl indexierter Seiten belegen nicht, dass jede Seite aktuell ist. Vergleiche gespeicherte Textabschnitte und Abrufdatum mit dem Original, bevor du dich auf eine kürzliche Änderung verlässt. Hat sich eine Seite geändert, ihr Sitemap-Datum aber nicht, holen die geplanten Scans das innerhalb von sieben Tagen nach; mit **Jetzt scannen** wird sie sofort gelesen. Zeigt die Quelle **Pausiert**, haben wiederholte Verbindungsfehler zur Wissensdatenbank die Scans angehalten. Lass einen Administrator die Verbindung unter **Einstellungen > Datenresidenz** korrigieren und wähle danach **Scans fortsetzen**. diff --git a/docs/en/platform/knowledge/crawling.md b/docs/en/platform/knowledge/crawling.md index 7319e2822a..99219525f2 100644 --- a/docs/en/platform/knowledge/crawling.md +++ b/docs/en/platform/knowledge/crawling.md @@ -42,6 +42,8 @@ For a whole website, the crawler uses the homepage and published sitemaps, inclu Scans are incremental. Unchanged content is skipped, changed content is indexed again, new pages are added, and removed pages leave the index — and so do pages `robots.txt` has come to disallow. A page that redirects to another address of the same site is indexed once, under the address it lands on. The row's page counts follow the scan as pages land, after discovery and after every stored batch, so the table moves while a scan runs. A URL list follows the same refresh schedule with its fixed selection. There is no separate publish step after successful indexing. +A scan does not request every page each time. When a page's sitemap entry carries a `` date and the crawler read the page after that date, the page is left alone until the date moves. A date without a time counts for its whole day, so a page changed today is read again on each scan until the following day. A dated page is still read again once its last read is more than seven days old, because a sitemap can keep a date while the page changes, as a page that lists the latest articles does. A page whose entry has no readable date, or that no sitemap lists, is requested on every scan. A site that publishes accurate `` dates in its sitemap is therefore asked for few pages on a repeat scan. + The crawler visits as an anonymous reader. Content that depends on a private session is not made accessible by adding its URL. It identifies itself on every request as `TaleBot/ (+https://docs.tale.dev/platform/knowledge/crawling)`, so a `robots.txt` group can address it by name — `User-agent: TaleBot` — to allow, throttle or refuse it alone. The crawler applies `robots.txt` `Disallow` rules for the `*` agent on every path a URL can enter by — the sitemaps, the link walk and the links a rendered JavaScript page reveals — and again before every fetch: a page a rule covers is never fetched, and a page a rule added later covers leaves the index on the next scan. The rules do not filter an explicit URL list: a listed address is your instruction. On each content fetch, an HTTP `X-Robots-Tag: noindex` or `none`, or an HTML `` tag, prevents indexing, including for listed URLs, and drops whatever an earlier scan stored of that page. These rules are courtesy, not access control: do not rely on Tale's crawler as an access-control mechanism. @@ -62,7 +64,7 @@ There is no configurable page cap, include/exclude path filter, or stop-scan but ## Check what was indexed -The table shows **Status**, the **Indexed** page count, **Scanned**, and **Interval**. Open the source row to inspect its page list, word and chunk counts, and last-crawled times. Expand a page to read its stored text chunks. A failed fetch shows its reason and number of consecutive failures. A page the crawler skipped on purpose — the source asked not to be indexed, the content type has no readable text, or a redirect left the site — reads **Skipped** with its reason and does not count as failed. +The table shows **Status**, the **Indexed** page count, **Scanned**, and **Interval**. Open the source row to inspect its page list, word and chunk counts, and last-crawled times. A page a scan left alone on its sitemap date keeps the time of its last read. Expand a page to read its stored text chunks. A failed fetch shows its reason and number of consecutive failures. A page the crawler skipped on purpose — the source asked not to be indexed, the content type has no readable text, or a redirect left the site — reads **Skipped** with its reason and does not count as failed. | Status | Meaning | | --- | --- | @@ -73,7 +75,7 @@ The table shows **Status**, the **Indexed** page count, **Scanned**, and **Inter A scan that was under way when Tale restarted, was updated or stopped without warning continues on its own within a few minutes, with the pages it had not reached yet. Pages that need the browser can follow up to a quarter of an hour later. The source reads **Scanning** throughout. -To scan outside the interval, open the source's row menu or its details and choose **Scan now**. Use it after the site changed, or after a failed scan, which otherwise retries on its own within two hours. The action is offered while the source is neither scanning nor being deleted. +To scan outside the interval, open the source's row menu or its details and choose **Scan now**. It requests every page again, whatever the sitemap dates say. Use it after the site changed, or after a failed scan, which otherwise retries on its own within two hours. The action is offered while the source is neither scanning nor being deleted. The page view also offers search over indexed content. Try a distinctive phrase from a page before relying on it in chat, then ask a specific question and inspect the citation. @@ -94,6 +96,6 @@ First check the address, source type, and latest scan time. Then open the source | **Error** with "The embedding model couldn't process the pages." | The embedding provider refused or failed the request: a rejected credential, an exhausted balance, a rate limit that held through every retry, or an outage. The source's details say which, with the provider's own reply beneath. An administrator repairs the model under **Settings > Data residency** or its credential under **Settings > AI providers**; saving either one scans the affected sources again. After an outage on the provider's side, choose **Scan now** once it is back. | | **Error** with "Scanning didn't run." | The crawler's browser could not start, so no page was rendered; its report is under **Technical details** in the source's details. The address is not the cause: ask your operator to check the sandbox service, then choose **Scan now**. | -A successful later fetch clears the previous error. A failed refresh can leave an earlier indexed copy available: **Active** and an indexed count do not prove every page is up to date. Compare the stored chunks and last-crawled information with the original before relying on a recent change. +A successful later fetch clears the previous error. A failed refresh can leave an earlier indexed copy available: **Active** and an indexed count do not prove every page is up to date. Compare the stored chunks and last-crawled information with the original before relying on a recent change. When a page changed and its sitemap date did not, the scheduled scans catch up within seven days; **Scan now** reads it at once. If the source shows **Paused**, repeated failures to reach the knowledge database stopped scans. Ask an administrator to repair the connection under **Settings > Data residency**, then use **Resume scanning**. diff --git a/docs/fr/platform/knowledge/crawling.md b/docs/fr/platform/knowledge/crawling.md index 7dd12e5d9d..e6c33ffc8b 100644 --- a/docs/fr/platform/knowledge/crawling.md +++ b/docs/fr/platform/knowledge/crawling.md @@ -42,6 +42,8 @@ Pour un site entier, le crawler utilise l’accueil et les sitemaps publiés, y Les scans sont incrémentaux : les contenus inchangés sont ignorés, les contenus modifiés réindexés, les nouvelles pages ajoutées et les pages retirées supprimées de l’index — comme les pages que le `robots.txt` en est venu à interdire. Une page qui redirige vers une autre adresse du même site n’est indexée qu’une fois, sous l’adresse où elle aboutit. Les compteurs de pages de la ligne suivent le scan à mesure que les pages arrivent, après la découverte et après chaque lot stocké, la table bouge donc pendant qu’un scan tourne. Une liste d’URL actualise sa sélection fixe au même rythme. Aucune publication séparée n’est nécessaire après l’indexation. +Un scan ne demande pas toutes les pages à chaque passage. Quand l’entrée d’une page dans le sitemap porte une date `` et que le crawler a lu la page après cette date, il la laisse de côté jusqu’à ce que la date change. Une date sans heure vaut pour toute la journée : une page modifiée aujourd’hui est donc relue à chaque scan jusqu’au lendemain. Une page datée est tout de même relue dès que sa dernière lecture remonte à plus de sept jours, car un sitemap peut garder sa date alors que la page change, comme une page qui liste les derniers articles. Une page dont l’entrée n’a pas de date lisible, ou qu’aucun sitemap ne mentionne, est demandée à chaque scan. Un site qui publie des dates `` exactes dans son sitemap reçoit donc peu de requêtes lors d’un nouveau scan. + Le crawler visite en lecteur anonyme. Ajouter une URL ne rend pas accessible un contenu privé. À chaque requête il se présente comme `TaleBot/ (+https://docs.tale.dev/platform/knowledge/crawling)`, si bien qu’un groupe de `robots.txt` peut le désigner par son nom — `User-agent: TaleBot` — pour l’autoriser, le ralentir ou le refuser seul. Le crawler applique les règles `Disallow` de `robots.txt` destinées à l’agent `*` sur chaque chemin par lequel une URL peut entrer — les sitemaps, le parcours de liens et les liens qu’une page JavaScript révèle après rendu — et de nouveau avant chaque récupération : une page qu’une règle couvre n’est jamais récupérée, et une page qu’une règle ajoutée plus tard couvre quitte l’index au scan suivant. Les règles ne filtrent pas une liste d’URL explicite : une adresse listée est ta consigne. À chaque récupération, un en-tête HTTP `X-Robots-Tag: noindex` ou `none`, ou une balise HTML ``, empêche l’indexation, y compris pour une URL de liste, et retire ce qu’un scan antérieur avait stocké de la page. Ces règles sont une courtoisie, pas un contrôle d’accès : si tu administres le site source, ne compte pas sur le crawler comme mécanisme de contrôle d’accès. @@ -62,7 +64,7 @@ Tu ne peux pas fixer ton propre plafond de pages, filtrer les chemins à inclure ## Vérifier le contenu indexé -Le tableau affiche **Statut**, **Indexé**, **Analysé** et **Intervalle**. La colonne **Indexé** indique un nombre de pages. Ouvre la ligne du site pour examiner la liste des pages, le nombre de mots et de fragments et la date du dernier passage. Déplie une page pour lire ses fragments de texte enregistrés. Un échec de récupération affiche sa cause et le nombre d’échecs consécutifs. Une page que le crawler laisse volontairement de côté — la source refuse l’indexation, le type de contenu ne fournit aucun texte lisible, ou une redirection quitte le site — apparaît comme **Ignorée** avec sa cause et ne compte pas comme un échec. +Le tableau affiche **Statut**, **Indexé**, **Analysé** et **Intervalle**. La colonne **Indexé** indique un nombre de pages. Ouvre la ligne du site pour examiner la liste des pages, le nombre de mots et de fragments et la date du dernier passage. Une page qu’un scan a laissée de côté en raison de sa date dans le sitemap garde la date de sa dernière lecture. Déplie une page pour lire ses fragments de texte enregistrés. Un échec de récupération affiche sa cause et le nombre d’échecs consécutifs. Une page que le crawler laisse volontairement de côté — la source refuse l’indexation, le type de contenu ne fournit aucun texte lisible, ou une redirection quitte le site — apparaît comme **Ignorée** avec sa cause et ne compte pas comme un échec. | Statut | Signification | | --- | --- | @@ -73,7 +75,7 @@ Le tableau affiche **Statut**, **Indexé**, **Analysé** et **Intervalle**. La c Un scan en cours au moment où Tale a été redémarré, mis à jour ou arrêté brutalement reprend de lui-même en quelques minutes, avec les pages qu’il n’avait pas encore atteintes. Les pages qui ont besoin du navigateur peuvent suivre jusqu’à un quart d’heure plus tard. La source affiche **En cours d'analyse** pendant tout ce temps. -Pour lancer un scan en dehors de l’intervalle, ouvre le menu de la ligne ou les détails de la source et choisis **Analyser maintenant**. C’est utile quand le site a changé ou après un scan en échec, qui sinon est relancé automatiquement dans les deux heures. L’action est proposée tant que la source n’est ni en cours d’analyse ni en cours de suppression. +Pour lancer un scan en dehors de l’intervalle, ouvre le menu de la ligne ou les détails de la source et choisis **Analyser maintenant**. Le scan redemande alors toutes les pages, quelles que soient les dates du sitemap. C’est utile quand le site a changé ou après un scan en échec, qui sinon est relancé automatiquement dans les deux heures. L’action est proposée tant que la source n’est ni en cours d’analyse ni en cours de suppression. La vue des pages permet aussi de rechercher dans le contenu indexé. Essaie une expression distinctive de la page avant de t’appuyer dessus dans Chat, puis pose une question précise et vérifie la citation. @@ -94,6 +96,6 @@ Vérifie d’abord l’adresse, le type de source et la date du dernier scan. Ou | **Erreur** avec « Le modèle d'embedding n'a pas pu traiter les pages. » | Le fournisseur d’embedding a refusé la requête ou n’a pas pu y répondre : identifiants rejetés, solde épuisé, limite de débit maintenue malgré toutes les tentatives, ou panne. Les détails de la source précisent lequel, avec la réponse du fournisseur en dessous. Un administrateur corrige le modèle dans **Paramètres > Résidence des données** ou ses identifiants dans **Paramètres > Fournisseurs IA** ; l’enregistrement relance le scan des sources concernées. Après une panne chez le fournisseur, choisis **Analyser maintenant** dès qu’il est de nouveau disponible. | | **Erreur** avec « L'analyse n'a pas eu lieu. » | Le navigateur du crawler n’a pas pu démarrer, aucune page n’a donc été rendue ; son rapport se trouve sous **Détails techniques** dans les détails de la source. L’adresse n’est pas en cause : demande à ton exploitant de vérifier le service de sandbox, puis choisis **Analyser maintenant**. | -Une récupération réussie efface l’erreur précédente. Si une actualisation échoue, la copie déjà indexée peut rester disponible : **Actif** et le nombre de pages indexées ne prouvent pas que chaque page est à jour. Compare les fragments enregistrés et la date du passage avec l’original avant de t’appuyer sur une modification récente. +Une récupération réussie efface l’erreur précédente. Si une actualisation échoue, la copie déjà indexée peut rester disponible : **Actif** et le nombre de pages indexées ne prouvent pas que chaque page est à jour. Compare les fragments enregistrés et la date du passage avec l’original avant de t’appuyer sur une modification récente. Quand une page a changé sans que sa date dans le sitemap change, les scans planifiés la rattrapent en sept jours au plus ; **Analyser maintenant** la relit tout de suite. Si la source affiche **Suspendu**, des échecs répétés d’accès à la base de connaissances ont arrêté les analyses. Demande à un administrateur de réparer la connexion dans **Paramètres > Résidence des données**, puis utilise **Reprendre l'analyse**. diff --git a/services/db/migrations/knowledge-db/public_web/00000000000013_website_url_sitemap_lastmod.sql b/services/db/migrations/knowledge-db/public_web/00000000000013_website_url_sitemap_lastmod.sql new file mode 100644 index 0000000000..a8d65db272 --- /dev/null +++ b/services/db/migrations/knowledge-db/public_web/00000000000013_website_url_sitemap_lastmod.sql @@ -0,0 +1,34 @@ +-- migrate:up +-- +-- What a site's sitemap says about when a page last changed, kept on the +-- page's row so a rescan can leave an unchanged page alone. Until here the +-- crawler read `` out of a sitemap and nothing else: every scan +-- requested every page again and compared content hashes afterwards, so a +-- 700-page site scanned every six hours answered some 8,000 requests and +-- two gigabytes a scan for pages its sitemap marked unchanged for months +-- (2026-10-06). +-- +-- sitemap_lastmod the latest instant the entry's `` can mean, as +-- the discovery of the last scan read it: a date-only +-- value covers its whole day in any time zone, a time +-- without a zone the same half day. NULL when that +-- discovery read none for the page — no sitemap lists +-- it, the entry carries no date or one that cannot be +-- read, or a person started the scan — and the page is +-- then requested as before. +-- +-- A page is left alone only while its stored text was read after this +-- instant; the rule, its margin and the weekly re-read live in the frontier +-- predicate (`crawl_action.ts`). Each discovery replaces what the one before +-- recorded, so a date a sitemap no longer states never keeps a page skipped. +-- +-- Pure ADD COLUMN IF NOT EXISTS: metadata-only, idempotent, rolling-safe — +-- the previous image neither reads nor writes the column. + +ALTER TABLE public_web.website_urls + ADD COLUMN IF NOT EXISTS sitemap_lastmod TIMESTAMPTZ; + +-- migrate:down +-- Deliberately empty, like the baseline: dropping the column would only send +-- every scan back to requesting every page. Remove with an explicit, +-- reviewed migration instead. diff --git a/services/platform/backend/core/knowledge/crawl.test.ts b/services/platform/backend/core/knowledge/crawl.test.ts index 7a7c3e40c3..49ea01626c 100644 --- a/services/platform/backend/core/knowledge/crawl.test.ts +++ b/services/platform/backend/core/knowledge/crawl.test.ts @@ -8,6 +8,7 @@ import { admitUrls, admitUrlsStatement, deregisterDomain, + recordSitemapLastmods, registerDomain, registerUrlList, reviveListedUrls, @@ -271,3 +272,80 @@ describe('reviveListedUrls', () => { expect(call?.params).toEqual(['example.test']); }); }); + +/** + * A page's sitemap date is what a rescan leaves it alone by, so each + * discovery replaces the record: a date the sitemap no longer states must + * not keep its page skipped, and a scan that records none requests every + * page. + */ +describe('recordSitemapLastmods', () => { + const DOMAIN = 'docs.example'; + + it('forgets the dates this scan did not read and writes the ones that moved, in one transaction', async () => { + const { sql, sent, begins } = recorder(); + + await recordSitemapLastmods( + sql, + DOMAIN, + new Map([ + ['https://docs.example/a', new Date('2026-10-07T12:00:00.000Z')], + ['https://docs.example/b', new Date('2026-01-03T12:00:00.000Z')], + ]), + ); + + expect(begins()).toBe(1); + expect(sent.every((statement) => statement.inTx)).toBe(true); + expect(sent).toHaveLength(2); + const [forget, write] = sent; + expect(forget?.text).toContain('SET sitemap_lastmod = NULL'); + expect(forget?.text).toContain('sitemap_lastmod IS NOT NULL'); + expect(forget?.text).toContain('NOT (url = ANY($2::text[]))'); + expect(forget?.params).toEqual([ + DOMAIN, + ['https://docs.example/a', 'https://docs.example/b'], + ]); + expect(write?.text).toContain( + 'SET sitemap_lastmod = v.lastmod::timestamptz', + ); + // A date that has not moved is not written again. + expect(write?.text).toContain( + 'u.sitemap_lastmod IS DISTINCT FROM v.lastmod::timestamptz', + ); + expect(write?.params).toEqual([ + DOMAIN, + ['https://docs.example/a', 'https://docs.example/b'], + ['2026-10-07T12:00:00.000Z', '2026-01-03T12:00:00.000Z'], + ]); + }); + + it('forgets every date and writes none when the scan read none', async () => { + const { sql, sent } = recorder(); + + await recordSitemapLastmods(sql, DOMAIN, new Map()); + + expect(sent).toHaveLength(1); + expect(sent[0]?.text).toContain('SET sitemap_lastmod = NULL'); + expect(sent[0]?.params).toEqual([DOMAIN, []]); + }); + + it('writes a large sitemap in batches', async () => { + const { sql, sent } = recorder(); + const lastmods = new Map( + Array.from({ length: URL_INSERT_BATCH + 1 }, (_, index) => [ + `https://docs.example/${index}`, + new Date('2026-01-03T12:00:00.000Z'), + ]), + ); + + await recordSitemapLastmods(sql, DOMAIN, lastmods); + + const writes = sent.filter((statement) => + statement.text.includes('FROM unnest('), + ); + expect(writes.map((statement) => statement.params[1])).toEqual([ + expect.objectContaining({ length: URL_INSERT_BATCH }), + expect.objectContaining({ length: 1 }), + ]); + }); +}); diff --git a/services/platform/backend/core/knowledge/crawl.ts b/services/platform/backend/core/knowledge/crawl.ts index 547060c82a..1e4ef189be 100644 --- a/services/platform/backend/core/knowledge/crawl.ts +++ b/services/platform/backend/core/knowledge/crawl.ts @@ -145,6 +145,53 @@ export async function reviveListedUrls( return rows.length; } +/** + * Record what this scan's sitemaps say about each page's last change, and + * forget what an earlier scan's said — the two in one transaction, so the + * frontier never reads half of either. + * + * `lastmods` REPLACES the domain's record: a page it does not name loses the + * date an earlier discovery wrote. That is what keeps a stale statement from + * skipping a page for ever — the sitemap dropped the entry or its date, the + * sitemap could not be fetched this time, discovery ran out of its budget + * before reaching it — and what an empty map means: a scan a person started + * records none, so it requests every page. A date that has not moved is not + * written again; on an ordinary rescan nearly every row is left untouched. + */ +export async function recordSitemapLastmods( + sql: Sql, + domain: string, + lastmods: ReadonlyMap, +): Promise { + const entries = [...lastmods]; + await sql.begin(async (tx) => { + await tx.unsafe( + `UPDATE ${PUBLIC_WEB_SCHEMA}.website_urls + SET sitemap_lastmod = NULL + WHERE domain = $1 AND sitemap_lastmod IS NOT NULL + AND NOT (url = ANY($2::text[]))`, + [domain, entries.map(([url]) => url)], + ); + for (let start = 0; start < entries.length; start += URL_INSERT_BATCH) { + const batch = entries.slice(start, start + URL_INSERT_BATCH); + // The instants travel as text and are cast by the server, as a scan's + // start does in the admission statement. + await tx.unsafe( + `UPDATE ${PUBLIC_WEB_SCHEMA}.website_urls AS u + SET sitemap_lastmod = v.lastmod::timestamptz + FROM unnest($2::text[], $3::text[]) AS v(url, lastmod) + WHERE u.domain = $1 AND u.url = v.url + AND u.sitemap_lastmod IS DISTINCT FROM v.lastmod::timestamptz`, + [ + domain, + batch.map(([url]) => url), + batch.map(([, lastmod]) => lastmod.toISOString()), + ], + ); + } + }); +} + /** Corpus → websites-row status vocabulary. The corpus distinguishes * `completed` (a finished scan) from `active`; the websites row treats both * as a healthy scanned site. A corpus row is born `idle` (the column diff --git a/services/platform/backend/core/knowledge/crawl_action.lastmod.test.ts b/services/platform/backend/core/knowledge/crawl_action.lastmod.test.ts new file mode 100644 index 0000000000..8070e76ee6 --- /dev/null +++ b/services/platform/backend/core/knowledge/crawl_action.lastmod.test.ts @@ -0,0 +1,255 @@ +// @vitest-environment node + +import type { Sql } from 'postgres'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { safeFetch } from '../../../lib/net/safe-fetch'; +import type { ActionCtx } from '../lib/ctx'; +import { scanWebsiteImpl } from './crawl_action'; +import { getKnowledgePoolForOrg } from './pool'; + +vi.mock('./pool', async (importOriginal) => ({ + ...(await importOriginal()), + getKnowledgePoolForOrg: vi.fn(), +})); +vi.mock('../../../lib/net/safe-fetch', async (importOriginal) => ({ + ...(await importOriginal()), + safeFetch: vi.fn(), +})); + +/** + * A rescan leaves alone the pages a site's sitemap says have not changed. + * The regression: the crawler read `` out of a sitemap and nothing + * else, so every scan requested every page again — probe, browser render + * and all the assets a render loads — and compared content hashes + * afterwards. A site that dated all 700 of its pages answered some 8,000 + * requests a scan, every six hours, for pages unchanged in months. + * + * Pinned on a recording double (which statements run, with which + * parameters); what the frontier predicate selects on real rows rides the + * integration check. + */ + +const SCAN = { + domain: 'docs.example', + orgSlug: 'acme', + organizationId: 'org-1', +}; + +/** Twelve pages, over the link-walk threshold, so discovery is the sitemap + * alone. The first two carry a date, the homepage twice. */ +const SITEMAP = ` + + https://docs.example2026-07-27 + https://docs.example/guide2026-10-06T08:00:00+02:00 + https://docs.example/2026-08-01 + https://docs.example/undated + https://docs.example/unreadablelast week + ${Array.from( + { length: 8 }, + (_, index) => `https://docs.example/more/${index}`, + ).join('\n ')} +`; + +interface Statement { + text: string; + params: unknown[]; + inTx: boolean; +} + +/** A corpus double that grants the claim and answers the frontier as + * empty: the link discovers, records and finishes. */ +function corpus(): { sql: Sql; statements: Statement[] } { + const statements: Statement[] = []; + const unsafeFor = + (inTx: boolean) => + (text: string, params: unknown[] = []): Promise => { + const statement = text.replace(/\s+/g, ' ').trim(); + statements.push({ text: statement, params, inTx }); + if (statement.startsWith('SELECT kind')) { + return Promise.resolve([{ kind: 'site', robots_disallow: null }]); + } + if (statement.includes("SET status = 'scanning'")) { + return Promise.resolve([{ domain: SCAN.domain }]); + } + if (statement.includes('AND COALESCE(')) { + return Promise.resolve([{ n: '2' }]); + } + if (statement.includes('FILTER (WHERE u.status')) { + return Promise.resolve([ + { stored: '12', attempted: '12', failed: '0', skipped: '0' }, + ]); + } + return Promise.resolve([]); + }; + const sql = { + unsafe: unsafeFor(false), + begin: async ( + run: (tx: { unsafe: ReturnType }) => Promise, + ) => run({ unsafe: unsafeFor(true) }), + }; + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- test double + return { sql: sql as unknown as Sql, statements }; +} + +function engineCtx(): ActionCtx { + const ctx = { + runMutation: vi.fn(async () => ({ paused: false })), + runQuery: vi.fn(), + scheduler: { runAfter: vi.fn() }, + }; + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only what the link reaches + return ctx as unknown as ActionCtx; +} + +const forgotten = (statements: Statement[]) => + statements.filter((statement) => + statement.text.includes('SET sitemap_lastmod = NULL'), + ); +const written = (statements: Statement[]) => + statements.filter((statement) => statement.text.includes('FROM unnest(')); +const frontier = (statements: Statement[]) => + statements.filter( + (statement) => + statement.text.startsWith('SELECT url, content_hash, listed') || + (statement.text.startsWith('SELECT count(*)::text AS n') && + statement.text.includes('AND NOT COALESCE(')), + ); + +const logged = (): string => + vi + .mocked(console.log) + .mock.calls.map((call) => String(call[0])) + .join('\n'); + +beforeEach(() => { + vi.mocked(getKnowledgePoolForOrg).mockReset(); + vi.mocked(safeFetch).mockReset(); + vi.mocked(safeFetch).mockImplementation(async (url) => { + const answer = (body: string) => ({ + status: 200, + statusText: 'OK', + headers: new Headers(), + body, + finalUrl: url, + }); + if (url.endsWith('/robots.txt')) { + return answer( + `User-agent: *\nSitemap: https://${SCAN.domain}/sitemap.xml\n`, + ); + } + if (url.endsWith('/sitemap.xml')) return answer(SITEMAP); + return { ...answer('gone'), status: 404, statusText: 'Not Found' }; + }); + // Cleared: a spy on a method that is already spied on keeps its calls. + vi.spyOn(console, 'log') + .mockClear() + .mockImplementation(() => undefined); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); +}); + +describe("scanWebsiteImpl — what a site's sitemap says about its pages", () => { + it('records the date of every page whose entry states one, and forgets the rest', async () => { + const { sql, statements } = corpus(); + vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); + + await scanWebsiteImpl(engineCtx(), SCAN); + + // The homepage is named twice: the later date stands. A date alone + // covers its whole day in any time zone. + const dated = [ + ['https://docs.example/', '2026-08-02T12:00:00.000Z'], + ['https://docs.example/guide', '2026-10-06T06:00:00.000Z'], + ]; + expect(written(statements).map((statement) => statement.params)).toEqual([ + [ + SCAN.domain, + dated.map(([url]) => url), + dated.map(([, lastmod]) => lastmod), + ], + ]); + // An entry without a date, or with one that cannot be read, leaves its + // page without one: such a page is requested. + expect(forgotten(statements).map((statement) => statement.params)).toEqual([ + [SCAN.domain, dated.map(([url]) => url)], + ]); + expect( + [...forgotten(statements), ...written(statements)].every( + (statement) => statement.inTx, + ), + ).toBe(true); + // After the admission: a page found by this scan has its row by then. + const texts = statements.map((statement) => statement.text); + expect( + texts.findIndex((text) => text.includes('RETURNING u.url')), + ).toBeLessThan(texts.findIndex((text) => text.includes('FROM unnest('))); + expect(logged()).toContain( + '12 URLs discovered, 2 not requested (unchanged by their sitemap dates)', + ); + }); + + it('leaves a page alone only while its stored text was read after the date, within the week [KNOW-R16]', async () => { + const { sql, statements } = corpus(); + vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); + + await scanWebsiteImpl(engineCtx(), SCAN); + + // Both readers of the frontier: the pages a link takes next, and how + // many are left for the next link. + const readers = frontier(statements); + expect(readers).toHaveLength(2); + for (const reader of readers) { + // Without a date, or never read, the clause is NULL and the page due. + expect(reader.text).toContain('AND NOT COALESCE(('); + expect(reader.text).toContain(', FALSE)'); + // The row holds the text of a read that succeeded. + expect(reader.text).toContain( + "status = 'active' AND content_hash IS NOT NULL AND last_error IS NULL", + ); + // That read came after the date, and a margin for a cache. + expect(reader.text).toContain( + "last_crawled_at > sitemap_lastmod + interval '1 hour'", + ); + // And within the week: a sitemap that never moves its dates. + expect(reader.text).toContain( + "last_crawled_at > $2::timestamptz - interval '7 days'", + ); + // Text stored and not indexed yet stays due. + expect(reader.text).toContain( + 'c.content_hash IS DISTINCT FROM website_urls.content_hash', + ); + } + }); + + it('records no date on a scan a person started, so every page is requested [KNOW-R16]', async () => { + const { sql, statements } = corpus(); + vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); + + await scanWebsiteImpl(engineCtx(), { ...SCAN, full: true }); + + expect(forgotten(statements).map((statement) => statement.params)).toEqual([ + [SCAN.domain, []], + ]); + expect(written(statements)).toEqual([]); + expect(logged()).toContain('12 URLs discovered'); + expect(logged()).not.toContain('not requested'); + }); + + // A later link runs no discovery: it reads the frontier under the dates + // the first link recorded. + it('reads the frontier under the same rule on a later link, and records nothing', async () => { + const { sql, statements } = corpus(); + vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); + + await scanWebsiteImpl(engineCtx(), { + ...SCAN, + continuation: 1, + scanStartedAt: '2026-10-06T13:00:00.000Z', + }); + + expect(safeFetch).not.toHaveBeenCalled(); + expect(forgotten(statements)).toEqual([]); + expect(written(statements)).toEqual([]); + expect(frontier(statements)).toHaveLength(2); + }); +}); diff --git a/services/platform/backend/core/knowledge/crawl_action.ts b/services/platform/backend/core/knowledge/crawl_action.ts index 70a1feb35d..4690f75402 100644 --- a/services/platform/backend/core/knowledge/crawl_action.ts +++ b/services/platform/backend/core/knowledge/crawl_action.ts @@ -13,7 +13,9 @@ * (`renderUrlsInSandbox`) so JS-rendered sites yield their real content. * The in-process probe stays the authority on page lifecycle — status * codes, deletes, size caps, SSRF guards — and content-hash comparison is - * the only change detection. + * the only change detection for a page that is requested. A page whose + * sitemap entry says it has not changed since it was last read is not + * requested at all (`UNCHANGED_BY_SITEMAP_SQL`). * * A scan is a CONTINUATION CHAIN, not one long action: a Convex node action * is hard-killed near ten minutes without running its catch, so each link @@ -50,6 +52,7 @@ import { normalizeCandidateUrl, paragraphsForHashing, parseRobots, + parseSitemapEntries, parseSitemapLocs, publicPageError, renderLaneHaltMessage, @@ -109,7 +112,12 @@ import { } from '../websites/scan_scheduling'; import { type PageFailureKind, PAGE_SKIP_KINDS_SQL } from '../websites/types'; import { readOrgEmbeddingConfig } from './connection'; -import { MAX_URLS_PER_DOMAIN, admitUrls, reviveListedUrls } from './crawl'; +import { + MAX_URLS_PER_DOMAIN, + admitUrls, + recordSitemapLastmods, + reviveListedUrls, +} from './crawl'; import { crawlDocumentMaxBytes } from './crawl_limits'; import { CRAWLER_PRODUCT_TOKEN, @@ -212,6 +220,19 @@ const MAX_FETCH_FAILURES = 5; * still lets a page revived on the site come back on its own. */ const BENCHED_PAGE_RETRY_INTERVAL = '7 days'; +/** How long a page is left alone on its sitemap's word that it has not + * changed ({@link UNCHANGED_BY_SITEMAP_SQL}) — a Postgres interval. Past it + * the page is read again whatever its entry says: a listing page changes + * with every article it lists and keeps its date, and a site can publish + * dates it never moves. A week bounds how stale such a page gets, at one + * request a week. */ +const SITEMAP_LASTMOD_MAX_AGE = '7 days'; +/** How long after the change a sitemap records a read of the page still + * counts as too early — a Postgres interval. A cache in front of the site + * can go on serving the earlier text for a while after the edit, and a read + * that met it would otherwise stand as the current one for a week. */ +const SITEMAP_LASTMOD_SETTLE = '1 hour'; + /** Pages the vector backfill reads per query (`PageIndexer.embedVectorless`). */ const EMBED_BATCH_PAGES = 20; @@ -252,6 +273,13 @@ export async function scanWebsiteImpl( * takes exactly that claim over instead of waiting out * {@link STUCK_SCAN_TAKEOVER}; a claim that moved since is left alone. */ takeover?: string; + /** Set on the first link of a scan a person started with "Scan now": + * every page is requested, whatever the site's sitemap says about when + * it last changed. Any other scan leaves an unchanged page alone + * ({@link UNCHANGED_BY_SITEMAP_SQL}) — a scheduled one, and the one the + * scheduler resumes after a restart, which continues as an ordinary + * scan whatever it began as. */ + full?: boolean; /** Aborted once the job this link runs in has ended under it — the * process is stopping, or the link outlived the job's expiry. */ signal?: AbortSignal; @@ -345,6 +373,7 @@ export async function scanWebsiteImpl( actionStartedAt + DISCOVERY_BUDGET_MS, robots, scanStartedAt, + { full: args.full === true }, ); const retired = await retireDisallowedRows(sql, args.domain, policy); if (retired > 0) { @@ -919,13 +948,20 @@ async function claimScan( * as the fallback) under the robots rules the scan read, and record them as * `discovered` rows for the fetch loop. `deadline` bounds the fetching: a * discovery cut short records what it has — the next scan's discovery pass - * tops the frontier up. */ + * tops the frontier up. + * + * It also records what the sitemaps say about each page's last change, which + * is what lets the fetch loop leave an unchanged page alone + * ({@link UNCHANGED_BY_SITEMAP_SQL}). A `full` scan — one a person started + * with "Scan now" — records none, and so requests every page: that is how + * someone says the site changed, whatever its sitemap claims. */ async function discoverAndRecordUrls( sql: Sql, domain: string, deadline: number, robots: RobotsRules, scanStartedAt: string, + options: { full: boolean }, ): Promise { const hosts = siteHosts(domain); const baseUrl = `https://${domain}/`; @@ -949,12 +985,19 @@ async function discoverAndRecordUrls( if (advertised.length > 0) sitemapCandidates = advertised; const urls = new Set(); - const admit = (candidate: string): boolean => { + // What the sitemaps say about the last change of each admitted page — the + // latest date when several entries name one page. + const lastmods = new Map(); + const admit = (candidate: string, lastmod: Date | null = null): boolean => { const normalized = normalizeCandidateUrl(candidate, baseUrl, hosts); if (!normalized) return false; if (isUrlDisallowed(normalized, robots)) return false; if (urls.size >= MAX_URLS_PER_DOMAIN) return true; urls.add(normalized); + const known = lastmods.get(normalized); + if (lastmod !== null && (known === undefined || lastmod > known)) { + lastmods.set(normalized, lastmod); + } return urls.size >= MAX_URLS_PER_DOMAIN; }; admit(baseUrl); @@ -1020,8 +1063,8 @@ async function discoverAndRecordUrls( continue; } let capped = false; - for (const loc of parseSitemapLocs(xml)) { - capped = admit(loc); + for (const entry of parseSitemapEntries(xml)) { + capped = admit(entry.loc, entry.lastmod); if (capped) break; } if (capped) break; @@ -1091,7 +1134,19 @@ async function discoverAndRecordUrls( listed: false, retiredBefore: scanStartedAt, }); - console.log(`[crawl] ${domain}: ${urls.size} URLs discovered`); + // After the admission: a page this scan found first has a row to carry + // its date only now. + await recordSitemapLastmods(sql, domain, options.full ? new Map() : lastmods); + const unchanged = options.full + ? 0 + : await countUnchangedBySitemap(sql, domain, scanStartedAt); + console.log( + `[crawl] ${domain}: ${urls.size} URLs discovered${ + unchanged > 0 + ? `, ${unchanged} not requested (unchanged by their sitemap dates)` + : '' + }`, + ); } interface DuePage { @@ -1102,11 +1157,60 @@ interface DuePage { readonly listed: boolean; } +/** + * True for a page a scan leaves alone on its sitemap's word: the entry's + * `` says the page has not changed since the crawler read it. + * `$2` is the scan's start. It holds only when every part does: + * + * - the row carries the text of a read that succeeded (`active`, a hash, no + * error) — so `last_crawled_at` is when that text was read, not when an + * attempt failed; + * - that read came after the instant the date can mean, and + * {@link SITEMAP_LASTMOD_SETTLE} more; + * - it is younger than {@link SITEMAP_LASTMOD_MAX_AGE}; + * - no chunk of the page was cut from other text: a scan that stored the + * new text and stopped before indexing it (an embedding model that + * failed) leaves the page due until it is indexed. + * + * It is NULL, not true, for a page without a sitemap date — one no sitemap + * lists, one found by a link, one of a URL list (a list has no discovery to + * read a date in) — and for a page never read: every doubt requests the + * page, as every scan did before (2026-10-06: a site whose sitemap dated + * all 700 of its pages answered some 8,000 requests a scan, every six + * hours, for pages unchanged in months). The content hash stays the judge + * of what a request brings back. + */ +const UNCHANGED_BY_SITEMAP_SQL = `( + status = 'active' AND content_hash IS NOT NULL AND last_error IS NULL + AND last_crawled_at > sitemap_lastmod + interval '${SITEMAP_LASTMOD_SETTLE}' + AND last_crawled_at > $2::timestamptz - interval '${SITEMAP_LASTMOD_MAX_AGE}' + AND NOT EXISTS ( + SELECT 1 FROM ${PUBLIC_WEB_SCHEMA}.chunks c + WHERE c.domain = website_urls.domain AND c.url = website_urls.url + AND c.content_hash IS DISTINCT FROM website_urls.content_hash))`; + const DUE_PAGE_PREDICATE = ` domain = $1 AND status <> 'deleted' AND (listed OR fail_count < ${MAX_FETCH_FAILURES} OR last_crawled_at < $2::timestamptz - interval '${BENCHED_PAGE_RETRY_INTERVAL}') - AND (last_crawled_at IS NULL OR last_crawled_at < $2::timestamptz)`; + AND (last_crawled_at IS NULL OR last_crawled_at < $2::timestamptz) + AND NOT COALESCE(${UNCHANGED_BY_SITEMAP_SQL}, FALSE)`; + +/** How many of the domain's pages this scan leaves alone on their sitemap's + * word — what the scan's log says beside how many it discovered. */ +async function countUnchangedBySitemap( + sql: Sql, + domain: string, + scanStartedAt: string, +): Promise { + const rows = await sql.unsafe<{ n: string }[]>( + `SELECT count(*)::text AS n FROM ${PUBLIC_WEB_SCHEMA}.website_urls + WHERE domain = $1 AND status <> 'deleted' + AND COALESCE(${UNCHANGED_BY_SITEMAP_SQL}, FALSE)`, + [domain, scanStartedAt], + ); + return Number(rows[0]?.n ?? 0); +} /** The next URLs this scan has not visited yet (never-crawled first). The * batch size matches the render batch, so one claim's HTML pages fill at diff --git a/services/platform/backend/domains/knowledge/spec.md b/services/platform/backend/domains/knowledge/spec.md index 0a42a37bda..2adc31f66d 100644 --- a/services/platform/backend/domains/knowledge/spec.md +++ b/services/platform/backend/domains/knowledge/spec.md @@ -269,6 +269,24 @@ networks for the crawler. networks, and the operator then turns that off → the next scan requests none of its pages and shows each as failed, with a private address as the reason. +### KNOW-R16 · The crawler does not request again a page its sitemap says is unchanged + +A site's sitemap can state when each page last changed. A scheduled scan leaves a page alone +when the crawler read it after that date, and requests every other page: one without a date, +one whose date moved, one whose last read failed. A date without a time counts for its whole +day. A page is requested again once its last read is more than seven days old, whatever its +date says, because a sitemap can keep a date while the page changes. **Scan now** requests +every page. + +- **Example**: The sitemap of `docs.example` dates `/pricing` 1 March, and the crawler last + read the page on 5 March → the scan on 6 March does not request it. +- **Example**: The site changes `/pricing` and its sitemap dates the page 7 March → the next + scan requests it. +- **Example**: `/blog` lists every new article and keeps its date of 1 January → it is + requested again a week after its last read. +- **Example**: Noah selects **Scan now** → every page of `docs.example` is requested, + `/pricing` included. + ## Not yet - **The knowledge settings**: who can change the embedding model and the knowledge database, diff --git a/services/platform/backend/domains/websites/service.embedding-follow.test.ts b/services/platform/backend/domains/websites/service.embedding-follow.test.ts index 8968a543e3..bb73a23d1a 100644 --- a/services/platform/backend/domains/websites/service.embedding-follow.test.ts +++ b/services/platform/backend/domains/websites/service.embedding-follow.test.ts @@ -122,6 +122,11 @@ describe('websitesAfterEmbeddingChange', () => { expect.objectContaining({ domain: 'docs.example' }), expect.objectContaining({ domain: 'failed.example' }), ]); + // Nothing says these sites changed: the scans embed stored text and + // retry what failed, and ask for no unchanged page again. + expect(scanJobs()).not.toContainEqual( + expect.objectContaining({ full: true }), + ); // The organization-wide hint the readiness notice re-reads on. expect(hints(queries)).toContainEqual(['org-1', null, 'website', null]); }); diff --git a/services/platform/backend/domains/websites/service.scan-now.test.ts b/services/platform/backend/domains/websites/service.scan-now.test.ts index 240178384f..a73854194f 100644 --- a/services/platform/backend/domains/websites/service.scan-now.test.ts +++ b/services/platform/backend/domains/websites/service.scan-now.test.ts @@ -38,7 +38,12 @@ import { } from '../../core/knowledge/crawl.ts'; import { scanWebsiteImpl } from '../../core/knowledge/crawl_action.ts'; import { addJobInTx } from '../../jobs/enqueue.ts'; -import { runWebsitesScan, scanWebsiteNow, type WebsiteRow } from './service.ts'; +import { + resumeScanning, + runWebsitesScan, + scanWebsiteNow, + type WebsiteRow, +} from './service.ts'; const row = (overrides: Partial = {}): WebsiteRow => ({ id: 'w-1', @@ -118,7 +123,12 @@ describe('scanWebsiteNow [WEB-R8]', () => { await expect(scanWebsiteNow(sql, row())).resolves.toEqual({ queued: true }); expect(scanJobs()).toEqual([ - { domain: 'example.com', orgSlug: 'acme', organizationId: 'org-1' }, + { + domain: 'example.com', + orgSlug: 'acme', + organizationId: 'org-1', + full: true, + }, ]); const update = queries.find((q) => q.text.startsWith('UPDATE app.websites'), @@ -145,6 +155,50 @@ describe('scanWebsiteNow [WEB-R8]', () => { ); }); +/** + * A scheduled scan leaves alone the pages a site's sitemap says have not + * changed. "Scan now" is how a person says the site did change, so its scan + * asks for every page; a scan the platform starts for its own reasons does + * not. + */ +describe('a scan a person starts, and one the platform starts', () => { + const ordinary = { + domain: 'example.com', + orgSlug: 'acme', + organizationId: 'org-1', + }; + + it('Scan now asks for every page [KNOW-R16]', async () => { + await scanWebsiteNow(fakeSql([row()]).sql, row()); + + expect(scanJobs()).toEqual([{ ...ordinary, full: true }]); + }); + + it('a scan the platform starts itself stays an ordinary one', async () => { + await scanWebsiteNow(fakeSql([row()]).sql, row(), { full: false }); + + expect(scanJobs()).toEqual([ordinary]); + }); + + // Resuming answers whether scanning works again, not a change of the site. + it('Resume scanning queues an ordinary scan', async () => { + await resumeScanning(fakeSql([row()]).sql, row()); + + expect(scanJobs()).toEqual([ordinary]); + }); + + it('the first link hands the word on to the engine', async () => { + vi.mocked(isMemberDomain).mockResolvedValue(true); + + await runWebsitesScan(fakeSql([row()]).sql, { ...ordinary, full: true }); + + expect(scanWebsiteImpl).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ ...ordinary, full: true }), + ); + }); +}); + describe('runWebsitesScan — a registration that never landed', () => { const payload = { domain: 'example.com', diff --git a/services/platform/backend/domains/websites/service.ts b/services/platform/backend/domains/websites/service.ts index 6f58d67e09..eee6ec8cec 100644 --- a/services/platform/backend/domains/websites/service.ts +++ b/services/platform/backend/domains/websites/service.ts @@ -1303,7 +1303,9 @@ export async function websitesAfterEmbeddingChange( for (const website of rows) { if (scanPausedAt(website.metadata ?? undefined) !== null) continue; if (website.status !== 'error' && !vectorless.has(website.domain)) continue; - if ((await scanWebsiteNow(sql, website)).queued) queued += 1; + if ((await scanWebsiteNow(sql, website, { full: false })).queued) { + queued += 1; + } } return { queued }; } @@ -1347,8 +1349,15 @@ export async function deregisterAndDeleteWebsite( * Put a site back on the crawl now: the row reads `scanning`, its failure * bookkeeping is cleared and a scan is queued — in one transaction, so a row * never reads `scanning` without the job that will scan it. + * + * A `full` scan requests every page; any other leaves alone the pages the + * site's sitemap says have not changed, as a scheduled scan does. */ -async function queueScan(sql: Sql, website: WebsiteRow): Promise { +async function queueScan( + sql: Sql, + website: WebsiteRow, + options: { full: boolean }, +): Promise { const orgSlug = await requireSlug(sql, website.organizationId); await sql.begin(async (tx) => { await patchWebsite(tx, { @@ -1369,16 +1378,19 @@ async function queueScan(sql: Sql, website: WebsiteRow): Promise { domain: website.domain, orgSlug, organizationId: website.organizationId, + ...(options.full ? { full: true } : {}), }); }); } -/** Resume paused scans and kick a verification scan now (the 0.4 twin). */ +/** Resume paused scans and kick a verification scan now (the 0.4 twin). The + * scan answers whether scanning works again, not a change of the site, so + * it is an ordinary one. */ export async function resumeScanning( sql: Sql, website: WebsiteRow, ): Promise { - await queueScan(sql, website); + await queueScan(sql, website, { full: false }); } /** @@ -1388,15 +1400,24 @@ export async function resumeScanning( * already scanning, or being deleted, is left as it is — the corpus claim * would turn a second scan away anyway — and the answer says nothing was * queued. + * + * On a person's word the scan is a `full` one and requests every page: + * "Scan now" is how someone says the site changed, and under the scheduled + * scan's rule it would read nothing of a site whose sitemap dates are + * behind its pages. A scan the platform starts for its own reasons — an + * embedding model was saved — passes `full: false`: nothing says the site + * changed, and what it is for (embedding stored text, retrying a failed + * scan) needs no page requested again. */ export async function scanWebsiteNow( sql: Sql, website: WebsiteRow, + options: { full: boolean } = { full: true }, ): Promise<{ queued: boolean }> { if (website.status === 'scanning' || website.status === 'deleting') { return { queued: false }; } - await queueScan(sql, website); + await queueScan(sql, website, options); return { queued: true }; } @@ -1664,6 +1685,7 @@ export async function runWebsitesScan( continuation?: number; scanStartedAt?: string; takeover?: string; + full?: boolean; }, job?: { signal?: AbortSignal; jobId?: string }, ): Promise { diff --git a/services/platform/backend/integration-check.ts b/services/platform/backend/integration-check.ts index 47077c91dd..22a7e8a9a6 100644 --- a/services/platform/backend/integration-check.ts +++ b/services/platform/backend/integration-check.ts @@ -33831,6 +33831,35 @@ async function checkWebsitesCrawl( // page, no robots.txt and no sitemap, so the scan that restores the // registration finishes on the homepage. const HEAL_DOMAIN = 'itest-heal.example'; + // A site whose sitemap dates its pages — the lastmod lanes (2026-10-06). + // Its own fixture, with a count of the requests each path answered, so a + // lane can tell a page a scan left alone from one it read again. Six + // sitemap URLs: under the link-walk threshold, so discovery also reads + // the homepage for links on every scan — the lanes judge the other pages. + const LASTMOD_DOMAIN = 'itest-lastmod.example'; + const OLD_LASTMOD = '2024-03-01'; + const lastmodDates = new Map([ + ['/', OLD_LASTMOD], + ['/old.txt', OLD_LASTMOD], + ['/zoned.txt', '2024-03-01T10:00:00+01:00'], + ['/undated.txt', null], + ['/stale.txt', OLD_LASTMOD], + ['/moved.txt', OLD_LASTMOD], + ]); + const lastmodBodies = new Map( + [...lastmodDates.keys()].map((route) => [ + route, + `Lastmod fixture page ${route} v1. Enough words about the sitemap dates of this page to survive the chunking thresholds of the pipeline.`, + ]), + ); + const lastmodHits = new Map(); + const lastmodSitemap = (): string => + `${[...lastmodDates] + .map( + ([route, date]) => + `https://${LASTMOD_DOMAIN}${route}${date === null ? '' : `${date}`}`, + ) + .join('')}`; const FAKE_HOSTS = new Set([ DOMAIN, `www.${DOMAIN}`, @@ -33901,6 +33930,26 @@ async function checkWebsitesCrawl( }, }); } + if ( + url.hostname === LASTMOD_DOMAIN || + url.hostname === `www.${LASTMOD_DOMAIN}` + ) { + lastmodHits.set(url.pathname, (lastmodHits.get(url.pathname) ?? 0) + 1); + const body = + url.pathname === '/robots.txt' + ? `User-agent: *\nSitemap: https://${LASTMOD_DOMAIN}/sitemap.xml\n` + : url.pathname === '/sitemap.xml' + ? lastmodSitemap() + : lastmodBodies.get(url.pathname); + if (body === undefined) return new Response('gone', { status: 404 }); + return new Response(body, { + headers: { + 'content-type': + url.pathname === '/sitemap.xml' ? 'application/xml' : 'text/plain', + 'content-length': String(body.length), + }, + }); + } if (url.hostname === HEAL_DOMAIN || url.hostname === `www.${HEAL_DOMAIN}`) { if (url.pathname !== '/') return new Response('gone', { status: 404 }); const body = @@ -35189,6 +35238,151 @@ async function checkWebsitesCrawl( await v1(`/websites/${robotsId}`, { method: 'DELETE' }); } + // 4h. A rescan leaves alone the pages the site's sitemap says have not + // changed (2026-10-06). The crawler read `` out of a sitemap + // and nothing else, so every scan requested every page again — a + // site that dated all its pages answered thousands of requests a + // scan for pages unchanged in months. A page is now requested when + // its entry carries no date, when the date moved past the last + // read, when the sitemap no longer states it, when the last read is + // older than a week, and on a scan a person started. + const lastmodCreated = z.looseObject({ id: z.string() }).safeParse( + await ( + await v1('/websites', { + body: { domain: LASTMOD_DOMAIN, scanInterval: '6h' }, + }) + ).json(), + ); + const lastmodId = lastmodCreated.success ? lastmodCreated.data.id : ''; + await drainCrawlJobs(); + const lastmodUrl = (route: string) => `https://${LASTMOD_DOMAIN}${route}`; + const lastmodPages = [ + '/old.txt', + '/zoned.txt', + '/undated.txt', + '/stale.txt', + '/moved.txt', + ]; + /** How often each page was requested since `before`, as `route:n`. */ + const requestedSince = (before: Map): string => + lastmodPages + .map( + (route) => + `${route}:${(lastmodHits.get(route) ?? 0) - (before.get(route) ?? 0)}`, + ) + .join(' '); + const lastmodRescan = async (): Promise => { + const before = new Map(lastmodHits); + await websites.runWebsitesScan(sql, { + domain: LASTMOD_DOMAIN, + orgSlug, + organizationId: orgId, + }); + await drainCrawlJobs(); + return requestedSince(before); + }; + /** The date each page's row carries, as `route:instant`. */ + const lastmodStored = async (): Promise => + ( + await pool<{ url: string; lastmod: string | null }[]>` + SELECT url, + to_char(sitemap_lastmod AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI"Z"') AS lastmod + FROM public_web.website_urls + WHERE domain = ${LASTMOD_DOMAIN} AND url <> ${lastmodUrl('/')} + ORDER BY url + ` + ) + .map( + (row) => + `${row.url.replace(`https://${LASTMOD_DOMAIN}`, '')}:${row.lastmod ?? 'none'}`, + ) + .join(' '); + const lastmodFirst = await pool<{ active: string }[]>` + SELECT count(*) FILTER (WHERE status = 'active')::text AS active + FROM public_web.website_urls WHERE domain = ${LASTMOD_DOMAIN} + `; + + // The first rescan: every dated page was read after its date, so none + // is requested; the page without a date is. + const lastmodSecond = await lastmodRescan(); + const lastmodDatesStored = await lastmodStored(); + record( + 'websites lastmod: a rescan requests no page its sitemap dates as unchanged, and the page without a date', + lastmodCreated.success && + lastmodFirst[0]?.active === '6' && + lastmodSecond === + '/old.txt:0 /zoned.txt:0 /undated.txt:1 /stale.txt:0 /moved.txt:0' && + // A date alone covers its whole day in any time zone; a time with a + // zone is that instant. + lastmodDatesStored === + '/moved.txt:2024-03-02T12:00Z /old.txt:2024-03-02T12:00Z /stale.txt:2024-03-02T12:00Z /undated.txt:none /zoned.txt:2024-03-01T09:00Z', + `created=${lastmodCreated.success} firstScanActive=${lastmodFirst[0]?.active ?? '?'}/6, rescan requested [${lastmodSecond}] (want old 0, zoned 0, undated 1, stale 0, moved 0), stored [${lastmodDatesStored}]`, + ); + + // A date that moved, a date the sitemap dropped, and a read older than + // a week: each is requested again, and the changed page is re-indexed. + lastmodDates.set('/moved.txt', new Date().toISOString()); + lastmodBodies.set( + '/moved.txt', + 'Lastmod fixture page /moved.txt v2. Rewritten words about the sitemap dates of this page so its content hash flips and its chunks are cut again.', + ); + lastmodDates.set('/zoned.txt', null); + await pool` + UPDATE public_web.website_urls + SET last_crawled_at = now() - interval '8 days' + WHERE domain = ${LASTMOD_DOMAIN} AND url = ${lastmodUrl('/stale.txt')} + `; + const lastmodThird = await lastmodRescan(); + const movedChunks = await pool<{ content: string }[]>` + SELECT chunk_content AS content FROM public_web.chunks + WHERE domain = ${LASTMOD_DOMAIN} AND url = ${lastmodUrl('/moved.txt')} + `; + const staleRead = await pool<{ recent: boolean }[]>` + SELECT last_crawled_at > now() - interval '1 hour' AS recent + FROM public_web.website_urls + WHERE domain = ${LASTMOD_DOMAIN} AND url = ${lastmodUrl('/stale.txt')} + `; + record( + 'websites lastmod: a moved date, a date the sitemap dropped and a read older than a week are requested again', + lastmodThird === + '/old.txt:0 /zoned.txt:1 /undated.txt:1 /stale.txt:1 /moved.txt:1' && + movedChunks.length >= 1 && + movedChunks.every((chunk) => chunk.content.includes('v2')) && + (staleRead[0]?.recent ?? false), + `rescan requested [${lastmodThird}] (want old 0, zoned 1, undated 1, stale 1, moved 1), movedChunks=${movedChunks.length} allV2=${movedChunks.every((chunk) => chunk.content.includes('v2'))}, staleReadNow=${staleRead[0]?.recent ?? '?'}`, + ); + + // "Scan now" is a person saying the site changed: every page is + // requested, and the scan records no date. The scheduled scan after it + // honours the dates again — and requests the page whose date moved + // within the hour once more, as a cache in front of the site may still + // have served its earlier text. + const beforeScanNow = new Map(lastmodHits); + const scanNow = z + .looseObject({ queued: z.boolean() }) + .safeParse(await (await post(`/${lastmodId}/scan`, {})).json()); + await drainCrawlJobs(); + const lastmodFull = requestedSince(beforeScanNow); + const lastmodAfterFull = await lastmodStored(); + const lastmodFifth = await lastmodRescan(); + const lastmodAfterFifth = await lastmodStored(); + record( + 'websites lastmod: Scan now requests every page, and the next scheduled scan honours the dates again', + scanNow.success && + scanNow.data.queued && + lastmodFull === + '/old.txt:1 /zoned.txt:1 /undated.txt:1 /stale.txt:1 /moved.txt:1' && + !lastmodAfterFull.includes('Z') && + lastmodFifth === + '/old.txt:0 /zoned.txt:1 /undated.txt:1 /stale.txt:0 /moved.txt:1' && + lastmodAfterFifth.includes('/old.txt:2024-03-02T12:00Z'), + `scanNow queued=${scanNow.success ? scanNow.data.queued : 'BAD SHAPE'} requested [${lastmodFull}] (want every page 1) stored [${lastmodAfterFull}] (want none), next scan requested [${lastmodFifth}] (want old 0, zoned 1, undated 1, stale 0, moved 1) stored [${lastmodAfterFifth}]`, + ); + if (lastmodId !== '') { + await v1(`/websites/${lastmodId}`, { method: 'DELETE' }); + } + // 4f. Round g, g4-5: a site whose every page failed is not "a successful // scan". The homepage is always admitted, its 503 is one attempted // page and nothing stored, so the scan ends on the documented diff --git a/services/platform/backend/jobs/task-list.ts b/services/platform/backend/jobs/task-list.ts index 7d645ee3cc..901f65e687 100644 --- a/services/platform/backend/jobs/task-list.ts +++ b/services/platform/backend/jobs/task-list.ts @@ -1183,6 +1183,7 @@ export function createTaskList(deps: TaskDeps): BackendTaskList { continuation: z.number().int().min(0).optional(), scanStartedAt: z.string().optional(), takeover: z.string().min(1).optional(), + full: z.boolean().optional(), }) .parse(payload); await runWebsitesScan(deps.sql, input, context); diff --git a/services/platform/backend/jobs/tasks.ts b/services/platform/backend/jobs/tasks.ts index a6a97315ac..53b3f70676 100644 --- a/services/platform/backend/jobs/tasks.ts +++ b/services/platform/backend/jobs/tasks.ts @@ -331,7 +331,10 @@ export interface TaskPayloads { /** One continuation link of a domain scan — the reused engine body * self-chains through this queue; the corpus-side claim is the fence. * `takeover` is set by the scheduler alone, on the first link of a scan - * it resumes: the heartbeat of the claim whose scan stopped. */ + * it resumes: the heartbeat of the claim whose scan stopped. `full` is + * set on the first link of a scan a person started: it requests every + * page, where a scheduled scan leaves alone the pages the site's sitemap + * says have not changed. */ 'websites.scan': { domain: string; orgSlug: string; @@ -339,6 +342,7 @@ export interface TaskPayloads { continuation?: number; scanStartedAt?: string; takeover?: string; + full?: boolean; }; /** Register a website (or URL list) in the corpus + kick its first scan * (the 0.4 `registerAndSync`, fire-and-forget behind the create). */ diff --git a/services/platform/lib/knowledge/crawl-parse.test.ts b/services/platform/lib/knowledge/crawl-parse.test.ts index 97c9d3a871..19b1c9f2ce 100644 --- a/services/platform/lib/knowledge/crawl-parse.test.ts +++ b/services/platform/lib/knowledge/crawl-parse.test.ts @@ -26,9 +26,11 @@ import { normalizeListedUrl, paragraphsForHashing, parseRobots, + parseSitemapEntries, parseSitemapLocs, robotsHeaderForbidsIndexing, siteHosts, + sitemapLastmodInstant, stripBoilerplate, robotsSitemapsFromStored, } from './crawl-parse'; @@ -253,6 +255,134 @@ describe('sitemap parsing', () => { }); }); +/** + * What a sitemap says about a page's last change decides whether a rescan + * requests the page at all, so the value is read toward its LATEST meaning: + * a scan that read the page before that instant cannot know it has the + * current text. The crawler read `` alone and requested every page on + * every scan, however long its sitemap said it had not changed. + */ +describe('sitemapLastmodInstant', () => { + const instant = (value: string): string | null => + sitemapLastmodInstant(value)?.toISOString() ?? null; + + it('reads a time with a zone as that instant', () => { + expect(instant('2026-10-06T14:30:00Z')).toBe('2026-10-06T14:30:00.000Z'); + expect(instant('2026-10-06T14:30:00+02:00')).toBe( + '2026-10-06T12:30:00.000Z', + ); + expect(instant('2026-10-06T14:30:00-0700')).toBe( + '2026-10-06T21:30:00.000Z', + ); + expect(instant('2026-10-06T14:30+02:00')).toBe('2026-10-06T12:30:00.000Z'); + expect(instant(' 2026-10-06T14:30:00.123456+00:00 ')).toBe( + '2026-10-06T14:30:00.000Z', + ); + }); + + // A date alone keeps its value through every edit of that day: the page + // is unchanged only once the day is over everywhere. + it('reads a date alone as the end of that day in the last time zone', () => { + expect(instant('2026-10-06')).toBe('2026-10-07T12:00:00.000Z'); + expect(instant('2024-02-29')).toBe('2024-03-01T12:00:00.000Z'); + expect(instant('2026-12-31')).toBe('2027-01-01T12:00:00.000Z'); + }); + + it('reads a time without a zone in the zone furthest behind UTC', () => { + expect(instant('2026-10-06T14:30:00')).toBe('2026-10-07T02:30:00.000Z'); + expect(instant('2026-10-06 14:30:00')).toBe('2026-10-07T02:30:00.000Z'); + }); + + it.each([ + ['a year alone', '2026'], + ['a month alone', '2026-10'], + ['a month that does not exist', '2026-13-01'], + ['a day that does not exist', '2026-02-30'], + ['an hour that does not exist', '2026-10-06T25:00:00Z'], + ['a zone that does not exist', '2026-10-06T14:30:00+19:00'], + ['another date format', '06.10.2026'], + ['an HTTP date', 'Tue, 06 Oct 2026 14:30:00 GMT'], + ['words', 'yesterday'], + ['nothing', ''], + ])('reads nothing out of %s', (_case, value) => { + expect(sitemapLastmodInstant(value)).toBeNull(); + }); +}); + +describe('parseSitemapEntries', () => { + const read = (xml: string) => + parseSitemapEntries(xml).map((entry) => [ + entry.loc, + entry.lastmod?.toISOString() ?? null, + ]); + + it("gives each page its own entry's date", () => { + const xml = ` + + + https://a.example + 2026-07-27 + monthly + + + + 2026-10-06T08:00:00+02:00 + https://a.example/en + + https://a.example/undated + + `; + expect(read(xml)).toEqual([ + ['https://a.example', '2026-07-28T12:00:00.000Z'], + ['https://a.example/en', '2026-10-06T06:00:00.000Z'], + ['https://a.example/undated', null], + ['https://a.example/cdata', '2026-01-03T12:00:00.000Z'], + ]); + }); + + it('leaves a page without a date when its entry carries one it cannot read', () => { + expect( + read( + 'https://a.example/xlast week', + ), + ).toEqual([['https://a.example/x', null]]); + }); + + // An image extension lists the image under its own tag, with its own + // address: neither is the page's. + it('reads the page address, not an image listed in its entry', () => { + expect( + read( + 'https://a.example/xhttps://a.example/x.png2026-01-02', + ), + ).toEqual([['https://a.example/x', '2026-01-03T12:00:00.000Z']]); + }); + + it.each([ + [ + 'a sitemap index', + 'https://a.example/pages.xml2026-01-02', + ], + [ + 'addresses outside any entry', + 'https://a.example/pages.xml2026-01-02', + ], + ])('lists every address of %s, without a date', (_case, xml) => { + expect(read(xml)).toEqual([['https://a.example/pages.xml', null]]); + }); + + it('lists the same addresses, in the same order, as the plain reader', () => { + const xml = `https://a.example/before + https://a.example/12026-01-02 + https://a.example/between + https://a.example/2?a=1&b=2 + https://a.example/after`; + expect(parseSitemapEntries(xml).map((entry) => entry.loc)).toEqual( + parseSitemapLocs(xml), + ); + }); +}); + describe('extractLinks', () => { it('reads single- and double-quoted hrefs', () => { expect( @@ -921,6 +1051,32 @@ describe('scanning markup that never closes its tags', () => { 'CDATA openers inside one sitemap entry', () => parseSitemapLocs(`${'`), ], + [ + 'sitemap blocks with no end', + () => parseSitemapEntries(''.repeat(RUN)), + ], + [ + 'sitemap blocks whose only end is the last one', + () => parseSitemapEntries(`${'`), + ], + [ + 'sitemap dates with no end inside one block', + () => parseSitemapEntries(`${''.repeat(RUN)}`), + ], + [ + 'sitemap dates whose only end is the last one', + () => + parseSitemapEntries(`${'`), + ], + [ + 'a sitemap of many small blocks', + () => + parseSitemapEntries( + 'https://a.example/x2026-01-02'.repeat( + RUN, + ), + ), + ], ['anchors with no target', () => extractLinks(`${'`)], [ 'anchors with an unquoted target', diff --git a/services/platform/lib/knowledge/crawl-parse.ts b/services/platform/lib/knowledge/crawl-parse.ts index 9632bec201..78d4836f9b 100644 --- a/services/platform/lib/knowledge/crawl-parse.ts +++ b/services/platform/lib/knowledge/crawl-parse.ts @@ -265,6 +265,147 @@ export function isSitemapIndex(xml: string): boolean { return /]/i.test(xml); } +/** One page a sitemap lists: its address and, when its entry states one + * that can be read, the latest instant its `` can mean + * ({@link sitemapLastmodInstant}). */ +export interface SitemapEntry { + readonly loc: string; + readonly lastmod: Date | null; +} + +const HOUR_MS = 60 * 60 * 1000; +/** How far behind UTC a clock can run: the last time zone on Earth ends its + * day twelve hours after UTC does. */ +const FURTHEST_WEST_MS = 12 * HOUR_MS; + +const LASTMOD_DATE = /^(\d{4})-(\d{2})-(\d{2})$/; +const LASTMOD_DATETIME = + /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2})(?:[.,]\d{1,9})?)?(Z|[+-]\d{2}(?::?\d{2})?)?$/i; +const LASTMOD_ZONE = /^([+-])(\d{2}):?(\d{2})?$/; + +/** The UTC epoch of these calendar fields, or null when they name no real + * moment — a thirteenth month, the 30th of February, 25 o'clock. */ +function utcEpoch( + year: number, + month: number, + day: number, + hour: number, + minute: number, + second: number, +): number | null { + const epoch = Date.UTC(year, month - 1, day, hour, minute, second); + const read = new Date(epoch); + return read.getUTCFullYear() === year && + read.getUTCMonth() === month - 1 && + read.getUTCDate() === day && + read.getUTCHours() === hour && + read.getUTCMinutes() === minute && + read.getUTCSeconds() === second + ? epoch + : null; +} + +/** + * The latest instant a sitemap `` can mean, or null for a value + * that says too little to act on. A scan leaves a page alone only when it + * read the page AFTER this instant, so every doubt is resolved toward the + * later reading and an unreadable value toward "request the page": + * + * - a date and a time with a zone (`2026-10-06T14:30:00+02:00`, `…Z`) is + * that instant; + * - a date and a time without a zone is read in the zone furthest behind + * UTC — the site's own clock is not known; + * - a date alone (`2026-10-06`, what most sitemaps carry) covers its whole + * day: the page can change at any hour of it and keep the same value, so + * it means the end of that day in the zone furthest behind UTC; + * - a month or a year alone, and anything that is not a W3C date, is null. + * + * Read with patterns of its own rather than `Date.parse`, which takes a + * date alone as UTC midnight and a time without a zone as the server's + * local time: the first would skip a page edited later that day, the second + * would make the answer depend on where the backend runs. + */ +export function sitemapLastmodInstant(value: string): Date | null { + const text = value.trim(); + const date = LASTMOD_DATE.exec(text); + if (date) { + const start = utcEpoch( + Number(date[1]), + Number(date[2]), + Number(date[3]), + 0, + 0, + 0, + ); + return start === null + ? null + : new Date(start + 24 * HOUR_MS + FURTHEST_WEST_MS); + } + const stamp = LASTMOD_DATETIME.exec(text); + if (!stamp) return null; + const local = utcEpoch( + Number(stamp[1]), + Number(stamp[2]), + Number(stamp[3]), + Number(stamp[4]), + Number(stamp[5]), + Number(stamp[6] ?? '0'), + ); + if (local === null) return null; + const zone = stamp[7]; + if (zone === undefined) return new Date(local + FURTHEST_WEST_MS); + if (zone.toUpperCase() === 'Z') return new Date(local); + const offset = LASTMOD_ZONE.exec(zone); + if (!offset) return null; + const hours = Number(offset[2]); + const minutes = Number(offset[3] ?? '0'); + if (hours > 14 || minutes > 59) return null; + const ahead = (hours * 60 + minutes) * 60 * 1000; + return new Date(offset[1] === '-' ? local + ahead : local - ahead); +} + +/** What one `` block says about its page's last change. */ +function entryLastmod(block: string): Date | null { + const match = /]*>([\s\S]*?)<\/lastmod>/i.exec( + upToLast(block, /<\/lastmod>/gi), + ); + if (!match) return null; + const raw = (match[1] ?? '').trim(); + const cdata = /^$/.exec(raw); + return sitemapLastmodInstant(cdata ? (cdata[1] ?? '') : raw); +} + +/** + * The pages a sitemap lists, each with what its entry says about the page's + * last change. The addresses are exactly the ones {@link parseSitemapLocs} + * reads, in document order: one inside a `` block takes that block's + * ``, and one outside any block — a document that is not quite a + * sitemap — has none, so its page is requested as before. + * + * The blocks are matched up to the last `` only, and each block's + * date up to its last `` (`markup-scan.ts`). + */ +export function parseSitemapEntries(xml: string): SitemapEntry[] { + const entries: SitemapEntry[] = []; + const undated = (text: string): void => { + for (const loc of parseSitemapLocs(text)) { + entries.push({ loc, lastmod: null }); + } + }; + let cursor = 0; + for (const block of upToLast(xml, /<\/url>/gi).matchAll( + /])[^<>]*>([\s\S]*?)<\/url>/gi, + )) { + undated(xml.slice(cursor, block.index)); + const body = block[1] ?? ''; + const lastmod = entryLastmod(body); + for (const loc of parseSitemapLocs(body)) entries.push({ loc, lastmod }); + cursor = block.index + block[0].length; + } + undated(xml.slice(cursor)); + return entries; +} + /** `href` targets of a page's anchors, entity-decoded — attribute values * are entity-encoded by spec, so `?a=1&b=2` in the markup means * `?a=1&b=2` to the browser and must mean the same to the crawler. */ diff --git a/services/platform/tests/manual/reference/automation.md b/services/platform/tests/manual/reference/automation.md index 30d2789724..d897618a1a 100644 --- a/services/platform/tests/manual/reference/automation.md +++ b/services/platform/tests/manual/reference/automation.md @@ -387,6 +387,7 @@ Legend: ✅ fully automated · 🔶 partially automated · ⛔ manual-only (no s | [knowledge](../suites/knowledge.md) | Website sources take the knowledge write role on the server too: a read-only Member is refused `RBAC_FORBIDDEN` (403) on add, edit, delete, **Resume scanning** and **Scan now** (`POST /api/app/websites`, `PATCH` and `DELETE …/{websiteId}`, `…/{websiteId}/resume`, `…/{websiteId}/scan`) before the service is reached, where every member of the organization could call them before; Editor, Developer, Admin and Owner pass, and the list, a row, the status sync and the search fact stay open to every member | ✅ route | `backend/domains/websites/routes.test.ts` | | [knowledge](../suites/knowledge.md) | A page is stamped as visited only once it is stored and indexed, so a link cut off in between leaves it due for the resumed scan; unchanged text whose chunks were cut from other text is indexed again instead of serving the old text; a row the running scan retired (an alias it followed, a page that answered 404) is not revived by the links of the next page, while discovery still revives what an earlier scan retired | ✅ unit + real Postgres | `backend/core/knowledge/crawl_action.vectors.test.ts`, `backend/core/knowledge/crawl.test.ts`, `backend/core/knowledge/crawl_action.redirects.test.ts`, `backend/integration-check.ts` (`checkWebsitesCrawl`) | | [knowledge](../suites/knowledge.md) | A page that redirects onto another host or path of its site (`www.` onto the apex, a missing trailing slash, a removed page sent to the homepage) is retired as an alias and its target tracked instead, unless robots.txt disallows the target; a listed URL and a redirect that only rewrites the query keep their row | ✅ unit | `backend/core/knowledge/crawl_action.redirects.test.ts` | +| [knowledge](../suites/knowledge.md) | A scheduled rescan leaves alone the pages a site's sitemap dates as unchanged: a page whose stored text was read after its entry's `` is not requested — a date alone covers its whole day in any time zone, a time without a zone the same half day, and a read within the hour after the date does not count; a page without a readable date, one the sitemap stopped dating, one whose date moved, one whose last read is older than seven days, one whose last attempt failed and one whose stored text is not indexed yet are requested; each discovery replaces the dates the one before recorded; **Scan now** requests every page and records no date, while **Resume scanning** and the scans that follow a saved embedding model stay ordinary ones | ✅ unit + real Postgres | `lib/knowledge/crawl-parse.test.ts`, `backend/core/knowledge/crawl.test.ts`, `backend/core/knowledge/crawl_action.lastmod.test.ts`, `backend/domains/websites/service.scan-now.test.ts`, `backend/domains/websites/service.embedding-follow.test.ts`, `backend/integration-check.ts` (`checkWebsitesCrawl`: the `websites lastmod` lanes) | | [knowledge](../suites/knowledge.md) / [settings](../suites/settings.md) | The crawler's browser session has room to run: the `default` session profile allows 512 tasks (a headless Chromium needs 160 and more; under 128 every navigation after the first two or three died `net::ERR_ABORTED`), and a browser that dies mid-batch ends the batch with the untouched pages handed back, not charged | ✅ unit | `services/sandbox/src/session/docker-session-args.test.ts`, `backend/core/node_only/sandbox/render_fetch.test.ts` | | [settings](../suites/settings.md) | `SET-F33` (MCP endpoint) — the wire contract behind the page: envelope validation, version negotiation, batches, tool-argument validation against the advertised schema (document tools included), `params` typing, the unissued `tools/list` cursor, 405 + `Allow` on other verbs, `isError` on failure-shaped results, `get_docs` serving a reference and never the builder prompt | ✅ automated | `backend/core/automations_builder/mcp_http.test.ts`, `lib/engine/selftest/dispatch.test.ts` | | [projects](../suites/projects.md) | REST chat lifecycle: door-side model resolution (`CHAT_MODEL_UNKNOWN` / `CHAT_MODEL_AMBIGUOUS`), model facts and the default marker on `GET /models`, message `status` (a stop settles as `cancelled`) and `usage` (`costEstimateCents` included), the 202’s pre-minted `messageId` and the `queued` poll state until the worker opens the turn, streaming progress on the poll (`since` and `reasoningSince` slice the text and the reasoning independently, each with its offset/length pair and the reset-to-0 rule), `reasoningEffort` / `maxOutputTokens` on a send (a cap above the model’s ceiling refused; the cap a per-turn budget the tool rounds share — a later round gets what the earlier ones left, a round with nothing left is not run, a cut round marks the turn `finishReason: length` and its calls settle `invalid_args` unrun), `usage.costEstimateCents` rounded to a millionth of a cent, a named `locale` asking for the reply language on the system prompt and on the message (`localeFixed` through the job into the runtime directive and the wire-only notice; the app lane untouched), archive with `archivedAt` echoed from the domain’s own stamp / rename / delete / cancel routes, the stop of a send still queued (202 `cancelling` → a `cancelled` reply without a model call) and the 404’s `data.lastMessageId` / `data.lastStatus`, hidden regenerate-siblings invisible by id, blank-prompt and locale refusals | ✅ automated | `backend/rest/v1-threads.test.ts`, `backend/rest/v1-threads.contract.test.ts`, `backend/rest/v1-threads.scope.test.ts`, `backend/domains/chat/rest-turn.test.ts`, `backend/domains/chat/store.test.ts`, `lib/chat/turn.test.ts`, `lib/chat/context.test.ts`, `lib/chat/effort.test.ts`, `lib/chat/tools.test.ts`, `lib/shared/utils/language-name.test.ts`, `scripts/openapi/spec.test.ts`, the `REST resources` lane in `backend:integration` (the queued stop and the 404’s data on the real schema) | From 56c9a8cb23e4c0fac614709c0d57a17c18d93a32 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Tue, 6 Oct 2026 22:12:19 +0800 Subject: [PATCH 2/3] fix(platform): render a crawled page again only when it changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crawler's one way to learn whether a page had changed was to do everything again: fetch it, render it in a browser with every image, script and stylesheet the render loads, and compare content hashes afterwards. A 700-page site scanned every six hours answered some 8,000 requests and two gigabytes a scan for pages unchanged in months. A scan now asks for each page once and opens the browser only for a page that changed. The request carries the validators of the page's last settled visit, and a 304 leaves the page as it is. A server that builds its pages on request has no modification time to give — no ETag, no Last-Modified, a new token in every response — so the text read out of the plain HTML is compared too. What a page is checked by is kept only when its plain HTML carried what the browser showed (nine tenths of its words); a page built by its JavaScript is rendered on every scan, as before. "Scan now" asks the same way as a scheduled scan. This replaces the first approach on this branch, which skipped pages by their sitemap `` and read every page again weekly: a sitemap dates a listing page by its own last edit, not by what it lists, and a clock-based re-read is the load the change is there to remove. --- .agents/repo.md | 13 +- docs/de/platform/knowledge/crawling.md | 8 +- docs/en/platform/knowledge/crawling.md | 8 +- docs/fr/platform/knowledge/crawling.md | 8 +- ...0000000000013_website_url_change_check.sql | 53 ++ ...0000000013_website_url_sitemap_lastmod.sql | 34 -- .../backend/core/knowledge/crawl.test.ts | 78 --- .../platform/backend/core/knowledge/crawl.ts | 47 -- .../crawl_action.change-check.test.ts | 575 ++++++++++++++++++ .../knowledge/crawl_action.lastmod.test.ts | 255 -------- .../knowledge/crawl_action.redirects.test.ts | 6 +- .../backend/core/knowledge/crawl_action.ts | 504 ++++++++++----- .../core/node_only/sandbox/render_fetch.ts | 2 +- .../backend/domains/knowledge/spec.md | 35 +- .../websites/service.embedding-follow.test.ts | 5 - .../domains/websites/service.scan-now.test.ts | 58 +- .../backend/domains/websites/service.ts | 32 +- .../platform/backend/domains/websites/spec.md | 2 +- .../platform/backend/integration-check.ts | 387 +++++++----- services/platform/backend/jobs/task-list.ts | 1 - services/platform/backend/jobs/tasks.ts | 6 +- .../lib/knowledge/crawl-parse.test.ts | 207 ++----- .../platform/lib/knowledge/crawl-parse.ts | 198 ++---- .../tests/manual/reference/automation.md | 2 +- 24 files changed, 1385 insertions(+), 1139 deletions(-) create mode 100644 services/db/migrations/knowledge-db/public_web/00000000000013_website_url_change_check.sql delete mode 100644 services/db/migrations/knowledge-db/public_web/00000000000013_website_url_sitemap_lastmod.sql create mode 100644 services/platform/backend/core/knowledge/crawl_action.change-check.test.ts delete mode 100644 services/platform/backend/core/knowledge/crawl_action.lastmod.test.ts diff --git a/.agents/repo.md b/.agents/repo.md index b3e4a5649a..2b178110f9 100644 --- a/.agents/repo.md +++ b/.agents/repo.md @@ -615,11 +615,14 @@ xlsx,odt}.ts`) still reach the catch-all as `failed` + `indexer_error` and are r refusing every delivery reads as never called (2026-09, round i). The reference says so. Paying it down means a `delivery_refused` skip reason stamped from the webhook door beside `start_refused`. -- **A page is fetched three to four times a scan** — the plain probe, the render pass and, for - the homepage, a create-time metadata probe. The robots parser now honours group selection by - product token (`User-agent: TaleBot` > `*`), `Allow` with longest-match precedence, `$` - end-anchors and `Crawl-delay` (`lib/knowledge/crawl-parse.ts`, 2026-09 round J); paying down the - remaining fetch count means one fetch per page per scan. +- **A changed page is fetched twice a scan** — the plain probe and the render pass — and the + homepage once more at create, for its metadata. A page that did not change is asked for once + and not rendered (2026-10: the probe's validators and the text of its plain HTML decide, + `core/knowledge/crawl_action.ts`); a page built by its JavaScript is still rendered on every + scan. The robots parser honours group selection by product token (`User-agent: TaleBot` > + `*`), `Allow` with longest-match precedence, `$` end-anchors and `Crawl-delay` + (`lib/knowledge/crawl-parse.ts`, 2026-09 round J). Paying down the rest means one fetch per + changed page: a server-rendered page read from the probe's own bytes, without the browser. - **A cancelled run answers `trace: null` and `effects: null`** where a failed run answers both, so what a cancel did not undo is readable only through `checkpoints` (2026-09, round i). Paying it down means keeping the partial trace the way the failed path does. diff --git a/docs/de/platform/knowledge/crawling.md b/docs/de/platform/knowledge/crawling.md index c7faabb1d1..f2860aace2 100644 --- a/docs/de/platform/knowledge/crawling.md +++ b/docs/de/platform/knowledge/crawling.md @@ -42,7 +42,7 @@ Bei einer ganzen Website nutzt der Crawler Startseite und veröffentlichte Sitem Scans arbeiten schrittweise: Unveränderte Inhalte werden übersprungen, geänderte erneut indexiert, neue Seiten hinzugefügt und entfernte aus dem Index genommen — ebenso Seiten, die die `robots.txt` inzwischen verbietet. Eine Seite, die auf eine andere Adresse derselben Website weiterleitet, wird nur einmal indexiert: unter der Adresse, auf der sie landet. Die Seitenzähler der Zeile folgen dem Scan, während Seiten landen, nach dem Entdecken und nach jedem gespeicherten Batch, die Tabelle bewegt sich also, während ein Scan läuft. Eine URL-Liste aktualisiert ihre feste Auswahl nach demselben Zeitplan. Nach erfolgreicher Indexierung ist keine gesonderte Veröffentlichung nötig. -Ein Scan ruft nicht jedes Mal jede Seite ab. Trägt der Sitemap-Eintrag einer Seite ein ``-Datum und hat der Crawler die Seite nach diesem Datum gelesen, lässt er sie in Ruhe, bis sich das Datum ändert. Ein Datum ohne Uhrzeit gilt für den ganzen Tag: Eine heute geänderte Seite wird deshalb bis zum Folgetag bei jedem Scan erneut gelesen. Auch eine Seite mit Datum wird wieder gelesen, sobald ihr letzter Abruf mehr als sieben Tage zurückliegt, denn eine Sitemap kann ihr Datum behalten, während sich die Seite ändert, etwa bei einer Übersichtsseite mit den neuesten Artikeln. Eine Seite, deren Eintrag kein lesbares Datum hat oder die in keiner Sitemap steht, wird bei jedem Scan abgerufen. Eine Website, die in ihrer Sitemap zutreffende ``-Daten veröffentlicht, erhält bei einem wiederholten Scan daher nur wenige Anfragen. +Ein Scan fragt jede Seite einmal ab. Dabei schickt er mit, was der Server der Website beim letzten Besuch über die Seite angegeben hat; antwortet der Server mit `304 Not Modified`, wird die Seite nicht erneut übertragen. Viele Server erzeugen ihre Seiten bei jeder Anfrage neu und nennen keinen Änderungszeitpunkt; dann vergleicht der Crawler den Text der gelieferten Seite. In beiden Fällen wird eine unveränderte Seite nicht im Browser geöffnet, ihre Bilder, Skripte und Stylesheets werden also nicht geladen. Die Ausnahme ist eine Seite, deren Inhalt erst ihr eigenes JavaScript aufbaut: Nur ein Browser kann sie lesen, deshalb wird sie bei jedem Scan gerendert. Der Crawler besucht die Seiten ohne Anmeldung. Eine URL macht private Inhalte nicht zugänglich. Bei jeder Anfrage stellt er sich als `TaleBot/ (+https://docs.tale.dev/platform/knowledge/crawling)` vor, sodass eine `robots.txt`-Gruppe ihn beim Namen nennen kann — `User-agent: TaleBot` —, um allein ihn zu erlauben, zu drosseln oder abzuweisen. @@ -64,7 +64,7 @@ Du kannst weder eine eigene Seitenobergrenze noch Pfadfilter festlegen oder eine ## Die indexierten Inhalte prüfen -Die Tabelle zeigt **Status**, die Seitenzahl unter **Indexiert**, **Gescannt** und **Intervall**. Öffne die Quellzeile, um die Seitenliste, Wort- und Chunk-Anzahl sowie den letzten Abruf zu prüfen. Eine Seite, die ein Scan wegen ihres Sitemap-Datums ausgelassen hat, behält die Zeit ihres letzten Abrufs. Klappe eine Seite auf, um die gespeicherten Textabschnitte zu lesen. Bei einem fehlgeschlagenen Abruf stehen dort Ursache und Anzahl aufeinanderfolgender Fehler. Eine Seite, die der Crawler bewusst auslässt — die Quelle wünscht keine Indexierung, der Inhaltstyp liefert keinen lesbaren Text, oder eine Weiterleitung führt von der Website weg — steht als **Übersprungen** mit ihrer Ursache in der Liste und zählt nicht als fehlgeschlagen. +Die Tabelle zeigt **Status**, die Seitenzahl unter **Indexiert**, **Gescannt** und **Intervall**. Öffne die Quellzeile, um die Seitenliste, Wort- und Chunk-Anzahl sowie den letzten Abruf zu prüfen. Klappe eine Seite auf, um die gespeicherten Textabschnitte zu lesen. Bei einem fehlgeschlagenen Abruf stehen dort Ursache und Anzahl aufeinanderfolgender Fehler. Eine Seite, die der Crawler bewusst auslässt — die Quelle wünscht keine Indexierung, der Inhaltstyp liefert keinen lesbaren Text, oder eine Weiterleitung führt von der Website weg — steht als **Übersprungen** mit ihrer Ursache in der Liste und zählt nicht als fehlgeschlagen. | Status | Bedeutung | | --- | --- | @@ -75,7 +75,7 @@ Die Tabelle zeigt **Status**, die Seitenzahl unter **Indexiert**, **Gescannt** u Ein Scan, der gerade lief, als Tale neu gestartet, aktualisiert oder ohne Vorwarnung beendet wurde, läuft innerhalb weniger Minuten von selbst weiter, und zwar mit den Seiten, die er noch nicht erreicht hatte. Seiten, die den Browser brauchen, können bis zu einer Viertelstunde später folgen. Die Quelle zeigt währenddessen **Wird gescannt**. -Um außerhalb des Intervalls zu scannen, öffne das Zeilenmenü der Quelle oder ihre Details und wähle **Jetzt scannen**. Dabei wird jede Seite erneut abgerufen, unabhängig von den Daten der Sitemap. Das hilft, wenn sich die Website geändert hat oder ein Scan fehlgeschlagen ist; ein fehlgeschlagener Scan wird sonst innerhalb von zwei Stunden von selbst wiederholt. Die Aktion steht zur Verfügung, solange die Quelle weder gescannt noch gelöscht wird. +Um außerhalb des Intervalls zu scannen, öffne das Zeilenmenü der Quelle oder ihre Details und wähle **Jetzt scannen**. Das hilft, wenn sich die Website geändert hat oder ein Scan fehlgeschlagen ist; ein fehlgeschlagener Scan wird sonst innerhalb von zwei Stunden von selbst wiederholt. Die Aktion steht zur Verfügung, solange die Quelle weder gescannt noch gelöscht wird. Die Seitenansicht bietet auch eine Suche im indexierten Inhalt. Suche nach einer auffälligen Formulierung der Seite, bevor du dich im Chat darauf verlässt. Stelle anschließend eine konkrete Frage und prüfe den Quellenbeleg. @@ -96,6 +96,6 @@ Prüfe zuerst Adresse, Quelltyp und letzte Scan-Zeit. Öffne danach die Quelle, | **Fehler** mit „Das Embedding-Modell konnte die Seiten nicht verarbeiten.“ | Der Embedding-Anbieter hat die Anfrage abgelehnt oder konnte sie nicht bedienen: abgelehnte Zugangsdaten, aufgebrauchtes Guthaben, ein Ratenlimit, das über alle Wiederholungen anhielt, oder ein Ausfall. Die Details der Quelle sagen, was davon zutrifft, mit der Antwort des Anbieters darunter. Ein Administrator repariert das Modell unter **Einstellungen > Datenresidenz** oder die Zugangsdaten unter **Einstellungen > KI-Anbieter**; beim Speichern werden die betroffenen Quellen erneut gescannt. Nach einem Ausfall beim Anbieter wählst du **Jetzt scannen**, sobald er wieder erreichbar ist. | | **Fehler** mit „Der Scan ist nicht gelaufen.“ | Der Browser des Crawlers konnte nicht starten, deshalb wurde keine Seite dargestellt; sein Bericht steht in den Details der Quelle unter **Technische Details**. An der Adresse liegt es nicht: Lass deinen Betreiber den Sandbox-Dienst prüfen und wähle danach **Jetzt scannen**. | -Ein späterer erfolgreicher Abruf entfernt den vorherigen Fehler. Nach einer fehlgeschlagenen Aktualisierung kann die früher indexierte Fassung weiterhin verfügbar sein: **Aktiv** und die Anzahl indexierter Seiten belegen nicht, dass jede Seite aktuell ist. Vergleiche gespeicherte Textabschnitte und Abrufdatum mit dem Original, bevor du dich auf eine kürzliche Änderung verlässt. Hat sich eine Seite geändert, ihr Sitemap-Datum aber nicht, holen die geplanten Scans das innerhalb von sieben Tagen nach; mit **Jetzt scannen** wird sie sofort gelesen. +Ein späterer erfolgreicher Abruf entfernt den vorherigen Fehler. Nach einer fehlgeschlagenen Aktualisierung kann die früher indexierte Fassung weiterhin verfügbar sein: **Aktiv** und die Anzahl indexierter Seiten belegen nicht, dass jede Seite aktuell ist. Vergleiche gespeicherte Textabschnitte und Abrufdatum mit dem Original, bevor du dich auf eine kürzliche Änderung verlässt. Zeigt die Quelle **Pausiert**, haben wiederholte Verbindungsfehler zur Wissensdatenbank die Scans angehalten. Lass einen Administrator die Verbindung unter **Einstellungen > Datenresidenz** korrigieren und wähle danach **Scans fortsetzen**. diff --git a/docs/en/platform/knowledge/crawling.md b/docs/en/platform/knowledge/crawling.md index 99219525f2..ef453b421c 100644 --- a/docs/en/platform/knowledge/crawling.md +++ b/docs/en/platform/knowledge/crawling.md @@ -42,7 +42,7 @@ For a whole website, the crawler uses the homepage and published sitemaps, inclu Scans are incremental. Unchanged content is skipped, changed content is indexed again, new pages are added, and removed pages leave the index — and so do pages `robots.txt` has come to disallow. A page that redirects to another address of the same site is indexed once, under the address it lands on. The row's page counts follow the scan as pages land, after discovery and after every stored batch, so the table moves while a scan runs. A URL list follows the same refresh schedule with its fixed selection. There is no separate publish step after successful indexing. -A scan does not request every page each time. When a page's sitemap entry carries a `` date and the crawler read the page after that date, the page is left alone until the date moves. A date without a time counts for its whole day, so a page changed today is read again on each scan until the following day. A dated page is still read again once its last read is more than seven days old, because a sitemap can keep a date while the page changes, as a page that lists the latest articles does. A page whose entry has no readable date, or that no sitemap lists, is requested on every scan. A site that publishes accurate `` dates in its sitemap is therefore asked for few pages on a repeat scan. +A scan asks for each page once. It sends along what the site's server said about the page at the last visit, and a server that answers `304 Not Modified` is not asked for the page again. Many servers build their pages on request and give no modification time; for those the crawler compares the text of the page it receives. Either way, a page that did not change is not opened in the browser, so none of its images, scripts and stylesheets are loaded. A page whose content is drawn by its own JavaScript is the exception: only a browser can read it, so it is rendered on every scan. The crawler visits as an anonymous reader. Content that depends on a private session is not made accessible by adding its URL. It identifies itself on every request as `TaleBot/ (+https://docs.tale.dev/platform/knowledge/crawling)`, so a `robots.txt` group can address it by name — `User-agent: TaleBot` — to allow, throttle or refuse it alone. @@ -64,7 +64,7 @@ There is no configurable page cap, include/exclude path filter, or stop-scan but ## Check what was indexed -The table shows **Status**, the **Indexed** page count, **Scanned**, and **Interval**. Open the source row to inspect its page list, word and chunk counts, and last-crawled times. A page a scan left alone on its sitemap date keeps the time of its last read. Expand a page to read its stored text chunks. A failed fetch shows its reason and number of consecutive failures. A page the crawler skipped on purpose — the source asked not to be indexed, the content type has no readable text, or a redirect left the site — reads **Skipped** with its reason and does not count as failed. +The table shows **Status**, the **Indexed** page count, **Scanned**, and **Interval**. Open the source row to inspect its page list, word and chunk counts, and last-crawled times. Expand a page to read its stored text chunks. A failed fetch shows its reason and number of consecutive failures. A page the crawler skipped on purpose — the source asked not to be indexed, the content type has no readable text, or a redirect left the site — reads **Skipped** with its reason and does not count as failed. | Status | Meaning | | --- | --- | @@ -75,7 +75,7 @@ The table shows **Status**, the **Indexed** page count, **Scanned**, and **Inter A scan that was under way when Tale restarted, was updated or stopped without warning continues on its own within a few minutes, with the pages it had not reached yet. Pages that need the browser can follow up to a quarter of an hour later. The source reads **Scanning** throughout. -To scan outside the interval, open the source's row menu or its details and choose **Scan now**. It requests every page again, whatever the sitemap dates say. Use it after the site changed, or after a failed scan, which otherwise retries on its own within two hours. The action is offered while the source is neither scanning nor being deleted. +To scan outside the interval, open the source's row menu or its details and choose **Scan now**. Use it after the site changed, or after a failed scan, which otherwise retries on its own within two hours. The action is offered while the source is neither scanning nor being deleted. The page view also offers search over indexed content. Try a distinctive phrase from a page before relying on it in chat, then ask a specific question and inspect the citation. @@ -96,6 +96,6 @@ First check the address, source type, and latest scan time. Then open the source | **Error** with "The embedding model couldn't process the pages." | The embedding provider refused or failed the request: a rejected credential, an exhausted balance, a rate limit that held through every retry, or an outage. The source's details say which, with the provider's own reply beneath. An administrator repairs the model under **Settings > Data residency** or its credential under **Settings > AI providers**; saving either one scans the affected sources again. After an outage on the provider's side, choose **Scan now** once it is back. | | **Error** with "Scanning didn't run." | The crawler's browser could not start, so no page was rendered; its report is under **Technical details** in the source's details. The address is not the cause: ask your operator to check the sandbox service, then choose **Scan now**. | -A successful later fetch clears the previous error. A failed refresh can leave an earlier indexed copy available: **Active** and an indexed count do not prove every page is up to date. Compare the stored chunks and last-crawled information with the original before relying on a recent change. When a page changed and its sitemap date did not, the scheduled scans catch up within seven days; **Scan now** reads it at once. +A successful later fetch clears the previous error. A failed refresh can leave an earlier indexed copy available: **Active** and an indexed count do not prove every page is up to date. Compare the stored chunks and last-crawled information with the original before relying on a recent change. If the source shows **Paused**, repeated failures to reach the knowledge database stopped scans. Ask an administrator to repair the connection under **Settings > Data residency**, then use **Resume scanning**. diff --git a/docs/fr/platform/knowledge/crawling.md b/docs/fr/platform/knowledge/crawling.md index e6c33ffc8b..08b4130c5d 100644 --- a/docs/fr/platform/knowledge/crawling.md +++ b/docs/fr/platform/knowledge/crawling.md @@ -42,7 +42,7 @@ Pour un site entier, le crawler utilise l’accueil et les sitemaps publiés, y Les scans sont incrémentaux : les contenus inchangés sont ignorés, les contenus modifiés réindexés, les nouvelles pages ajoutées et les pages retirées supprimées de l’index — comme les pages que le `robots.txt` en est venu à interdire. Une page qui redirige vers une autre adresse du même site n’est indexée qu’une fois, sous l’adresse où elle aboutit. Les compteurs de pages de la ligne suivent le scan à mesure que les pages arrivent, après la découverte et après chaque lot stocké, la table bouge donc pendant qu’un scan tourne. Une liste d’URL actualise sa sélection fixe au même rythme. Aucune publication séparée n’est nécessaire après l’indexation. -Un scan ne demande pas toutes les pages à chaque passage. Quand l’entrée d’une page dans le sitemap porte une date `` et que le crawler a lu la page après cette date, il la laisse de côté jusqu’à ce que la date change. Une date sans heure vaut pour toute la journée : une page modifiée aujourd’hui est donc relue à chaque scan jusqu’au lendemain. Une page datée est tout de même relue dès que sa dernière lecture remonte à plus de sept jours, car un sitemap peut garder sa date alors que la page change, comme une page qui liste les derniers articles. Une page dont l’entrée n’a pas de date lisible, ou qu’aucun sitemap ne mentionne, est demandée à chaque scan. Un site qui publie des dates `` exactes dans son sitemap reçoit donc peu de requêtes lors d’un nouveau scan. +Un scan demande chaque page une fois. Il joint à sa requête ce que le serveur du site avait indiqué sur la page lors de la dernière visite, et un serveur qui répond `304 Not Modified` n’a pas à renvoyer la page. Beaucoup de serveurs construisent leurs pages à chaque requête et ne donnent aucune date de modification ; le crawler compare alors le texte de la page reçue. Dans les deux cas, une page inchangée n’est pas ouverte dans le navigateur, et ses images, scripts et feuilles de style ne sont donc pas chargés. L’exception est une page dont le contenu est construit par son propre JavaScript : seul un navigateur peut la lire, elle est donc rendue à chaque scan. Le crawler visite en lecteur anonyme. Ajouter une URL ne rend pas accessible un contenu privé. À chaque requête il se présente comme `TaleBot/ (+https://docs.tale.dev/platform/knowledge/crawling)`, si bien qu’un groupe de `robots.txt` peut le désigner par son nom — `User-agent: TaleBot` — pour l’autoriser, le ralentir ou le refuser seul. @@ -64,7 +64,7 @@ Tu ne peux pas fixer ton propre plafond de pages, filtrer les chemins à inclure ## Vérifier le contenu indexé -Le tableau affiche **Statut**, **Indexé**, **Analysé** et **Intervalle**. La colonne **Indexé** indique un nombre de pages. Ouvre la ligne du site pour examiner la liste des pages, le nombre de mots et de fragments et la date du dernier passage. Une page qu’un scan a laissée de côté en raison de sa date dans le sitemap garde la date de sa dernière lecture. Déplie une page pour lire ses fragments de texte enregistrés. Un échec de récupération affiche sa cause et le nombre d’échecs consécutifs. Une page que le crawler laisse volontairement de côté — la source refuse l’indexation, le type de contenu ne fournit aucun texte lisible, ou une redirection quitte le site — apparaît comme **Ignorée** avec sa cause et ne compte pas comme un échec. +Le tableau affiche **Statut**, **Indexé**, **Analysé** et **Intervalle**. La colonne **Indexé** indique un nombre de pages. Ouvre la ligne du site pour examiner la liste des pages, le nombre de mots et de fragments et la date du dernier passage. Déplie une page pour lire ses fragments de texte enregistrés. Un échec de récupération affiche sa cause et le nombre d’échecs consécutifs. Une page que le crawler laisse volontairement de côté — la source refuse l’indexation, le type de contenu ne fournit aucun texte lisible, ou une redirection quitte le site — apparaît comme **Ignorée** avec sa cause et ne compte pas comme un échec. | Statut | Signification | | --- | --- | @@ -75,7 +75,7 @@ Le tableau affiche **Statut**, **Indexé**, **Analysé** et **Intervalle**. La c Un scan en cours au moment où Tale a été redémarré, mis à jour ou arrêté brutalement reprend de lui-même en quelques minutes, avec les pages qu’il n’avait pas encore atteintes. Les pages qui ont besoin du navigateur peuvent suivre jusqu’à un quart d’heure plus tard. La source affiche **En cours d'analyse** pendant tout ce temps. -Pour lancer un scan en dehors de l’intervalle, ouvre le menu de la ligne ou les détails de la source et choisis **Analyser maintenant**. Le scan redemande alors toutes les pages, quelles que soient les dates du sitemap. C’est utile quand le site a changé ou après un scan en échec, qui sinon est relancé automatiquement dans les deux heures. L’action est proposée tant que la source n’est ni en cours d’analyse ni en cours de suppression. +Pour lancer un scan en dehors de l’intervalle, ouvre le menu de la ligne ou les détails de la source et choisis **Analyser maintenant**. C’est utile quand le site a changé ou après un scan en échec, qui sinon est relancé automatiquement dans les deux heures. L’action est proposée tant que la source n’est ni en cours d’analyse ni en cours de suppression. La vue des pages permet aussi de rechercher dans le contenu indexé. Essaie une expression distinctive de la page avant de t’appuyer dessus dans Chat, puis pose une question précise et vérifie la citation. @@ -96,6 +96,6 @@ Vérifie d’abord l’adresse, le type de source et la date du dernier scan. Ou | **Erreur** avec « Le modèle d'embedding n'a pas pu traiter les pages. » | Le fournisseur d’embedding a refusé la requête ou n’a pas pu y répondre : identifiants rejetés, solde épuisé, limite de débit maintenue malgré toutes les tentatives, ou panne. Les détails de la source précisent lequel, avec la réponse du fournisseur en dessous. Un administrateur corrige le modèle dans **Paramètres > Résidence des données** ou ses identifiants dans **Paramètres > Fournisseurs IA** ; l’enregistrement relance le scan des sources concernées. Après une panne chez le fournisseur, choisis **Analyser maintenant** dès qu’il est de nouveau disponible. | | **Erreur** avec « L'analyse n'a pas eu lieu. » | Le navigateur du crawler n’a pas pu démarrer, aucune page n’a donc été rendue ; son rapport se trouve sous **Détails techniques** dans les détails de la source. L’adresse n’est pas en cause : demande à ton exploitant de vérifier le service de sandbox, puis choisis **Analyser maintenant**. | -Une récupération réussie efface l’erreur précédente. Si une actualisation échoue, la copie déjà indexée peut rester disponible : **Actif** et le nombre de pages indexées ne prouvent pas que chaque page est à jour. Compare les fragments enregistrés et la date du passage avec l’original avant de t’appuyer sur une modification récente. Quand une page a changé sans que sa date dans le sitemap change, les scans planifiés la rattrapent en sept jours au plus ; **Analyser maintenant** la relit tout de suite. +Une récupération réussie efface l’erreur précédente. Si une actualisation échoue, la copie déjà indexée peut rester disponible : **Actif** et le nombre de pages indexées ne prouvent pas que chaque page est à jour. Compare les fragments enregistrés et la date du passage avec l’original avant de t’appuyer sur une modification récente. Si la source affiche **Suspendu**, des échecs répétés d’accès à la base de connaissances ont arrêté les analyses. Demande à un administrateur de réparer la connexion dans **Paramètres > Résidence des données**, puis utilise **Reprendre l'analyse**. diff --git a/services/db/migrations/knowledge-db/public_web/00000000000013_website_url_change_check.sql b/services/db/migrations/knowledge-db/public_web/00000000000013_website_url_change_check.sql new file mode 100644 index 0000000000..e130f95e96 --- /dev/null +++ b/services/db/migrations/knowledge-db/public_web/00000000000013_website_url_change_check.sql @@ -0,0 +1,53 @@ +-- migrate:up +-- +-- What a scan checks a page by, before it spends a browser on it. Until +-- here the only way the crawler could tell whether a page had changed was +-- to do everything again: fetch it, render it in a browser with every image, +-- script and stylesheet the render loads, and compare content hashes +-- afterwards. A 700-page site scanned every six hours answered some 8,000 +-- requests and two gigabytes a scan for pages that had not changed in +-- months (2026-10-06). +-- +-- A scan now asks for each page once, and opens the browser only for a page +-- that did change. The row keeps what that one request is judged by: +-- +-- etag the `ETag` of the response the stored text came from +-- last_modified its `Last-Modified`; the two are sent back as +-- `If-None-Match` / `If-Modified-Since`, and a server that +-- answers 304 is not asked for the page at all +-- probe_hash the hash of the text read out of the page's plain HTML, +-- for the servers that send neither: the same text again +-- is the same page +-- +-- All three describe the response whose text the row stores AND has +-- indexed: they are written when a page settles and cleared when its +-- content is purged. They stay NULL for a page whose plain HTML does not +-- carry what a browser shows — a page built by its JavaScript — because a +-- 304 on such a shell, or the same shell text, says nothing about the +-- content; that page is rendered on every scan, as every page was. +-- +-- `etag` and `last_modified` are in the baseline and went unused when the +-- render lane arrived; they are named here too so a corpus that was +-- bootstrapped without them converges. ADD COLUMN IF NOT EXISTS is +-- metadata-only, idempotent and rolling-safe — the previous image neither +-- reads nor writes the columns. +-- +-- What the two columns still hold was written before the render lane, for +-- every page alike: a value there may belong to a JavaScript shell, and +-- sent back it would have that page answered 304 and never rendered again. +-- It is cleared once, here. Applied again (a corpus without a migration +-- ledger re-applies every file) it only costs each page one full visit. + +ALTER TABLE public_web.website_urls + ADD COLUMN IF NOT EXISTS etag TEXT, + ADD COLUMN IF NOT EXISTS last_modified TEXT, + ADD COLUMN IF NOT EXISTS probe_hash TEXT; + +UPDATE public_web.website_urls + SET etag = NULL, last_modified = NULL + WHERE etag IS NOT NULL OR last_modified IS NOT NULL; + +-- migrate:down +-- Deliberately empty, like the baseline: dropping the columns would only +-- send every scan back to rendering every page. Remove with an explicit, +-- reviewed migration instead. diff --git a/services/db/migrations/knowledge-db/public_web/00000000000013_website_url_sitemap_lastmod.sql b/services/db/migrations/knowledge-db/public_web/00000000000013_website_url_sitemap_lastmod.sql deleted file mode 100644 index a8d65db272..0000000000 --- a/services/db/migrations/knowledge-db/public_web/00000000000013_website_url_sitemap_lastmod.sql +++ /dev/null @@ -1,34 +0,0 @@ --- migrate:up --- --- What a site's sitemap says about when a page last changed, kept on the --- page's row so a rescan can leave an unchanged page alone. Until here the --- crawler read `` out of a sitemap and nothing else: every scan --- requested every page again and compared content hashes afterwards, so a --- 700-page site scanned every six hours answered some 8,000 requests and --- two gigabytes a scan for pages its sitemap marked unchanged for months --- (2026-10-06). --- --- sitemap_lastmod the latest instant the entry's `` can mean, as --- the discovery of the last scan read it: a date-only --- value covers its whole day in any time zone, a time --- without a zone the same half day. NULL when that --- discovery read none for the page — no sitemap lists --- it, the entry carries no date or one that cannot be --- read, or a person started the scan — and the page is --- then requested as before. --- --- A page is left alone only while its stored text was read after this --- instant; the rule, its margin and the weekly re-read live in the frontier --- predicate (`crawl_action.ts`). Each discovery replaces what the one before --- recorded, so a date a sitemap no longer states never keeps a page skipped. --- --- Pure ADD COLUMN IF NOT EXISTS: metadata-only, idempotent, rolling-safe — --- the previous image neither reads nor writes the column. - -ALTER TABLE public_web.website_urls - ADD COLUMN IF NOT EXISTS sitemap_lastmod TIMESTAMPTZ; - --- migrate:down --- Deliberately empty, like the baseline: dropping the column would only send --- every scan back to requesting every page. Remove with an explicit, --- reviewed migration instead. diff --git a/services/platform/backend/core/knowledge/crawl.test.ts b/services/platform/backend/core/knowledge/crawl.test.ts index 49ea01626c..7a7c3e40c3 100644 --- a/services/platform/backend/core/knowledge/crawl.test.ts +++ b/services/platform/backend/core/knowledge/crawl.test.ts @@ -8,7 +8,6 @@ import { admitUrls, admitUrlsStatement, deregisterDomain, - recordSitemapLastmods, registerDomain, registerUrlList, reviveListedUrls, @@ -272,80 +271,3 @@ describe('reviveListedUrls', () => { expect(call?.params).toEqual(['example.test']); }); }); - -/** - * A page's sitemap date is what a rescan leaves it alone by, so each - * discovery replaces the record: a date the sitemap no longer states must - * not keep its page skipped, and a scan that records none requests every - * page. - */ -describe('recordSitemapLastmods', () => { - const DOMAIN = 'docs.example'; - - it('forgets the dates this scan did not read and writes the ones that moved, in one transaction', async () => { - const { sql, sent, begins } = recorder(); - - await recordSitemapLastmods( - sql, - DOMAIN, - new Map([ - ['https://docs.example/a', new Date('2026-10-07T12:00:00.000Z')], - ['https://docs.example/b', new Date('2026-01-03T12:00:00.000Z')], - ]), - ); - - expect(begins()).toBe(1); - expect(sent.every((statement) => statement.inTx)).toBe(true); - expect(sent).toHaveLength(2); - const [forget, write] = sent; - expect(forget?.text).toContain('SET sitemap_lastmod = NULL'); - expect(forget?.text).toContain('sitemap_lastmod IS NOT NULL'); - expect(forget?.text).toContain('NOT (url = ANY($2::text[]))'); - expect(forget?.params).toEqual([ - DOMAIN, - ['https://docs.example/a', 'https://docs.example/b'], - ]); - expect(write?.text).toContain( - 'SET sitemap_lastmod = v.lastmod::timestamptz', - ); - // A date that has not moved is not written again. - expect(write?.text).toContain( - 'u.sitemap_lastmod IS DISTINCT FROM v.lastmod::timestamptz', - ); - expect(write?.params).toEqual([ - DOMAIN, - ['https://docs.example/a', 'https://docs.example/b'], - ['2026-10-07T12:00:00.000Z', '2026-01-03T12:00:00.000Z'], - ]); - }); - - it('forgets every date and writes none when the scan read none', async () => { - const { sql, sent } = recorder(); - - await recordSitemapLastmods(sql, DOMAIN, new Map()); - - expect(sent).toHaveLength(1); - expect(sent[0]?.text).toContain('SET sitemap_lastmod = NULL'); - expect(sent[0]?.params).toEqual([DOMAIN, []]); - }); - - it('writes a large sitemap in batches', async () => { - const { sql, sent } = recorder(); - const lastmods = new Map( - Array.from({ length: URL_INSERT_BATCH + 1 }, (_, index) => [ - `https://docs.example/${index}`, - new Date('2026-01-03T12:00:00.000Z'), - ]), - ); - - await recordSitemapLastmods(sql, DOMAIN, lastmods); - - const writes = sent.filter((statement) => - statement.text.includes('FROM unnest('), - ); - expect(writes.map((statement) => statement.params[1])).toEqual([ - expect.objectContaining({ length: URL_INSERT_BATCH }), - expect.objectContaining({ length: 1 }), - ]); - }); -}); diff --git a/services/platform/backend/core/knowledge/crawl.ts b/services/platform/backend/core/knowledge/crawl.ts index 1e4ef189be..547060c82a 100644 --- a/services/platform/backend/core/knowledge/crawl.ts +++ b/services/platform/backend/core/knowledge/crawl.ts @@ -145,53 +145,6 @@ export async function reviveListedUrls( return rows.length; } -/** - * Record what this scan's sitemaps say about each page's last change, and - * forget what an earlier scan's said — the two in one transaction, so the - * frontier never reads half of either. - * - * `lastmods` REPLACES the domain's record: a page it does not name loses the - * date an earlier discovery wrote. That is what keeps a stale statement from - * skipping a page for ever — the sitemap dropped the entry or its date, the - * sitemap could not be fetched this time, discovery ran out of its budget - * before reaching it — and what an empty map means: a scan a person started - * records none, so it requests every page. A date that has not moved is not - * written again; on an ordinary rescan nearly every row is left untouched. - */ -export async function recordSitemapLastmods( - sql: Sql, - domain: string, - lastmods: ReadonlyMap, -): Promise { - const entries = [...lastmods]; - await sql.begin(async (tx) => { - await tx.unsafe( - `UPDATE ${PUBLIC_WEB_SCHEMA}.website_urls - SET sitemap_lastmod = NULL - WHERE domain = $1 AND sitemap_lastmod IS NOT NULL - AND NOT (url = ANY($2::text[]))`, - [domain, entries.map(([url]) => url)], - ); - for (let start = 0; start < entries.length; start += URL_INSERT_BATCH) { - const batch = entries.slice(start, start + URL_INSERT_BATCH); - // The instants travel as text and are cast by the server, as a scan's - // start does in the admission statement. - await tx.unsafe( - `UPDATE ${PUBLIC_WEB_SCHEMA}.website_urls AS u - SET sitemap_lastmod = v.lastmod::timestamptz - FROM unnest($2::text[], $3::text[]) AS v(url, lastmod) - WHERE u.domain = $1 AND u.url = v.url - AND u.sitemap_lastmod IS DISTINCT FROM v.lastmod::timestamptz`, - [ - domain, - batch.map(([url]) => url), - batch.map(([, lastmod]) => lastmod.toISOString()), - ], - ); - } - }); -} - /** Corpus → websites-row status vocabulary. The corpus distinguishes * `completed` (a finished scan) from `active`; the websites row treats both * as a healthy scanned site. A corpus row is born `idle` (the column diff --git a/services/platform/backend/core/knowledge/crawl_action.change-check.test.ts b/services/platform/backend/core/knowledge/crawl_action.change-check.test.ts new file mode 100644 index 0000000000..d2e544c6ba --- /dev/null +++ b/services/platform/backend/core/knowledge/crawl_action.change-check.test.ts @@ -0,0 +1,575 @@ +// @vitest-environment node + +import { computeContentHash } from '@tale/shared/utils/hashing'; +import type { Sql } from 'postgres'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { EMPTY_ROBOTS_POLICY } from '../../../lib/knowledge/crawl-parse'; +import { htmlToText } from '../../../lib/knowledge/html-to-text'; +import { safeFetchBinary } from '../../../lib/net/safe-fetch'; +import type { ActionCtx } from '../lib/ctx'; +import { renderUrlsInSandbox } from '../node_only/sandbox/render_fetch'; +import { + checkAfterRender, + fetchAndStorePage, + scanWebsiteImpl, +} from './crawl_action'; +import { getKnowledgePoolForOrg } from './pool'; + +vi.mock('../../../lib/net/safe-fetch', async (importOriginal) => ({ + ...(await importOriginal()), + safeFetchBinary: vi.fn(), +})); +vi.mock('../node_only/sandbox/render_fetch', async (importOriginal) => ({ + ...(await importOriginal< + typeof import('../node_only/sandbox/render_fetch') + >()), + renderUrlsInSandbox: vi.fn(), +})); +vi.mock('./pool', async (importOriginal) => ({ + ...(await importOriginal()), + getKnowledgePoolForOrg: vi.fn(), +})); + +/** + * A scan asks for each page once and opens the browser only for a page that + * changed. The regression: the crawler's one way to learn whether a page + * had changed was to do everything again — fetch it, render it with every + * image, script and stylesheet the render loads, and compare content hashes + * afterwards. A 700-page site scanned every six hours answered some 8,000 + * requests and two gigabytes a scan for pages unchanged in months. + * + * The request now carries the validators of the last visit, and a 304 ends + * it. A server that builds its pages on request has no modification time to + * give — no `ETag`, no `Last-Modified`, a new token in every response — so + * the text read out of the plain HTML is compared too. A page whose plain + * HTML does not carry what a browser shows keeps nothing to be checked by: + * a 304 on such a shell proves nothing, and that page is rendered again. + * + * Pinned on recording doubles; what the statements do to real rows rides + * the integration check. + */ + +const DOMAIN = 'docs.example'; +const URL = 'https://docs.example/guide'; + +const body = (words: string, token: string): string => + `Guide

Guide

${words}

`; +const TEXT_V1 = + 'Tale reads the public pages of a website and answers questions from them.'; +const TEXT_V2 = + 'Tale reads the public pages of a website, and every changed page again.'; +const hashOf = (html: string): string => computeContentHash(htmlToText(html)); + +interface Statement { + text: string; + params: unknown[]; +} + +function corpus(): { sql: Sql; statements: Statement[] } { + const statements: Statement[] = []; + const unsafe = (text: string, params: unknown[] = []): Promise => { + statements.push({ text: text.replace(/\s+/g, ' ').trim(), params }); + return Promise.resolve(text.includes('RETURNING u.url') ? [{}] : []); + }; + const sql = { + unsafe, + begin: async (run: (tx: { unsafe: typeof unsafe }) => Promise) => + run({ unsafe }), + }; + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- test double + return { sql: sql as unknown as Sql, statements }; +} + +interface Answer { + status?: number; + type?: string; + body?: string | Uint8Array; + headers?: Record; + finalUrl?: string; +} + +const answers = (answer: Answer): void => { + vi.mocked(safeFetchBinary).mockResolvedValue({ + status: answer.status ?? 200, + statusText: answer.status === 304 ? 'Not Modified' : 'OK', + headers: new Headers({ + 'content-type': answer.type ?? 'text/html; charset=utf-8', + ...answer.headers, + }), + body: new Blob([answer.body ?? '']), + finalUrl: answer.finalUrl ?? URL, + }); +}; + +/** A page a past scan stored, indexed and left something to check it by. */ +const settled = (check: { + etag?: string | null; + last_modified?: string | null; + probe_hash?: string | null; +}) => ({ + url: URL, + content_hash: 'stored-text-hash', + listed: false, + etag: null, + last_modified: null, + probe_hash: null, + indexed: true, + ...check, +}); + +const sentHeaders = (): Record => { + const options = vi.mocked(safeFetchBinary).mock.calls[0]?.[1]; + return { ...options?.headers }; +}; +const stamps = (statements: Statement[]) => + statements.filter((statement) => + statement.text.includes('last_crawled_at = NOW()'), + ); +const probe = (page: Parameters[2]) => + fetchAndStorePage(corpus().sql, DOMAIN, page, EMPTY_ROBOTS_POLICY); + +beforeEach(() => { + vi.mocked(safeFetchBinary).mockReset(); + vi.mocked(renderUrlsInSandbox).mockReset(); + vi.mocked(getKnowledgePoolForOrg).mockReset(); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); +}); + +describe('fetchAndStorePage — asking whether a page changed', () => { + it('sends the validators of the last settled visit', async () => { + answers({ status: 304 }); + + await probe( + settled({ + etag: '"v1"', + last_modified: 'Fri, 10 Jul 2026 09:11:16 GMT', + }), + ); + + expect(sentHeaders()).toMatchObject({ + 'If-None-Match': '"v1"', + 'If-Modified-Since': 'Fri, 10 Jul 2026 09:11:16 GMT', + }); + expect(sentHeaders()['User-Agent']).toContain('TaleBot/'); + }); + + // "Unchanged" would leave a page without text, or with the chunks of its + // earlier text, exactly as it is. + it.each([ + ['holds no text', { content_hash: null }], + ['holds text the index does not have yet', { indexed: false }], + ])('asks unconditionally for a page whose row %s', async (_case, state) => { + answers({ body: body(TEXT_V1, 'a') }); + + const outcome = await probe({ + ...settled({ etag: '"v1"', probe_hash: hashOf(body(TEXT_V1, 'a')) }), + ...state, + }); + + expect(sentHeaders()).not.toHaveProperty('If-None-Match'); + expect(outcome).toMatchObject({ kind: 'render' }); + }); + + it('leaves a page as it is when the server answers 304 [KNOW-R16]', async () => { + answers({ status: 304 }); + const { sql, statements } = corpus(); + + await expect( + fetchAndStorePage( + sql, + DOMAIN, + settled({ etag: '"v1"' }), + EMPTY_ROBOTS_POLICY, + ), + ).resolves.toBe('not_modified'); + + // One write: the visit, and the end of whatever failure stood before. + expect(statements).toHaveLength(1); + expect(statements[0]?.text).toContain('last_crawled_at = NOW()'); + expect(statements[0]?.text).toContain('fail_count = 0'); + expect(statements[0]?.text).toContain('last_error = NULL'); + // A 304 brings no validators: the row keeps its own. + expect(statements[0]?.text).not.toContain('etag'); + expect(statements[0]?.params).toEqual([DOMAIN, URL]); + }); + + it('reads a 304 nobody asked for as the error it is', async () => { + answers({ status: 304 }); + const { sql, statements } = corpus(); + + await expect( + fetchAndStorePage(sql, DOMAIN, settled({}), EMPTY_ROBOTS_POLICY), + ).resolves.toBe('failed'); + + expect(statements[0]?.params).toContain('http_error'); + }); + + // A page that is unchanged is still a page: the gates every response + // passes come first. + it('retires an alias and honours X-Robots-Tag on a 304 too', async () => { + answers({ status: 304, finalUrl: 'https://docs.example/manual' }); + await expect(probe(settled({ etag: '"v1"' }))).resolves.toBe('unchanged'); + + answers({ status: 304, headers: { 'x-robots-tag': 'noindex' } }); + await expect(probe(settled({ etag: '"v1"' }))).resolves.toBe('failed'); + }); +}); + +describe('fetchAndStorePage — a server that gives no modification time', () => { + // Every response of such a server differs in its bytes: a token in a meta + // tag, a nonce in a script. The text a reader sees is what is compared. + it('leaves a page as it is when its plain HTML reads as it did [KNOW-R16]', async () => { + const stored = body(TEXT_V1, 'token-of-the-last-visit'); + const now = body(TEXT_V1, 'token-of-this-visit'); + expect(now).not.toBe(stored); + answers({ body: now, headers: { etag: '"regenerated"' } }); + const { sql, statements } = corpus(); + + await expect( + fetchAndStorePage( + sql, + DOMAIN, + settled({ probe_hash: hashOf(stored) }), + EMPTY_ROBOTS_POLICY, + ), + ).resolves.toBe('not_modified'); + + expect(statements).toHaveLength(1); + expect(statements[0]?.text).toContain('last_crawled_at = NOW()'); + // The validators of THIS response replace the row's. + expect(statements[0]?.params).toEqual([ + DOMAIN, + URL, + '"regenerated"', + null, + hashOf(stored), + ]); + }); + + it('sends a page whose text changed to the browser, with what the request found', async () => { + const now = body(TEXT_V2, 'b'); + answers({ + body: now, + headers: { 'last-modified': 'Tue, 06 Oct 2026 08:00:00 GMT' }, + }); + const { sql, statements } = corpus(); + + await expect( + fetchAndStorePage( + sql, + DOMAIN, + settled({ probe_hash: hashOf(body(TEXT_V1, 'a')) }), + EMPTY_ROBOTS_POLICY, + ), + ).resolves.toEqual({ + kind: 'render', + probe: { + text: htmlToText(now), + hash: hashOf(now), + etag: null, + lastModified: 'Tue, 06 Oct 2026 08:00:00 GMT', + }, + }); + + // The row stays unmarked until the render batch settles it. + expect(statements).toEqual([]); + }); + + // A page built by its JavaScript keeps nothing to be checked by: the same + // shell says nothing about what the page shows. + it('renders a page only a browser can read on every scan [KNOW-R16]', async () => { + const shell = body('Loading…', 'a'); + answers({ body: shell, headers: { etag: '"shell"' } }); + + await expect(probe(settled({}))).resolves.toMatchObject({ + kind: 'render', + }); + expect(sentHeaders()).not.toHaveProperty('If-None-Match'); + }); + + // The tag can change without a word of the text moving. + it('still honours a robots meta tag on a page whose text did not change', async () => { + const stored = body(TEXT_V1, 'a'); + const withdrawn = stored.replace( + '', + '<meta name="robots" content="noindex"><title>', + ); + expect(hashOf(withdrawn)).toBe(hashOf(stored)); + answers({ body: withdrawn }); + const { sql, statements } = corpus(); + + await expect( + fetchAndStorePage( + sql, + DOMAIN, + settled({ probe_hash: hashOf(stored) }), + EMPTY_ROBOTS_POLICY, + ), + ).resolves.toBe('failed'); + + expect( + statements.some((statement) => + statement.params.includes('robots_noindex'), + ), + ).toBe(true); + // What it was checked by goes with its content. + const purge = statements.find((statement) => + statement.text.includes('SET content = NULL'), + ); + expect(purge?.text).toContain( + 'etag = NULL, last_modified = NULL, probe_hash = NULL', + ); + }); + + // Read as UTF-8, every accented word of such a page comes out broken and + // would never match what the browser shows. + it('reads a page in the charset its content type names', async () => { + const html = body('Über die Größe der Änderung', 'a'); + answers({ + type: 'text/html; charset=ISO-8859-1', + body: Uint8Array.from(Buffer.from(html, 'latin1')), + }); + + await expect(probe(settled({}))).resolves.toMatchObject({ + probe: { hash: hashOf(html) }, + }); + }); +}); + +describe('fetchAndStorePage — a document', () => { + // A document's bytes are its content, so its validators always hold. + it('hands the validators of a stored document on, to be kept once it is indexed', async () => { + answers({ + type: 'text/plain', + body: 'Release notes. Enough words to be stored as the text of this page.', + headers: { + etag: '"doc-1"', + 'last-modified': 'Fri, 10 Jul 2026 09:11:16 GMT', + }, + }); + const { sql, statements } = corpus(); + + await expect( + fetchAndStorePage( + sql, + DOMAIN, + { url: URL, content_hash: null, listed: false }, + EMPTY_ROBOTS_POLICY, + ), + ).resolves.toEqual({ + kind: 'stored', + outcome: 'changed', + check: { + etag: '"doc-1"', + lastModified: 'Fri, 10 Jul 2026 09:11:16 GMT', + probeHash: null, + }, + }); + + // Stored, not yet stamped: the scan indexes it first. + expect(stamps(statements)).toEqual([]); + }); + + it('is not downloaded again when the server answers 304', async () => { + answers({ status: 304, type: 'application/pdf' }); + + await expect(probe(settled({ etag: '"doc-1"' }))).resolves.toBe( + 'not_modified', + ); + }); +}); + +describe('checkAfterRender', () => { + const plain = htmlToText(body(TEXT_V1, 'a')); + const found = { + text: plain, + hash: computeContentHash(plain), + etag: '"v2"', + lastModified: null, + }; + + it('keeps what the request found when the plain HTML carried what the browser showed', () => { + expect(checkAfterRender(found, plain)).toEqual({ + etag: '"v2"', + lastModified: null, + probeHash: found.hash, + }); + }); + + it.each([ + [ + 'the browser showed a page its plain HTML does not carry', + found, + `${TEXT_V2} ${TEXT_V2} A long article that only the page's own scripts fetched and rendered for the reader.`, + ], + [ + 'the plain HTML was too large to read', + { ...found, text: null, hash: null }, + plain, + ], + ])('keeps nothing when %s', (_case, probed, rendered) => { + expect(checkAfterRender(probed, rendered)).toEqual({ + etag: null, + lastModified: null, + probeHash: null, + }); + }); +}); + +/** + * One link of a scan, on a later link (no discovery): the frontier hands it + * one page, whose request says it changed. + */ +describe('scanWebsiteImpl — a page that changed', () => { + const SCAN = { + domain: DOMAIN, + orgSlug: 'acme', + organizationId: 'org-1', + continuation: 1, + scanStartedAt: '2026-10-06T13:00:00.000Z', + }; + + /** The frontier holds `page` once; its stored text is already what the + * browser shows, so storing it changes nothing in the index. */ + function scanCorpus(page: Record<string, unknown>): { + sql: Sql; + statements: Statement[]; + } { + const statements: Statement[] = []; + let handed = false; + const unsafe = ( + text: string, + params: unknown[] = [], + ): Promise<unknown[]> => { + const statement = text.replace(/\s+/g, ' ').trim(); + statements.push({ text: statement, params }); + if (statement.startsWith('SELECT kind')) { + return Promise.resolve([{ kind: 'site', robots_disallow: null }]); + } + if (statement.startsWith('SELECT url, content_hash, listed')) { + if (handed) return Promise.resolve([]); + handed = true; + return Promise.resolve([page]); + } + if (statement.includes('bool_or(embedding IS NULL)')) { + return Promise.resolve([ + { present: true, current: true, vectorless: false }, + ]); + } + if (statement.includes('FILTER (WHERE u.status')) { + return Promise.resolve([ + { stored: '1', attempted: '1', failed: '0', skipped: '0' }, + ]); + } + return Promise.resolve([]); + }; + const sql = { + unsafe, + begin: async (run: (tx: { unsafe: typeof unsafe }) => Promise<void>) => + run({ unsafe }), + }; + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- test double + return { sql: sql as unknown as Sql, statements }; + } + + function engineCtx(): ActionCtx { + const ctx = { + runMutation: vi.fn(async () => ({ paused: false })), + runQuery: vi.fn(), + scheduler: { runAfter: vi.fn() }, + }; + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only what the link reaches + return ctx as unknown as ActionCtx; + } + + const shows = (html: string): void => { + vi.mocked(renderUrlsInSandbox).mockResolvedValue({ + outcomes: new Map([ + [URL, { kind: 'ok', status: 200, finalUrl: URL, html }], + ]), + halted: null, + }); + }; + const settledWrite = (statements: Statement[]) => + statements.find( + (statement) => + statement.text.includes('last_crawled_at = NOW()') && + statement.text.includes('probe_hash = $5'), + ); + + it('reads what a page is checked by out of the frontier, with whether its text is indexed', async () => { + const { sql, statements } = scanCorpus(settled({})); + vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); + answers({ status: 304 }); + + await scanWebsiteImpl(engineCtx(), SCAN); + + const frontier = statements.find((statement) => + statement.text.startsWith('SELECT url, content_hash, listed'), + ); + expect(frontier?.text).toContain('etag, last_modified, probe_hash'); + expect(frontier?.text).toContain( + 'c.content_hash IS DISTINCT FROM website_urls.content_hash ) AS indexed', + ); + }); + + it('opens no browser for a page that did not change [KNOW-R16]', async () => { + const { sql } = scanCorpus(settled({ etag: '"v1"' })); + vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); + answers({ status: 304 }); + + await scanWebsiteImpl(engineCtx(), SCAN); + + expect(renderUrlsInSandbox).not.toHaveBeenCalled(); + expect(vi.mocked(console.log).mock.calls.flat().join('\n')).toContain( + '1 page(s) requested, 1 unchanged, 0 rendered', + ); + }); + + it('renders a changed page and keeps what to check it by next time', async () => { + const now = body(TEXT_V2, 'b'); + const { sql, statements } = scanCorpus( + settled({ probe_hash: hashOf(body(TEXT_V1, 'a')) }), + ); + vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); + answers({ body: now, headers: { etag: '"v2"' } }); + // The browser shows what the plain HTML already said. + shows(now); + + await scanWebsiteImpl(engineCtx(), SCAN); + + expect(renderUrlsInSandbox).toHaveBeenCalledTimes(1); + expect(settledWrite(statements)?.params).toEqual([ + DOMAIN, + URL, + '"v2"', + null, + hashOf(now), + ]); + }); + + it('keeps nothing to check a page by when only the browser could read it', async () => { + const shell = body('Loading…', 'a'); + const { sql, statements } = scanCorpus(settled({})); + vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); + answers({ body: shell, headers: { etag: '"shell"' } }); + shows( + body( + `${TEXT_V1} ${TEXT_V2} A long article that only the page's own scripts fetched and rendered for the reader.`, + 'a', + ), + ); + + await scanWebsiteImpl(engineCtx(), SCAN); + + expect(settledWrite(statements)?.params).toEqual([ + DOMAIN, + URL, + null, + null, + null, + ]); + }); +}); diff --git a/services/platform/backend/core/knowledge/crawl_action.lastmod.test.ts b/services/platform/backend/core/knowledge/crawl_action.lastmod.test.ts deleted file mode 100644 index 8070e76ee6..0000000000 --- a/services/platform/backend/core/knowledge/crawl_action.lastmod.test.ts +++ /dev/null @@ -1,255 +0,0 @@ -// @vitest-environment node - -import type { Sql } from 'postgres'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; - -import { safeFetch } from '../../../lib/net/safe-fetch'; -import type { ActionCtx } from '../lib/ctx'; -import { scanWebsiteImpl } from './crawl_action'; -import { getKnowledgePoolForOrg } from './pool'; - -vi.mock('./pool', async (importOriginal) => ({ - ...(await importOriginal<typeof import('./pool')>()), - getKnowledgePoolForOrg: vi.fn(), -})); -vi.mock('../../../lib/net/safe-fetch', async (importOriginal) => ({ - ...(await importOriginal<typeof import('../../../lib/net/safe-fetch')>()), - safeFetch: vi.fn(), -})); - -/** - * A rescan leaves alone the pages a site's sitemap says have not changed. - * The regression: the crawler read `<loc>` out of a sitemap and nothing - * else, so every scan requested every page again — probe, browser render - * and all the assets a render loads — and compared content hashes - * afterwards. A site that dated all 700 of its pages answered some 8,000 - * requests a scan, every six hours, for pages unchanged in months. - * - * Pinned on a recording double (which statements run, with which - * parameters); what the frontier predicate selects on real rows rides the - * integration check. - */ - -const SCAN = { - domain: 'docs.example', - orgSlug: 'acme', - organizationId: 'org-1', -}; - -/** Twelve pages, over the link-walk threshold, so discovery is the sitemap - * alone. The first two carry a date, the homepage twice. */ -const SITEMAP = `<?xml version="1.0" encoding="UTF-8"?> -<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> - <url><loc>https://docs.example</loc><lastmod>2026-07-27</lastmod></url> - <url><loc>https://docs.example/guide</loc><lastmod>2026-10-06T08:00:00+02:00</lastmod></url> - <url><loc>https://docs.example/</loc><lastmod>2026-08-01</lastmod></url> - <url><loc>https://docs.example/undated</loc></url> - <url><loc>https://docs.example/unreadable</loc><lastmod>last week</lastmod></url> - ${Array.from( - { length: 8 }, - (_, index) => `<url><loc>https://docs.example/more/${index}</loc></url>`, - ).join('\n ')} -</urlset>`; - -interface Statement { - text: string; - params: unknown[]; - inTx: boolean; -} - -/** A corpus double that grants the claim and answers the frontier as - * empty: the link discovers, records and finishes. */ -function corpus(): { sql: Sql; statements: Statement[] } { - const statements: Statement[] = []; - const unsafeFor = - (inTx: boolean) => - (text: string, params: unknown[] = []): Promise<unknown[]> => { - const statement = text.replace(/\s+/g, ' ').trim(); - statements.push({ text: statement, params, inTx }); - if (statement.startsWith('SELECT kind')) { - return Promise.resolve([{ kind: 'site', robots_disallow: null }]); - } - if (statement.includes("SET status = 'scanning'")) { - return Promise.resolve([{ domain: SCAN.domain }]); - } - if (statement.includes('AND COALESCE(')) { - return Promise.resolve([{ n: '2' }]); - } - if (statement.includes('FILTER (WHERE u.status')) { - return Promise.resolve([ - { stored: '12', attempted: '12', failed: '0', skipped: '0' }, - ]); - } - return Promise.resolve([]); - }; - const sql = { - unsafe: unsafeFor(false), - begin: async ( - run: (tx: { unsafe: ReturnType<typeof unsafeFor> }) => Promise<void>, - ) => run({ unsafe: unsafeFor(true) }), - }; - // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- test double - return { sql: sql as unknown as Sql, statements }; -} - -function engineCtx(): ActionCtx { - const ctx = { - runMutation: vi.fn(async () => ({ paused: false })), - runQuery: vi.fn(), - scheduler: { runAfter: vi.fn() }, - }; - // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only what the link reaches - return ctx as unknown as ActionCtx; -} - -const forgotten = (statements: Statement[]) => - statements.filter((statement) => - statement.text.includes('SET sitemap_lastmod = NULL'), - ); -const written = (statements: Statement[]) => - statements.filter((statement) => statement.text.includes('FROM unnest(')); -const frontier = (statements: Statement[]) => - statements.filter( - (statement) => - statement.text.startsWith('SELECT url, content_hash, listed') || - (statement.text.startsWith('SELECT count(*)::text AS n') && - statement.text.includes('AND NOT COALESCE(')), - ); - -const logged = (): string => - vi - .mocked(console.log) - .mock.calls.map((call) => String(call[0])) - .join('\n'); - -beforeEach(() => { - vi.mocked(getKnowledgePoolForOrg).mockReset(); - vi.mocked(safeFetch).mockReset(); - vi.mocked(safeFetch).mockImplementation(async (url) => { - const answer = (body: string) => ({ - status: 200, - statusText: 'OK', - headers: new Headers(), - body, - finalUrl: url, - }); - if (url.endsWith('/robots.txt')) { - return answer( - `User-agent: *\nSitemap: https://${SCAN.domain}/sitemap.xml\n`, - ); - } - if (url.endsWith('/sitemap.xml')) return answer(SITEMAP); - return { ...answer('gone'), status: 404, statusText: 'Not Found' }; - }); - // Cleared: a spy on a method that is already spied on keeps its calls. - vi.spyOn(console, 'log') - .mockClear() - .mockImplementation(() => undefined); - vi.spyOn(console, 'warn').mockImplementation(() => undefined); -}); - -describe("scanWebsiteImpl — what a site's sitemap says about its pages", () => { - it('records the date of every page whose entry states one, and forgets the rest', async () => { - const { sql, statements } = corpus(); - vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); - - await scanWebsiteImpl(engineCtx(), SCAN); - - // The homepage is named twice: the later date stands. A date alone - // covers its whole day in any time zone. - const dated = [ - ['https://docs.example/', '2026-08-02T12:00:00.000Z'], - ['https://docs.example/guide', '2026-10-06T06:00:00.000Z'], - ]; - expect(written(statements).map((statement) => statement.params)).toEqual([ - [ - SCAN.domain, - dated.map(([url]) => url), - dated.map(([, lastmod]) => lastmod), - ], - ]); - // An entry without a date, or with one that cannot be read, leaves its - // page without one: such a page is requested. - expect(forgotten(statements).map((statement) => statement.params)).toEqual([ - [SCAN.domain, dated.map(([url]) => url)], - ]); - expect( - [...forgotten(statements), ...written(statements)].every( - (statement) => statement.inTx, - ), - ).toBe(true); - // After the admission: a page found by this scan has its row by then. - const texts = statements.map((statement) => statement.text); - expect( - texts.findIndex((text) => text.includes('RETURNING u.url')), - ).toBeLessThan(texts.findIndex((text) => text.includes('FROM unnest('))); - expect(logged()).toContain( - '12 URLs discovered, 2 not requested (unchanged by their sitemap dates)', - ); - }); - - it('leaves a page alone only while its stored text was read after the date, within the week [KNOW-R16]', async () => { - const { sql, statements } = corpus(); - vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); - - await scanWebsiteImpl(engineCtx(), SCAN); - - // Both readers of the frontier: the pages a link takes next, and how - // many are left for the next link. - const readers = frontier(statements); - expect(readers).toHaveLength(2); - for (const reader of readers) { - // Without a date, or never read, the clause is NULL and the page due. - expect(reader.text).toContain('AND NOT COALESCE(('); - expect(reader.text).toContain(', FALSE)'); - // The row holds the text of a read that succeeded. - expect(reader.text).toContain( - "status = 'active' AND content_hash IS NOT NULL AND last_error IS NULL", - ); - // That read came after the date, and a margin for a cache. - expect(reader.text).toContain( - "last_crawled_at > sitemap_lastmod + interval '1 hour'", - ); - // And within the week: a sitemap that never moves its dates. - expect(reader.text).toContain( - "last_crawled_at > $2::timestamptz - interval '7 days'", - ); - // Text stored and not indexed yet stays due. - expect(reader.text).toContain( - 'c.content_hash IS DISTINCT FROM website_urls.content_hash', - ); - } - }); - - it('records no date on a scan a person started, so every page is requested [KNOW-R16]', async () => { - const { sql, statements } = corpus(); - vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); - - await scanWebsiteImpl(engineCtx(), { ...SCAN, full: true }); - - expect(forgotten(statements).map((statement) => statement.params)).toEqual([ - [SCAN.domain, []], - ]); - expect(written(statements)).toEqual([]); - expect(logged()).toContain('12 URLs discovered'); - expect(logged()).not.toContain('not requested'); - }); - - // A later link runs no discovery: it reads the frontier under the dates - // the first link recorded. - it('reads the frontier under the same rule on a later link, and records nothing', async () => { - const { sql, statements } = corpus(); - vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); - - await scanWebsiteImpl(engineCtx(), { - ...SCAN, - continuation: 1, - scanStartedAt: '2026-10-06T13:00:00.000Z', - }); - - expect(safeFetch).not.toHaveBeenCalled(); - expect(forgotten(statements)).toEqual([]); - expect(written(statements)).toEqual([]); - expect(frontier(statements)).toHaveLength(2); - }); -}); diff --git a/services/platform/backend/core/knowledge/crawl_action.redirects.test.ts b/services/platform/backend/core/knowledge/crawl_action.redirects.test.ts index 2f601f2068..bc374df4d9 100644 --- a/services/platform/backend/core/knowledge/crawl_action.redirects.test.ts +++ b/services/platform/backend/core/knowledge/crawl_action.redirects.test.ts @@ -176,7 +176,7 @@ describe('fetchAndStorePage — a redirect inside the site', () => { await expect( fetchAndStorePage(sql, DOMAIN, page(from, listed), EMPTY_ROBOTS_POLICY), - ).resolves.toBe('render'); + ).resolves.toMatchObject({ kind: 'render' }); expect(retired(statements)).toEqual([]); expect(admitted(statements)).toEqual([]); @@ -221,7 +221,7 @@ describe('fetchAndStorePage — a page the crawler does not request', () => { await expect( fetchAndStorePage(sql, DOMAIN, page(url, true), PRIVATE_RULE), - ).resolves.toBe('render'); + ).resolves.toMatchObject({ kind: 'render' }); expect(safeFetchBinary).toHaveBeenCalledTimes(1); expect(retired(statements)).toEqual([]); @@ -258,7 +258,7 @@ describe('fetchAndStorePage — a page the crawler does not request', () => { await expect( fetchAndStorePage(sql, 'intranet.corp', page(url), EMPTY_ROBOTS_POLICY), - ).resolves.toBe('render'); + ).resolves.toMatchObject({ kind: 'render' }); expect(safeFetchBinary).toHaveBeenCalledWith( url, diff --git a/services/platform/backend/core/knowledge/crawl_action.ts b/services/platform/backend/core/knowledge/crawl_action.ts index 4690f75402..bc579f7db8 100644 --- a/services/platform/backend/core/knowledge/crawl_action.ts +++ b/services/platform/backend/core/knowledge/crawl_action.ts @@ -12,10 +12,13 @@ * in-process; HTML is rendered in batches by a sandboxed browser * (`renderUrlsInSandbox`) so JS-rendered sites yield their real content. * The in-process probe stays the authority on page lifecycle — status - * codes, deletes, size caps, SSRF guards — and content-hash comparison is - * the only change detection for a page that is requested. A page whose - * sitemap entry says it has not changed since it was last read is not - * requested at all (`UNCHANGED_BY_SITEMAP_SQL`). + * codes, deletes, size caps, SSRF guards — and it is also where a scan + * finds out, with that one request, whether a page changed at all: a + * server that answers 304 to the validators of the last visit, or plain + * HTML that reads the same as it did then, is a page left as it is. Only a + * page that did change — or one whose plain HTML does not carry what a + * browser shows — is rendered (`PageCheck`). Content-hash comparison then + * decides what a stored page means for the index. * * A scan is a CONTINUATION CHAIN, not one long action: a Convex node action * is hard-killed near ten minutes without running its catch, so each link @@ -52,8 +55,8 @@ import { normalizeCandidateUrl, paragraphsForHashing, parseRobots, - parseSitemapEntries, parseSitemapLocs, + plainTextCarriesRendered, publicPageError, renderLaneHaltMessage, ROBOTS_TXT_MAX_BYTES, @@ -100,6 +103,7 @@ import { orgSlugFromIdOrNull } from '../lib/helpers/org_slug'; import { extractText } from '../lib/knowledge/extraction/router'; import { sniffDocumentExtension } from '../lib/knowledge/extraction/sniff'; import { + RENDER_MAX_HTML_BYTES, RenderCapacityError, renderCapacityPollMs, renderUrlsInSandbox, @@ -112,12 +116,7 @@ import { } from '../websites/scan_scheduling'; import { type PageFailureKind, PAGE_SKIP_KINDS_SQL } from '../websites/types'; import { readOrgEmbeddingConfig } from './connection'; -import { - MAX_URLS_PER_DOMAIN, - admitUrls, - recordSitemapLastmods, - reviveListedUrls, -} from './crawl'; +import { MAX_URLS_PER_DOMAIN, admitUrls, reviveListedUrls } from './crawl'; import { crawlDocumentMaxBytes } from './crawl_limits'; import { CRAWLER_PRODUCT_TOKEN, @@ -220,19 +219,6 @@ const MAX_FETCH_FAILURES = 5; * still lets a page revived on the site come back on its own. */ const BENCHED_PAGE_RETRY_INTERVAL = '7 days'; -/** How long a page is left alone on its sitemap's word that it has not - * changed ({@link UNCHANGED_BY_SITEMAP_SQL}) — a Postgres interval. Past it - * the page is read again whatever its entry says: a listing page changes - * with every article it lists and keeps its date, and a site can publish - * dates it never moves. A week bounds how stale such a page gets, at one - * request a week. */ -const SITEMAP_LASTMOD_MAX_AGE = '7 days'; -/** How long after the change a sitemap records a read of the page still - * counts as too early — a Postgres interval. A cache in front of the site - * can go on serving the earlier text for a while after the edit, and a read - * that met it would otherwise stand as the current one for a week. */ -const SITEMAP_LASTMOD_SETTLE = '1 hour'; - /** Pages the vector backfill reads per query (`PageIndexer.embedVectorless`). */ const EMBED_BATCH_PAGES = 20; @@ -273,13 +259,6 @@ export async function scanWebsiteImpl( * takes exactly that claim over instead of waiting out * {@link STUCK_SCAN_TAKEOVER}; a claim that moved since is left alone. */ takeover?: string; - /** Set on the first link of a scan a person started with "Scan now": - * every page is requested, whatever the site's sitemap says about when - * it last changed. Any other scan leaves an unchanged page alone - * ({@link UNCHANGED_BY_SITEMAP_SQL}) — a scheduled one, and the one the - * scheduler resumes after a restart, which continues as an ordinary - * scan whatever it began as. */ - full?: boolean; /** Aborted once the job this link runs in has ended under it — the * process is stopping, or the link outlived the job's expiry. */ signal?: AbortSignal; @@ -373,7 +352,6 @@ export async function scanWebsiteImpl( actionStartedAt + DISCOVERY_BUDGET_MS, robots, scanStartedAt, - { full: args.full === true }, ); const retired = await retireDisallowedRows(sql, args.domain, policy); if (retired > 0) { @@ -400,8 +378,12 @@ export async function scanWebsiteImpl( const hardWall = actionStartedAt + ACTION_HARD_WALL_MS; const deadline = Math.min(linkStartedAt + SCAN_BUDGET_MS, hardWall); const indexer = new PageIndexer(ctx, sql, identity); - const renderQueue: DuePage[] = []; + const renderQueue: { page: DuePage; probe: PageProbe }[] = []; let renderBatchCounter = 0; + // What this link's requests found, for the scan's log: how many pages + // it asked for, how many of them had not changed, how many it had to + // open a browser for. + const seen = { requested: 0, unchanged: 0, rendered: 0 }; // Set once the organization's render sessions stayed spent for the // rest of this link's window: the link stops fetching (every HTML // page it probed would only queue behind the same wait) and hands the @@ -456,7 +438,7 @@ export async function scanWebsiteImpl( } const batch = renderQueue.splice(0); renderBatchCounter += 1; - const results = await renderBatch(batch); + const results = await renderBatch(batch.map((entry) => entry.page)); if (results === null) { // The batch's rows are unmarked and stay due; the next link // renders them once a session is free. @@ -470,7 +452,7 @@ export async function scanWebsiteImpl( // A navigation the browser lost twice is the row's to show, without // a strike; the lane's own fault (`halted`) fails the scan below, // once what rendered is stored. - for (const page of batch) { + for (const { page, probe } of batch) { const outcome = results.outcomes.get(page.url) ?? { kind: 'not_attempted' as const, }; @@ -501,15 +483,25 @@ export async function scanWebsiteImpl( }); continue; } + const renderedText = htmlToText(outcome.html); const stored = await storePageText( sql, args.domain, page, htmlTitle(outcome.html), - htmlToText(outcome.html), + renderedText, ); await indexer.settle(page.url, stored); - await markPageCrawled(sql, args.domain, page.url); + // Whether the next scan can judge this page by one request is + // decided here, from the two texts of this visit: what the plain + // HTML said and what the browser showed. + await markPageCrawled( + sql, + args.domain, + page.url, + checkAfterRender(probe, renderedText), + ); + seen.rendered += 1; if (kind === 'site') { await admitRenderedLinks( sql, @@ -549,11 +541,22 @@ export async function scanWebsiteImpl( policy, scanStartedAt, ); - if (outcome === 'render') renderQueue.push(page); - else if (outcome !== 'failed') { - await indexer.settle(page.url, outcome); - await markPageCrawled(sql, args.domain, page.url); + seen.requested += 1; + if (outcome === 'not_modified') { + seen.unchanged += 1; + } else if (typeof outcome === 'object') { + if (outcome.kind === 'render') { + renderQueue.push({ page, probe: outcome.probe }); + } else { + // A document without validators is downloaded to be judged: + // the same text is the same document. + if (outcome.outcome === 'unchanged') seen.unchanged += 1; + await indexer.settle(page.url, outcome.outcome); + await markPageCrawled(sql, args.domain, page.url, outcome.check); + } } + // Any other outcome wrote its own row: a retired page, a failed + // attempt. await sleep(fetchDelayMs(policy)); if (renderQueue.length >= RENDER_BATCH_SIZE) await flushRenderBatch(); } @@ -565,6 +568,11 @@ export async function scanWebsiteImpl( // of at the link's end (2026-09-14 evaluation, h5). await fanOutRowSync(ctx, sql, args.domain); } + if (seen.requested > 0) { + console.log( + `[crawl] ${args.domain}: ${seen.requested} page(s) requested, ${seen.unchanged} unchanged, ${seen.rendered} rendered`, + ); + } // What was stored without vectors — by this scan before an admin saved // a model, or by an earlier one — is embedded from its stored text // once a model can, in the time the link has left; what is left keeps @@ -948,20 +956,13 @@ async function claimScan( * as the fallback) under the robots rules the scan read, and record them as * `discovered` rows for the fetch loop. `deadline` bounds the fetching: a * discovery cut short records what it has — the next scan's discovery pass - * tops the frontier up. - * - * It also records what the sitemaps say about each page's last change, which - * is what lets the fetch loop leave an unchanged page alone - * ({@link UNCHANGED_BY_SITEMAP_SQL}). A `full` scan — one a person started - * with "Scan now" — records none, and so requests every page: that is how - * someone says the site changed, whatever its sitemap claims. */ + * tops the frontier up. */ async function discoverAndRecordUrls( sql: Sql, domain: string, deadline: number, robots: RobotsRules, scanStartedAt: string, - options: { full: boolean }, ): Promise<void> { const hosts = siteHosts(domain); const baseUrl = `https://${domain}/`; @@ -985,19 +986,12 @@ async function discoverAndRecordUrls( if (advertised.length > 0) sitemapCandidates = advertised; const urls = new Set<string>(); - // What the sitemaps say about the last change of each admitted page — the - // latest date when several entries name one page. - const lastmods = new Map<string, Date>(); - const admit = (candidate: string, lastmod: Date | null = null): boolean => { + const admit = (candidate: string): boolean => { const normalized = normalizeCandidateUrl(candidate, baseUrl, hosts); if (!normalized) return false; if (isUrlDisallowed(normalized, robots)) return false; if (urls.size >= MAX_URLS_PER_DOMAIN) return true; urls.add(normalized); - const known = lastmods.get(normalized); - if (lastmod !== null && (known === undefined || lastmod > known)) { - lastmods.set(normalized, lastmod); - } return urls.size >= MAX_URLS_PER_DOMAIN; }; admit(baseUrl); @@ -1063,8 +1057,8 @@ async function discoverAndRecordUrls( continue; } let capped = false; - for (const entry of parseSitemapEntries(xml)) { - capped = admit(entry.loc, entry.lastmod); + for (const loc of parseSitemapLocs(xml)) { + capped = admit(loc); if (capped) break; } if (capped) break; @@ -1134,19 +1128,7 @@ async function discoverAndRecordUrls( listed: false, retiredBefore: scanStartedAt, }); - // After the admission: a page this scan found first has a row to carry - // its date only now. - await recordSitemapLastmods(sql, domain, options.full ? new Map() : lastmods); - const unchanged = options.full - ? 0 - : await countUnchangedBySitemap(sql, domain, scanStartedAt); - console.log( - `[crawl] ${domain}: ${urls.size} URLs discovered${ - unchanged > 0 - ? `, ${unchanged} not requested (unchanged by their sitemap dates)` - : '' - }`, - ); + console.log(`[crawl] ${domain}: ${urls.size} URLs discovered`); } interface DuePage { @@ -1155,62 +1137,178 @@ interface DuePage { /** An operator-listed URL: the robots rules do not govern it, and a 404 * keeps its row. */ readonly listed: boolean; + /** What the page's last settled visit left to check it by + * ({@link PageCheck}); absent or null when it left nothing. */ + readonly etag?: string | null; + readonly last_modified?: string | null; + readonly probe_hash?: string | null; + /** False while a chunk of the page is cut from other text than the row + * stores — a scan stored the new text and stopped before indexing it. */ + readonly indexed?: boolean; } /** - * True for a page a scan leaves alone on its sitemap's word: the entry's - * `<lastmod>` says the page has not changed since the crawler read it. - * `$2` is the scan's start. It holds only when every part does: + * What one request can check a page by, kept from the response its stored + * text came from. * - * - the row carries the text of a read that succeeded (`active`, a hash, no - * error) — so `last_crawled_at` is when that text was read, not when an - * attempt failed; - * - that read came after the instant the date can mean, and - * {@link SITEMAP_LASTMOD_SETTLE} more; - * - it is younger than {@link SITEMAP_LASTMOD_MAX_AGE}; - * - no chunk of the page was cut from other text: a scan that stored the - * new text and stopped before indexing it (an embedding model that - * failed) leaves the page due until it is indexed. + * `etag` and `lastModified` are that response's validators: sent back as + * `If-None-Match` / `If-Modified-Since`, they let a server that tracks its + * pages answer 304 without sending the page. Many servers that build their + * pages on request send neither — the page has no modification time to + * give — so `probeHash` is the hash of the text read out of the plain, + * unrendered HTML: the same text again is the same page, whatever tokens + * and nonces the markup around it carries. * - * It is NULL, not true, for a page without a sitemap date — one no sitemap - * lists, one found by a link, one of a URL list (a list has no discovery to - * read a date in) — and for a page never read: every doubt requests the - * page, as every scan did before (2026-10-06: a site whose sitemap dated - * all 700 of its pages answered some 8,000 requests a scan, every six - * hours, for pages unchanged in months). The content hash stays the judge - * of what a request brings back. + * The three describe one response, and only a response whose text is + * stored AND indexed: they are written when a page settles + * ({@link markPageCrawled}) and cleared when its content is purged. A page + * whose plain HTML does not carry what a browser shows keeps none of them + * ({@link checkAfterRender}): a 304 on a JavaScript shell, or the same + * shell text, proves nothing about the content, which is why the render + * lane dropped conditional requests for every page when it arrived. That + * page is rendered on every scan; every other page is rendered only when + * its one request says it changed. */ -const UNCHANGED_BY_SITEMAP_SQL = `( - status = 'active' AND content_hash IS NOT NULL AND last_error IS NULL - AND last_crawled_at > sitemap_lastmod + interval '${SITEMAP_LASTMOD_SETTLE}' - AND last_crawled_at > $2::timestamptz - interval '${SITEMAP_LASTMOD_MAX_AGE}' - AND NOT EXISTS ( - SELECT 1 FROM ${PUBLIC_WEB_SCHEMA}.chunks c - WHERE c.domain = website_urls.domain AND c.url = website_urls.url - AND c.content_hash IS DISTINCT FROM website_urls.content_hash))`; +interface PageCheck { + readonly etag: string | null; + readonly lastModified: string | null; + readonly probeHash: string | null; +} + +const NO_PAGE_CHECK: PageCheck = { + etag: null, + lastModified: null, + probeHash: null, +}; + +/** What a row offers this scan to check its page by: nothing unless it + * holds the page's text and that text is indexed — "unchanged" would + * otherwise leave a page without text, or with chunks of its earlier text, + * exactly as it is. */ +function storedCheck(page: DuePage): PageCheck { + if (page.content_hash === null || page.indexed === false) { + return NO_PAGE_CHECK; + } + return { + etag: page.etag ?? null, + lastModified: page.last_modified ?? null, + probeHash: page.probe_hash ?? null, + }; +} + +/** A validator longer than this is not kept: it travels back in a request + * header on every later scan. */ +const VALIDATOR_MAX_CHARS = 512; + +function validatorOf(value: string | null): string | null { + const trimmed = value?.trim() ?? ''; + return trimmed === '' || trimmed.length > VALIDATOR_MAX_CHARS + ? null + : trimmed; +} + +/** The validators a response carries, to check its page by next time. */ +function responseValidators( + headers: Headers, +): Pick<PageCheck, 'etag' | 'lastModified'> { + return { + etag: validatorOf(headers.get('etag')), + lastModified: validatorOf(headers.get('last-modified')), + }; +} + +/** The conditional headers that ask "has it changed since?" — none when + * the row holds no validator. */ +function conditionalHeaders(check: PageCheck): Record<string, string> { + const headers: Record<string, string> = {}; + if (check.etag !== null) headers['If-None-Match'] = check.etag; + if (check.lastModified !== null) { + headers['If-Modified-Since'] = check.lastModified; + } + return headers; +} + +/** The plain HTML of a probed page, and the text and hash a later scan + * compares against. */ +interface PlainHtml { + readonly html: string; + readonly text: string; + readonly hash: string; +} + +/** What the probe of an HTML page hands the render batch: the text of its + * plain HTML (null when the page is too large to read twice) and the + * validators of the response, for {@link checkAfterRender} to judge once + * the browser has shown the page. */ +interface PageProbe { + readonly text: string | null; + readonly hash: string | null; + readonly etag: string | null; + readonly lastModified: string | null; +} + +/** Decode a page under the charset its `Content-Type` names — a page in + * another encoding read as UTF-8 comes out with every accented word + * broken, and would never match what the browser shows. UTF-8 without one. */ +function decodeHtml(bytes: Uint8Array, contentType: string): string { + const label = /charset\s*=\s*"?([^";\s]+)/i.exec(contentType)?.[1]; + if (label !== undefined && label.toLowerCase() !== 'utf-8') { + try { + return new TextDecoder(label).decode(bytes); + } catch (error) { + console.warn( + `[crawl] unknown charset "${label}", reading the page as UTF-8:`, + error instanceof Error ? error.message : error, + ); + } + } + return new TextDecoder().decode(bytes); +} + +/** Read the text out of a probed page's plain HTML — null for a page over + * the bound a rendered page has, which is rendered without the comparison. */ +async function readPlainHtml( + body: Blob, + contentType: string, +): Promise<PlainHtml | null> { + if (body.size > RENDER_MAX_HTML_BYTES) return null; + const html = decodeHtml( + new Uint8Array(await body.arrayBuffer()), + contentType, + ); + const text = htmlToText(html); + return { html, text, hash: computeContentHash(text) }; +} + +/** + * What a rendered page leaves to check it by next time: its probe's + * validators and plain-text hash when the plain HTML carried what the + * browser showed ({@link plainTextCarriesRendered}), nothing when it did + * not — that page is rendered again on every scan. Exported for tests only. + */ +export function checkAfterRender( + probe: PageProbe, + renderedText: string, +): PageCheck { + if ( + probe.text === null || + probe.hash === null || + !plainTextCarriesRendered(probe.text, renderedText) + ) { + return NO_PAGE_CHECK; + } + return { + etag: probe.etag, + lastModified: probe.lastModified, + probeHash: probe.hash, + }; +} const DUE_PAGE_PREDICATE = ` domain = $1 AND status <> 'deleted' AND (listed OR fail_count < ${MAX_FETCH_FAILURES} OR last_crawled_at < $2::timestamptz - interval '${BENCHED_PAGE_RETRY_INTERVAL}') - AND (last_crawled_at IS NULL OR last_crawled_at < $2::timestamptz) - AND NOT COALESCE(${UNCHANGED_BY_SITEMAP_SQL}, FALSE)`; - -/** How many of the domain's pages this scan leaves alone on their sitemap's - * word — what the scan's log says beside how many it discovered. */ -async function countUnchangedBySitemap( - sql: Sql, - domain: string, - scanStartedAt: string, -): Promise<number> { - const rows = await sql.unsafe<{ n: string }[]>( - `SELECT count(*)::text AS n FROM ${PUBLIC_WEB_SCHEMA}.website_urls - WHERE domain = $1 AND status <> 'deleted' - AND COALESCE(${UNCHANGED_BY_SITEMAP_SQL}, FALSE)`, - [domain, scanStartedAt], - ); - return Number(rows[0]?.n ?? 0); -} + AND (last_crawled_at IS NULL OR last_crawled_at < $2::timestamptz)`; /** The next URLs this scan has not visited yet (never-crawled first). The * batch size matches the render batch, so one claim's HTML pages fill at @@ -1222,7 +1320,13 @@ async function nextDuePages( limit: number, ): Promise<DuePage[]> { return await sql.unsafe<DuePage[]>( - `SELECT url, content_hash, listed + `SELECT url, content_hash, listed, etag, last_modified, probe_hash, + NOT EXISTS ( + SELECT 1 FROM ${PUBLIC_WEB_SCHEMA}.chunks c + WHERE c.domain = website_urls.domain + AND c.url = website_urls.url + AND c.content_hash IS DISTINCT FROM website_urls.content_hash + ) AS indexed FROM ${PUBLIC_WEB_SCHEMA}.website_urls WHERE ${DUE_PAGE_PREDICATE} ORDER BY last_crawled_at ASC NULLS FIRST, url ASC @@ -1244,7 +1348,30 @@ async function countDuePages( return Number(rows[0]?.n ?? 0); } -type FetchOutcome = StoreOutcome | 'failed' | 'render'; +/** + * What a probe made of a page: + * + * - `not_modified` — requested and found as it was: the server answered + * 304, or the plain HTML reads as it did. The row is stamped; nothing is + * rendered, stored or indexed. + * - `unchanged` — the row was retired (gone, disallowed, an alias) and + * nothing is left to do for it. + * - `failed` — the attempt is on the row, with its reason. + * - `stored` — a document or plain text was stored in-process; the scan + * indexes it and then stamps it with what to check it by next time. + * - `render` — an HTML page that changed, or that only a browser can + * read: it joins the render batch with what its probe found. + */ +export type FetchOutcome = + | 'not_modified' + | 'unchanged' + | 'failed' + | { + readonly kind: 'stored'; + readonly outcome: StoreOutcome; + readonly check: PageCheck; + } + | { readonly kind: 'render'; readonly probe: PageProbe }; /** Whether a redirect landed on another host or path of the site — not * merely on the same address with another query string. */ @@ -1257,14 +1384,22 @@ function isAnotherAddress(from: string, to: string): boolean { } /** - * Probe one page and dispatch on its content type: binaries and plain text - * are extracted and stored in-process; HTML reports `render` (body - * discarded, row left unmarked) so the caller batches it through the - * sandboxed browser. The probe is the sole authority on page LIFECYCLE — - * status codes, deletes, size caps, and the SSRF guard for every byte - * download. Change detection is the stored content hash alone: a 304 on an - * SPA shell proves nothing about rendered content, so no conditional - * validators are sent. Exported for tests only. + * Probe one page — the one request a scan makes for it — and say what is + * left to do ({@link FetchOutcome}). + * + * The request carries the validators of the page's last settled visit, when + * the row holds any ({@link storedCheck}); a 304 answer is a page left as + * it is. Otherwise the response is dispatched on its content type: + * binaries and plain text are extracted and stored in-process; an HTML + * page whose plain text reads as it did at that visit is left as it is + * too, and any other HTML page reports `render`, row unmarked, so the + * caller batches it through the sandboxed browser. + * + * The probe is the sole authority on page LIFECYCLE — status codes, + * deletes, size caps, and the SSRF guard for every byte download — and a + * page found unchanged passes the same gates first: a redirect that makes + * it an alias, an `X-Robots-Tag` or a robots meta tag that withdraws it. + * Exported for tests only. */ export async function fetchAndStorePage( sql: Sql, @@ -1284,6 +1419,8 @@ export async function fetchAndStorePage( return 'unchanged'; } + const check = storedCheck(page); + const conditional = conditionalHeaders(check); let response; try { assertCrawlableUrl(page.url); @@ -1293,7 +1430,7 @@ export async function fetchAndStorePage( allowedHosts: [...hosts], allowPrivateAddresses: privateCrawlHostsAllowed(), httpsOnly: true, - headers: crawlerRequestHeaders(), + headers: { ...crawlerRequestHeaders(), ...conditional }, }); } catch (error) { const cause = error instanceof Error ? error.message : String(error); @@ -1343,7 +1480,11 @@ export async function fetchAndStorePage( await retirePage(sql, domain, page.url); return 'unchanged'; } - if (response.status < 200 || response.status >= 300) { + // 304 answers the validators this request carried: the page is what the + // row stores. Without them it is an error like any other status. + const notModified = + response.status === 304 && Object.keys(conditional).length > 0; + if (!notModified && (response.status < 200 || response.status >= 300)) { await recordPageFailure(sql, domain, page.url, { kind: 'http_error', message: `The page answered HTTP ${response.status}${ @@ -1391,6 +1532,12 @@ export async function fetchAndStorePage( }); return 'failed'; } + if (notModified) { + // The server vouches for the page as the row stores it: no body came, + // and none is asked for. The validators the row holds stay. + await markPageUnchanged(sql, domain, page.url); + return 'not_modified'; + } const contentType = response.headers.get('content-type') ?? ''; let dispatch = classifyContentType( contentType, @@ -1411,10 +1558,48 @@ export async function fetchAndStorePage( return 'failed'; } if (dispatch.kind === 'html') { - // Content comes from the rendered DOM, not this probe body — the page - // joins the render batch and its row stays unmarked until the batch - // settles it. - return 'render'; + const plain = await readPlainHtml(response.body, contentType); + const validators = responseValidators(response.headers); + if ( + plain !== null && + check.probeHash !== null && + plain.hash === check.probeHash + ) { + // The plain HTML reads as it did when the page was last stored, and + // at that visit it carried what the browser showed: the page has not + // changed, and no browser is opened to learn that. Its links were + // admitted when it was last rendered, and text that reads the same + // links to the same pages. Its own wish can have changed without a + // word of its text moving, so the robots tag is still read — off + // this body, the only one there is. + const noindex = robotsMetaNoindexDirective(plain.html); + if (noindex !== null) { + await purgePageContent(sql, domain, page.url); + await recordPageFailure(sql, domain, page.url, { + kind: 'robots_noindex', + message: `The origin asked not to index this page (<meta name="robots" content="${noindex}">)`, + }); + return 'failed'; + } + await markPageUnchanged(sql, domain, page.url, { + ...validators, + probeHash: plain.hash, + }); + return 'not_modified'; + } + // The page changed, was never stored, or is one only a browser can + // read: content comes from the rendered DOM, so it joins the render + // batch and its row stays unmarked until the batch settles it. What + // this request found travels with it, to be judged against what the + // browser shows. + return { + kind: 'render', + probe: { + text: plain?.text ?? null, + hash: plain?.hash ?? null, + ...validators, + }, + }; } const bytes = new Uint8Array(await response.body.arrayBuffer()); if (dispatch.kind === 'sniff') { @@ -1470,7 +1655,13 @@ export async function fetchAndStorePage( } title = name; } - return await storePageText(sql, domain, page, title, text); + // A document's bytes are its content, so its validators always hold: a + // 304 next time spares the download and the extraction. + return { + kind: 'stored', + outcome: await storePageText(sql, domain, page, title, text), + check: { ...responseValidators(response.headers), probeHash: null }, + }; } /** @@ -1552,16 +1743,54 @@ export async function storePageText( return 'changed'; } -/** Stamp a page as visited by this scan, once it is stored and indexed. */ +/** Stamp a page as visited by this scan, once it is stored and indexed, and + * record what the next scan checks it by ({@link PageCheck}) — in the same + * write, so the check never describes text the index does not hold yet. */ async function markPageCrawled( sql: Sql, domain: string, url: string, + check: PageCheck, ): Promise<void> { await sql.unsafe( - `UPDATE ${PUBLIC_WEB_SCHEMA}.website_urls SET last_crawled_at = NOW() + `UPDATE ${PUBLIC_WEB_SCHEMA}.website_urls + SET last_crawled_at = NOW(), + etag = $3, last_modified = $4, probe_hash = $5 WHERE domain = $1 AND url = $2`, - [domain, url], + [domain, url, check.etag, check.lastModified, check.probeHash], + ); +} + +/** + * Stamp a page this scan requested and found as the row stores it. The + * request reached the page, so a failure an earlier scan recorded is over: + * its count and its reason are cleared, as a store clears them. `check` + * replaces what the row is checked by — the validators of a response whose + * text read the same; a 304 brings none and the row keeps its own. + */ +async function markPageUnchanged( + sql: Sql, + domain: string, + url: string, + check?: PageCheck, +): Promise<void> { + const cleared = `last_crawled_at = NOW(), fail_count = 0, + last_error = NULL, last_error_kind = NULL, last_error_at = NULL`; + if (check === undefined) { + await sql.unsafe( + `UPDATE ${PUBLIC_WEB_SCHEMA}.website_urls + SET ${cleared} + WHERE domain = $1 AND url = $2`, + [domain, url], + ); + return; + } + await sql.unsafe( + `UPDATE ${PUBLIC_WEB_SCHEMA}.website_urls + SET ${cleared}, + etag = $3, last_modified = $4, probe_hash = $5 + WHERE domain = $1 AND url = $2`, + [domain, url, check.etag, check.lastModified, check.probeHash], ); } @@ -1588,6 +1817,7 @@ async function purgePageContentIn( await tx.unsafe( `UPDATE ${PUBLIC_WEB_SCHEMA}.website_urls SET content = NULL, content_hash = NULL, word_count = 0, + etag = NULL, last_modified = NULL, probe_hash = NULL, status = CASE WHEN status = 'deleted' THEN status ELSE 'discovered' END WHERE domain = $1 AND url = $2`, [domain, url], diff --git a/services/platform/backend/core/node_only/sandbox/render_fetch.ts b/services/platform/backend/core/node_only/sandbox/render_fetch.ts index 84b69f4db6..83d4d2ab59 100644 --- a/services/platform/backend/core/node_only/sandbox/render_fetch.ts +++ b/services/platform/backend/core/node_only/sandbox/render_fetch.ts @@ -73,7 +73,7 @@ const WORKER_EXIT_MARGIN_MS = 20_000; * back fails the batch deterministically (the crawl would retry the same * batch forever), so the worker bounds the serialized file before it admits * each page. */ -const RENDER_MAX_HTML_BYTES = 6 * 1024 * 1024; +export const RENDER_MAX_HTML_BYTES = 6 * 1024 * 1024; const RENDER_MAX_TOTAL_BYTES = 15 * 1024 * 1024; export type RenderPageOutcome = diff --git a/services/platform/backend/domains/knowledge/spec.md b/services/platform/backend/domains/knowledge/spec.md index 2adc31f66d..6fd373e042 100644 --- a/services/platform/backend/domains/knowledge/spec.md +++ b/services/platform/backend/domains/knowledge/spec.md @@ -269,23 +269,22 @@ networks for the crawler. networks, and the operator then turns that off → the next scan requests none of its pages and shows each as failed, with a private address as the reason. -### KNOW-R16 · The crawler does not request again a page its sitemap says is unchanged - -A site's sitemap can state when each page last changed. A scheduled scan leaves a page alone -when the crawler read it after that date, and requests every other page: one without a date, -one whose date moved, one whose last read failed. A date without a time counts for its whole -day. A page is requested again once its last read is more than seven days old, whatever its -date says, because a sitemap can keep a date while the page changes. **Scan now** requests -every page. - -- **Example**: The sitemap of `docs.example` dates `/pricing` 1 March, and the crawler last - read the page on 5 March → the scan on 6 March does not request it. -- **Example**: The site changes `/pricing` and its sitemap dates the page 7 March → the next - scan requests it. -- **Example**: `/blog` lists every new article and keeps its date of 1 January → it is - requested again a week after its last read. -- **Example**: Noah selects **Scan now** → every page of `docs.example` is requested, - `/pricing` included. +### KNOW-R16 · The crawler opens a page in the browser again only when it changed + +A scan asks for each page once. The page is left as it is when the site's server answers +that it has not changed, or, for a server that gives no modification time, when the text of +the page reads as it did at the last visit. Only a page that changed is opened in the +browser and indexed again. A page whose content only its own JavaScript draws is the +exception: it is opened on every scan. **Scan now** asks the same way as a scheduled scan. + +- **Example**: `docs.example` answers `304 Not Modified` for `/pricing` → the page is not + downloaded, and no browser is opened for it. +- **Example**: `/blog` is built on every request, with a new token in each response around + the same text → the scan reads it once and leaves it as it is. +- **Example**: The site adds an article to `/blog` → the next scan finds other text, opens + the page in the browser and indexes it again. +- **Example**: Noah selects **Scan now** → every page is asked once, and only the changed + ones are opened in the browser. ## Not yet @@ -314,7 +313,7 @@ every page. `core/knowledge/crawl_limits.ts`, `lib/knowledge/crawl-parse.ts`). Of the limits, only the size limit for one document has a test, and that test holds the setting, not the refusal. The contract debt ledger in [`.agents/repo.md`](../../../../../.agents/repo.md) records that - the crawler's clocks and limits are not on the API, and that a page is fetched several times + the crawler's clocks and limits are not on the API, and that a changed page is fetched twice a scan. - **`KNOW-R15` covers a page's own address.** A public name that leads to a private address, a redirect into a private network, and the cloud provider addresses that the operator's diff --git a/services/platform/backend/domains/websites/service.embedding-follow.test.ts b/services/platform/backend/domains/websites/service.embedding-follow.test.ts index bb73a23d1a..8968a543e3 100644 --- a/services/platform/backend/domains/websites/service.embedding-follow.test.ts +++ b/services/platform/backend/domains/websites/service.embedding-follow.test.ts @@ -122,11 +122,6 @@ describe('websitesAfterEmbeddingChange', () => { expect.objectContaining({ domain: 'docs.example' }), expect.objectContaining({ domain: 'failed.example' }), ]); - // Nothing says these sites changed: the scans embed stored text and - // retry what failed, and ask for no unchanged page again. - expect(scanJobs()).not.toContainEqual( - expect.objectContaining({ full: true }), - ); // The organization-wide hint the readiness notice re-reads on. expect(hints(queries)).toContainEqual(['org-1', null, 'website', null]); }); diff --git a/services/platform/backend/domains/websites/service.scan-now.test.ts b/services/platform/backend/domains/websites/service.scan-now.test.ts index a73854194f..240178384f 100644 --- a/services/platform/backend/domains/websites/service.scan-now.test.ts +++ b/services/platform/backend/domains/websites/service.scan-now.test.ts @@ -38,12 +38,7 @@ import { } from '../../core/knowledge/crawl.ts'; import { scanWebsiteImpl } from '../../core/knowledge/crawl_action.ts'; import { addJobInTx } from '../../jobs/enqueue.ts'; -import { - resumeScanning, - runWebsitesScan, - scanWebsiteNow, - type WebsiteRow, -} from './service.ts'; +import { runWebsitesScan, scanWebsiteNow, type WebsiteRow } from './service.ts'; const row = (overrides: Partial<WebsiteRow> = {}): WebsiteRow => ({ id: 'w-1', @@ -123,12 +118,7 @@ describe('scanWebsiteNow [WEB-R8]', () => { await expect(scanWebsiteNow(sql, row())).resolves.toEqual({ queued: true }); expect(scanJobs()).toEqual([ - { - domain: 'example.com', - orgSlug: 'acme', - organizationId: 'org-1', - full: true, - }, + { domain: 'example.com', orgSlug: 'acme', organizationId: 'org-1' }, ]); const update = queries.find((q) => q.text.startsWith('UPDATE app.websites'), @@ -155,50 +145,6 @@ describe('scanWebsiteNow [WEB-R8]', () => { ); }); -/** - * A scheduled scan leaves alone the pages a site's sitemap says have not - * changed. "Scan now" is how a person says the site did change, so its scan - * asks for every page; a scan the platform starts for its own reasons does - * not. - */ -describe('a scan a person starts, and one the platform starts', () => { - const ordinary = { - domain: 'example.com', - orgSlug: 'acme', - organizationId: 'org-1', - }; - - it('Scan now asks for every page [KNOW-R16]', async () => { - await scanWebsiteNow(fakeSql([row()]).sql, row()); - - expect(scanJobs()).toEqual([{ ...ordinary, full: true }]); - }); - - it('a scan the platform starts itself stays an ordinary one', async () => { - await scanWebsiteNow(fakeSql([row()]).sql, row(), { full: false }); - - expect(scanJobs()).toEqual([ordinary]); - }); - - // Resuming answers whether scanning works again, not a change of the site. - it('Resume scanning queues an ordinary scan', async () => { - await resumeScanning(fakeSql([row()]).sql, row()); - - expect(scanJobs()).toEqual([ordinary]); - }); - - it('the first link hands the word on to the engine', async () => { - vi.mocked(isMemberDomain).mockResolvedValue(true); - - await runWebsitesScan(fakeSql([row()]).sql, { ...ordinary, full: true }); - - expect(scanWebsiteImpl).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ ...ordinary, full: true }), - ); - }); -}); - describe('runWebsitesScan — a registration that never landed', () => { const payload = { domain: 'example.com', diff --git a/services/platform/backend/domains/websites/service.ts b/services/platform/backend/domains/websites/service.ts index eee6ec8cec..6f58d67e09 100644 --- a/services/platform/backend/domains/websites/service.ts +++ b/services/platform/backend/domains/websites/service.ts @@ -1303,9 +1303,7 @@ export async function websitesAfterEmbeddingChange( for (const website of rows) { if (scanPausedAt(website.metadata ?? undefined) !== null) continue; if (website.status !== 'error' && !vectorless.has(website.domain)) continue; - if ((await scanWebsiteNow(sql, website, { full: false })).queued) { - queued += 1; - } + if ((await scanWebsiteNow(sql, website)).queued) queued += 1; } return { queued }; } @@ -1349,15 +1347,8 @@ export async function deregisterAndDeleteWebsite( * Put a site back on the crawl now: the row reads `scanning`, its failure * bookkeeping is cleared and a scan is queued — in one transaction, so a row * never reads `scanning` without the job that will scan it. - * - * A `full` scan requests every page; any other leaves alone the pages the - * site's sitemap says have not changed, as a scheduled scan does. */ -async function queueScan( - sql: Sql, - website: WebsiteRow, - options: { full: boolean }, -): Promise<void> { +async function queueScan(sql: Sql, website: WebsiteRow): Promise<void> { const orgSlug = await requireSlug(sql, website.organizationId); await sql.begin(async (tx) => { await patchWebsite(tx, { @@ -1378,19 +1369,16 @@ async function queueScan( domain: website.domain, orgSlug, organizationId: website.organizationId, - ...(options.full ? { full: true } : {}), }); }); } -/** Resume paused scans and kick a verification scan now (the 0.4 twin). The - * scan answers whether scanning works again, not a change of the site, so - * it is an ordinary one. */ +/** Resume paused scans and kick a verification scan now (the 0.4 twin). */ export async function resumeScanning( sql: Sql, website: WebsiteRow, ): Promise<void> { - await queueScan(sql, website, { full: false }); + await queueScan(sql, website); } /** @@ -1400,24 +1388,15 @@ export async function resumeScanning( * already scanning, or being deleted, is left as it is — the corpus claim * would turn a second scan away anyway — and the answer says nothing was * queued. - * - * On a person's word the scan is a `full` one and requests every page: - * "Scan now" is how someone says the site changed, and under the scheduled - * scan's rule it would read nothing of a site whose sitemap dates are - * behind its pages. A scan the platform starts for its own reasons — an - * embedding model was saved — passes `full: false`: nothing says the site - * changed, and what it is for (embedding stored text, retrying a failed - * scan) needs no page requested again. */ export async function scanWebsiteNow( sql: Sql, website: WebsiteRow, - options: { full: boolean } = { full: true }, ): Promise<{ queued: boolean }> { if (website.status === 'scanning' || website.status === 'deleting') { return { queued: false }; } - await queueScan(sql, website, options); + await queueScan(sql, website); return { queued: true }; } @@ -1685,7 +1664,6 @@ export async function runWebsitesScan( continuation?: number; scanStartedAt?: string; takeover?: string; - full?: boolean; }, job?: { signal?: AbortSignal; jobId?: string }, ): Promise<void> { diff --git a/services/platform/backend/domains/websites/spec.md b/services/platform/backend/domains/websites/spec.md index d7744e756c..ab1965c97f 100644 --- a/services/platform/backend/domains/websites/spec.md +++ b/services/platform/backend/domains/websites/spec.md @@ -103,6 +103,6 @@ can call it. Without either, the websites page says that chat cannot search the - **Websites over the REST API**: the limits of its lists and searches, and the shape of its answers (`rest/v1-websites.ts`). Its tests hold `WEB-R2` to `WEB-R4`. - **The length of the waiting time in `WEB-R6` and the number of pickups in `WEB-R7`.** -- **The crawler's clocks and limits are not on the API**, and a page is fetched several times +- **The crawler's clocks and limits are not on the API**, and a changed page is fetched twice a scan; the contract debt ledger in [`.agents/repo.md`](../../../../../.agents/repo.md) records both. diff --git a/services/platform/backend/integration-check.ts b/services/platform/backend/integration-check.ts index 22a7e8a9a6..f79e1d4885 100644 --- a/services/platform/backend/integration-check.ts +++ b/services/platform/backend/integration-check.ts @@ -34,12 +34,14 @@ import path from 'node:path'; import { serve } from '@hono/node-server'; import { transactSerializable } from '@tale/shared/db/serializable'; +import { computeContentHash } from '@tale/shared/utils/hashing'; import type { PgBoss } from 'pg-boss'; import type { Sql, TransactionSql } from 'postgres'; import * as XLSX from 'xlsx'; import { z } from 'zod'; import { robotsPolicyFromStored } from '../lib/knowledge/crawl-parse.ts'; +import { htmlToText } from '../lib/knowledge/html-to-text.ts'; import { setSafeFetchResolverForTests } from '../lib/net/safe-fetch.ts'; import { objectStorageConnectionFileSchema } from '../lib/shared/schemas/object_storage.ts'; import { createApp } from './app.ts'; @@ -33831,35 +33833,37 @@ async function checkWebsitesCrawl( // page, no robots.txt and no sitemap, so the scan that restores the // registration finishes on the homepage. const HEAL_DOMAIN = 'itest-heal.example'; - // A site whose sitemap dates its pages — the lastmod lanes (2026-10-06). - // Its own fixture, with a count of the requests each path answered, so a - // lane can tell a page a scan left alone from one it read again. Six - // sitemap URLs: under the link-walk threshold, so discovery also reads - // the homepage for links on every scan — the lanes judge the other pages. - const LASTMOD_DOMAIN = 'itest-lastmod.example'; - const OLD_LASTMOD = '2024-03-01'; - const lastmodDates = new Map<string, string | null>([ - ['/', OLD_LASTMOD], - ['/old.txt', OLD_LASTMOD], - ['/zoned.txt', '2024-03-01T10:00:00+01:00'], - ['/undated.txt', null], - ['/stale.txt', OLD_LASTMOD], - ['/moved.txt', OLD_LASTMOD], + // A site for the change-check lanes (2026-10-06). Its fake server answers + // a conditional request the way a real one does — 304 when the validator + // it is asked about still stands — and counts what it sent for each page: + // the page, or a 304 in its place. `/page.html` is the page of a server + // that gives no modification time: no validator, and a new token in every + // response, around text that does not change. + const CHECK_DOMAIN = 'itest-check.example'; + const checkPage = (word: string, version: string): string => + `Change-check fixture ${word} page ${version}. Enough words about the ${word} subsystem to survive the chunking thresholds of the pipeline.`; + const checkSite = new Map< + string, + { body: string; etag?: string; lastModified?: string } + >([ + ['/', { body: checkPage('home', 'v1') }], + ['/tagged.txt', { body: checkPage('tagged', 'v1'), etag: '"t1"' }], + [ + '/dated.txt', + { + body: checkPage('dated', 'v1'), + lastModified: 'Fri, 10 Jul 2026 09:11:16 GMT', + }, + ], + ['/plain.txt', { body: checkPage('plain', 'v1') }], ]); - const lastmodBodies = new Map<string, string>( - [...lastmodDates.keys()].map((route) => [ - route, - `Lastmod fixture page ${route} v1. Enough words about the sitemap dates of this page to survive the chunking thresholds of the pipeline.`, - ]), - ); - const lastmodHits = new Map<string, number>(); - const lastmodSitemap = (): string => - `<?xml version="1.0"?><urlset>${[...lastmodDates] - .map( - ([route, date]) => - `<url><loc>https://${LASTMOD_DOMAIN}${route}</loc>${date === null ? '' : `<lastmod>${date}</lastmod>`}</url>`, - ) - .join('')}</urlset>`; + const checkSitemapRoutes = [...checkSite.keys()]; + const checkSent = new Map<string, { bodies: number; notModified: number }>(); + let checkToken = 0; + const checkHtml = (): string => { + checkToken += 1; + return `<html><head><meta name="csrf-token" content="token-${checkToken}"><title>Check fixture

${checkPage('html', 'v1')}

`; + }; const FAKE_HOSTS = new Set([ DOMAIN, `www.${DOMAIN}`, @@ -33931,22 +33935,62 @@ async function checkWebsitesCrawl( }); } if ( - url.hostname === LASTMOD_DOMAIN || - url.hostname === `www.${LASTMOD_DOMAIN}` + url.hostname === CHECK_DOMAIN || + url.hostname === `www.${CHECK_DOMAIN}` ) { - lastmodHits.set(url.pathname, (lastmodHits.get(url.pathname) ?? 0) + 1); - const body = - url.pathname === '/robots.txt' - ? `User-agent: *\nSitemap: https://${LASTMOD_DOMAIN}/sitemap.xml\n` - : url.pathname === '/sitemap.xml' - ? lastmodSitemap() - : lastmodBodies.get(url.pathname); - if (body === undefined) return new Response('gone', { status: 404 }); - return new Response(body, { + const answer = (body: string, type: string): Response => + new Response(body, { + headers: { + 'content-type': type, + 'content-length': String(body.length), + }, + }); + if (url.pathname === '/robots.txt') { + return answer( + `User-agent: *\nSitemap: https://${CHECK_DOMAIN}/sitemap.xml\n`, + 'text/plain', + ); + } + if (url.pathname === '/sitemap.xml') { + return answer( + `${checkSitemapRoutes + .map( + (route) => + `https://${CHECK_DOMAIN}${route}`, + ) + .join('')}`, + 'application/xml', + ); + } + const sent = checkSent.get(url.pathname) ?? { bodies: 0, notModified: 0 }; + checkSent.set(url.pathname, sent); + if (url.pathname === '/page.html') { + sent.bodies += 1; + return answer(checkHtml(), 'text/html; charset=utf-8'); + } + const page = checkSite.get(url.pathname); + if (!page) return new Response('gone', { status: 404 }); + const validators = { + ...(page.etag === undefined ? {} : { etag: page.etag }), + ...(page.lastModified === undefined + ? {} + : { 'last-modified': page.lastModified }), + }; + const asked = new Headers(init?.headers); + if ( + (page.etag !== undefined && asked.get('if-none-match') === page.etag) || + (page.lastModified !== undefined && + asked.get('if-modified-since') === page.lastModified) + ) { + sent.notModified += 1; + return new Response(null, { status: 304, headers: validators }); + } + sent.bodies += 1; + return new Response(page.body, { headers: { - 'content-type': - url.pathname === '/sitemap.xml' ? 'application/xml' : 'text/plain', - 'content-length': String(body.length), + 'content-type': 'text/plain', + 'content-length': String(page.body.length), + ...validators, }, }); } @@ -35238,149 +35282,180 @@ async function checkWebsitesCrawl( await v1(`/websites/${robotsId}`, { method: 'DELETE' }); } - // 4h. A rescan leaves alone the pages the site's sitemap says have not - // changed (2026-10-06). The crawler read `` out of a sitemap - // and nothing else, so every scan requested every page again — a - // site that dated all its pages answered thousands of requests a - // scan for pages unchanged in months. A page is now requested when - // its entry carries no date, when the date moved past the last - // read, when the sitemap no longer states it, when the last read is - // older than a week, and on a scan a person started. - const lastmodCreated = z.looseObject({ id: z.string() }).safeParse( + // 4h. A scan asks for each page once, and a page that did not change is + // neither downloaded nor rendered again (2026-10-06). The crawler's + // one way to learn whether a page had changed was to do everything + // again: fetch it, render it with every asset a render loads, and + // compare content hashes afterwards — some 8,000 requests a scan on + // a 700-page site whose pages had not changed in months. The request + // now carries the validators of the last visit, a 304 ends it, and + // for a server that gives no modification time the text of the + // plain HTML is compared. "Scan now" asks the same way. + const checkCreated = z.looseObject({ id: z.string() }).safeParse( await ( await v1('/websites', { - body: { domain: LASTMOD_DOMAIN, scanInterval: '6h' }, + body: { domain: CHECK_DOMAIN, scanInterval: '6h' }, }) ).json(), ); - const lastmodId = lastmodCreated.success ? lastmodCreated.data.id : ''; + const checkId = checkCreated.success ? checkCreated.data.id : ''; await drainCrawlJobs(); - const lastmodUrl = (route: string) => `https://${LASTMOD_DOMAIN}${route}`; - const lastmodPages = [ - '/old.txt', - '/zoned.txt', - '/undated.txt', - '/stale.txt', - '/moved.txt', + const checkUrl = (route: string) => `https://${CHECK_DOMAIN}${route}`; + const checkRoutes = [ + '/tagged.txt', + '/dated.txt', + '/plain.txt', + '/page.html', ]; - /** How often each page was requested since `before`, as `route:n`. */ - const requestedSince = (before: Map): string => - lastmodPages - .map( - (route) => - `${route}:${(lastmodHits.get(route) ?? 0) - (before.get(route) ?? 0)}`, - ) + const checkSnapshot = () => + new Map([...checkSent].map(([route, sent]) => [route, { ...sent }])); + /** What the site sent for each page since `before`, as + * `route:bodies/304s`. */ + const sentSince = (before: ReturnType): string => + checkRoutes + .map((route) => { + const now = checkSent.get(route) ?? { bodies: 0, notModified: 0 }; + const then = before.get(route) ?? { bodies: 0, notModified: 0 }; + return `${route}:${now.bodies - then.bodies}/${now.notModified - then.notModified}`; + }) .join(' '); - const lastmodRescan = async (): Promise => { - const before = new Map(lastmodHits); + /** Every page reads as last visited two days ago, so a fresh stamp is + * this scan's. */ + const checkRescan = async (): Promise => { + await pool` + UPDATE public_web.website_urls + SET last_crawled_at = now() - interval '2 days' + WHERE domain = ${CHECK_DOMAIN} + `; + const before = checkSnapshot(); await websites.runWebsitesScan(sql, { - domain: LASTMOD_DOMAIN, + domain: CHECK_DOMAIN, orgSlug, organizationId: orgId, }); await drainCrawlJobs(); - return requestedSince(before); + return sentSince(before); }; - /** The date each page's row carries, as `route:instant`. */ - const lastmodStored = async (): Promise => + const checkRow = async (route: string) => ( - await pool<{ url: string; lastmod: string | null }[]>` - SELECT url, - to_char(sitemap_lastmod AT TIME ZONE 'UTC', - 'YYYY-MM-DD"T"HH24:MI"Z"') AS lastmod - FROM public_web.website_urls - WHERE domain = ${LASTMOD_DOMAIN} AND url <> ${lastmodUrl('/')} - ORDER BY url + await pool< + { + status: string; + etag: string | null; + lastModified: string | null; + probeHash: string | null; + visited: boolean; + chunks: string; + text: string | null; + }[] + >` + SELECT u.status, u.etag, u.last_modified AS "lastModified", + u.probe_hash AS "probeHash", + u.last_crawled_at > now() - interval '1 hour' AS visited, + (SELECT count(*)::text FROM public_web.chunks c + WHERE c.domain = u.domain AND c.url = u.url) AS chunks, + (SELECT string_agg(c.chunk_content, ' ') FROM public_web.chunks c + WHERE c.domain = u.domain AND c.url = u.url) AS text + FROM public_web.website_urls u + WHERE u.domain = ${CHECK_DOMAIN} AND u.url = ${checkUrl(route)} ` - ) - .map( - (row) => - `${row.url.replace(`https://${LASTMOD_DOMAIN}`, '')}:${row.lastmod ?? 'none'}`, - ) - .join(' '); - const lastmodFirst = await pool<{ active: string }[]>` - SELECT count(*) FILTER (WHERE status = 'active')::text AS active - FROM public_web.website_urls WHERE domain = ${LASTMOD_DOMAIN} - `; + )[0]; - // The first rescan: every dated page was read after its date, so none - // is requested; the page without a date is. - const lastmodSecond = await lastmodRescan(); - const lastmodDatesStored = await lastmodStored(); + // The first scan stored every page and kept what each response gave to + // check it by. The rescan: the two pages the server vouches for are + // not sent again; the one without a validator is. + const taggedFirst = await checkRow('/tagged.txt'); + const datedFirst = await checkRow('/dated.txt'); + const checkSecond = await checkRescan(); + const taggedSecond = await checkRow('/tagged.txt'); + const datedSecond = await checkRow('/dated.txt'); record( - 'websites lastmod: a rescan requests no page its sitemap dates as unchanged, and the page without a date', - lastmodCreated.success && - lastmodFirst[0]?.active === '6' && - lastmodSecond === - '/old.txt:0 /zoned.txt:0 /undated.txt:1 /stale.txt:0 /moved.txt:0' && - // A date alone covers its whole day in any time zone; a time with a - // zone is that instant. - lastmodDatesStored === - '/moved.txt:2024-03-02T12:00Z /old.txt:2024-03-02T12:00Z /stale.txt:2024-03-02T12:00Z /undated.txt:none /zoned.txt:2024-03-01T09:00Z', - `created=${lastmodCreated.success} firstScanActive=${lastmodFirst[0]?.active ?? '?'}/6, rescan requested [${lastmodSecond}] (want old 0, zoned 0, undated 1, stale 0, moved 0), stored [${lastmodDatesStored}]`, - ); - - // A date that moved, a date the sitemap dropped, and a read older than - // a week: each is requested again, and the changed page is re-indexed. - lastmodDates.set('/moved.txt', new Date().toISOString()); - lastmodBodies.set( - '/moved.txt', - 'Lastmod fixture page /moved.txt v2. Rewritten words about the sitemap dates of this page so its content hash flips and its chunks are cut again.', - ); - lastmodDates.set('/zoned.txt', null); + 'websites change check: a page the server vouches for is not downloaded again, and is stamped as visited', + checkCreated.success && + taggedFirst?.etag === '"t1"' && + datedFirst?.lastModified === 'Fri, 10 Jul 2026 09:11:16 GMT' && + checkSecond === + '/tagged.txt:0/1 /dated.txt:0/1 /plain.txt:1/0 /page.html:0/0' && + (taggedSecond?.visited ?? false) && + (datedSecond?.visited ?? false) && + taggedSecond?.status === 'active' && + taggedSecond.chunks === taggedFirst.chunks && + Number(taggedSecond.chunks) >= 1, + `created=${checkCreated.success} first scan kept etag=${taggedFirst?.etag ?? 'none'}/"t1" lastModified=${datedFirst?.lastModified ?? 'none'}, rescan sent bodies/304s [${checkSecond}] (want tagged 0/1, dated 0/1, plain 1/0), stamped tagged=${taggedSecond?.visited ?? '?'} dated=${datedSecond?.visited ?? '?'}, chunks ${taggedFirst?.chunks ?? '?'}→${taggedSecond?.chunks ?? '?'}`, + ); + + // A page that changed is sent and indexed again. And an HTML page of a + // server without validators — a new token in every response, the same + // text — is found unchanged by that text: the scan ends without a + // render session, which this harness could not open. + checkSite.set('/tagged.txt', { + body: checkPage('tagged', 'v2'), + etag: '"t2"', + }); + const htmlText = htmlToText(checkHtml()); + const htmlHash = computeContentHash(htmlText); await pool` - UPDATE public_web.website_urls - SET last_crawled_at = now() - interval '8 days' - WHERE domain = ${LASTMOD_DOMAIN} AND url = ${lastmodUrl('/stale.txt')} + INSERT INTO public_web.website_urls + (domain, url, status, discovered_at, listed, last_crawled_at, + word_count, content, content_hash, probe_hash) + VALUES (${CHECK_DOMAIN}, ${checkUrl('/page.html')}, 'active', NOW(), FALSE, + NOW() - INTERVAL '2 days', 20, ${htmlText}, ${htmlHash}, ${htmlHash}) + ON CONFLICT (domain, url) DO NOTHING `; - const lastmodThird = await lastmodRescan(); - const movedChunks = await pool<{ content: string }[]>` - SELECT chunk_content AS content FROM public_web.chunks - WHERE domain = ${LASTMOD_DOMAIN} AND url = ${lastmodUrl('/moved.txt')} + await pool` + INSERT INTO public_web.chunks + (domain, url, title, content_hash, chunk_index, chunk_content) + VALUES (${CHECK_DOMAIN}, ${checkUrl('/page.html')}, 'Check fixture', ${htmlHash}, 0, ${htmlText}) + ON CONFLICT DO NOTHING `; - const staleRead = await pool<{ recent: boolean }[]>` - SELECT last_crawled_at > now() - interval '1 hour' AS recent - FROM public_web.website_urls - WHERE domain = ${LASTMOD_DOMAIN} AND url = ${lastmodUrl('/stale.txt')} + checkSitemapRoutes.push('/page.html'); + const checkThird = await checkRescan(); + const taggedThird = await checkRow('/tagged.txt'); + const htmlThird = await checkRow('/page.html'); + const checkCorpus = await pool<{ status: string; error: string | null }[]>` + SELECT status, error FROM public_web.websites WHERE domain = ${CHECK_DOMAIN} `; record( - 'websites lastmod: a moved date, a date the sitemap dropped and a read older than a week are requested again', - lastmodThird === - '/old.txt:0 /zoned.txt:1 /undated.txt:1 /stale.txt:1 /moved.txt:1' && - movedChunks.length >= 1 && - movedChunks.every((chunk) => chunk.content.includes('v2')) && - (staleRead[0]?.recent ?? false), - `rescan requested [${lastmodThird}] (want old 0, zoned 1, undated 1, stale 1, moved 1), movedChunks=${movedChunks.length} allV2=${movedChunks.every((chunk) => chunk.content.includes('v2'))}, staleReadNow=${staleRead[0]?.recent ?? '?'}`, - ); - - // "Scan now" is a person saying the site changed: every page is - // requested, and the scan records no date. The scheduled scan after it - // honours the dates again — and requests the page whose date moved - // within the hour once more, as a cache in front of the site may still - // have served its earlier text. - const beforeScanNow = new Map(lastmodHits); + 'websites change check: a changed page is downloaded and indexed again, and an HTML page whose text reads the same ends the scan without a browser', + checkThird === + '/tagged.txt:1/0 /dated.txt:0/1 /plain.txt:1/0 /page.html:1/0' && + taggedThird?.etag === '"t2"' && + (taggedThird.text ?? '').includes('v2') && + !(taggedThird.text ?? '').includes('v1') && + htmlThird?.status === 'active' && + htmlThird.visited && + htmlThird.probeHash === htmlHash && + htmlThird.chunks === '1' && + checkCorpus[0]?.status === 'completed' && + checkCorpus[0].error === null, + `rescan sent bodies/304s [${checkThird}] (want tagged 1/0, dated 0/1, plain 1/0, html 1/0), tagged etag=${taggedThird?.etag ?? 'none'}/"t2" reindexed=${(taggedThird?.text ?? '').includes('v2')}, html=${htmlThird?.status ?? 'MISSING'}/active stamped=${htmlThird?.visited ?? '?'} hashKept=${htmlThird?.probeHash === htmlHash} chunks=${htmlThird?.chunks ?? '?'}/1, scan=${checkCorpus[0]?.status ?? '?'}/completed error=${checkCorpus[0]?.error ?? 'null'}`, + ); + + // "Scan now" is the same scan, started by a person: it asks each page + // the same way and is answered the same way. + await pool` + UPDATE public_web.website_urls + SET last_crawled_at = now() - interval '2 days' + WHERE domain = ${CHECK_DOMAIN} + `; + const beforeScanNow = checkSnapshot(); const scanNow = z .looseObject({ queued: z.boolean() }) - .safeParse(await (await post(`/${lastmodId}/scan`, {})).json()); + .safeParse(await (await post(`/${checkId}/scan`, {})).json()); await drainCrawlJobs(); - const lastmodFull = requestedSince(beforeScanNow); - const lastmodAfterFull = await lastmodStored(); - const lastmodFifth = await lastmodRescan(); - const lastmodAfterFifth = await lastmodStored(); + const checkScanNow = sentSince(beforeScanNow); + const htmlScanNow = await checkRow('/page.html'); record( - 'websites lastmod: Scan now requests every page, and the next scheduled scan honours the dates again', + 'websites change check: Scan now asks each page the same way as a scheduled scan', scanNow.success && scanNow.data.queued && - lastmodFull === - '/old.txt:1 /zoned.txt:1 /undated.txt:1 /stale.txt:1 /moved.txt:1' && - !lastmodAfterFull.includes('Z') && - lastmodFifth === - '/old.txt:0 /zoned.txt:1 /undated.txt:1 /stale.txt:0 /moved.txt:1' && - lastmodAfterFifth.includes('/old.txt:2024-03-02T12:00Z'), - `scanNow queued=${scanNow.success ? scanNow.data.queued : 'BAD SHAPE'} requested [${lastmodFull}] (want every page 1) stored [${lastmodAfterFull}] (want none), next scan requested [${lastmodFifth}] (want old 0, zoned 1, undated 1, stale 0, moved 1) stored [${lastmodAfterFifth}]`, - ); - if (lastmodId !== '') { - await v1(`/websites/${lastmodId}`, { method: 'DELETE' }); + checkScanNow === + '/tagged.txt:0/1 /dated.txt:0/1 /plain.txt:1/0 /page.html:1/0' && + (htmlScanNow?.visited ?? false), + `scanNow queued=${scanNow.success ? scanNow.data.queued : 'BAD SHAPE'} sent bodies/304s [${checkScanNow}] (want tagged 0/1, dated 0/1, plain 1/0, html 1/0) htmlStamped=${htmlScanNow?.visited ?? '?'}`, + ); + if (checkId !== '') { + await v1(`/websites/${checkId}`, { method: 'DELETE' }); } // 4f. Round g, g4-5: a site whose every page failed is not "a successful diff --git a/services/platform/backend/jobs/task-list.ts b/services/platform/backend/jobs/task-list.ts index 901f65e687..7d645ee3cc 100644 --- a/services/platform/backend/jobs/task-list.ts +++ b/services/platform/backend/jobs/task-list.ts @@ -1183,7 +1183,6 @@ export function createTaskList(deps: TaskDeps): BackendTaskList { continuation: z.number().int().min(0).optional(), scanStartedAt: z.string().optional(), takeover: z.string().min(1).optional(), - full: z.boolean().optional(), }) .parse(payload); await runWebsitesScan(deps.sql, input, context); diff --git a/services/platform/backend/jobs/tasks.ts b/services/platform/backend/jobs/tasks.ts index 53b3f70676..a6a97315ac 100644 --- a/services/platform/backend/jobs/tasks.ts +++ b/services/platform/backend/jobs/tasks.ts @@ -331,10 +331,7 @@ export interface TaskPayloads { /** One continuation link of a domain scan — the reused engine body * self-chains through this queue; the corpus-side claim is the fence. * `takeover` is set by the scheduler alone, on the first link of a scan - * it resumes: the heartbeat of the claim whose scan stopped. `full` is - * set on the first link of a scan a person started: it requests every - * page, where a scheduled scan leaves alone the pages the site's sitemap - * says have not changed. */ + * it resumes: the heartbeat of the claim whose scan stopped. */ 'websites.scan': { domain: string; orgSlug: string; @@ -342,7 +339,6 @@ export interface TaskPayloads { continuation?: number; scanStartedAt?: string; takeover?: string; - full?: boolean; }; /** Register a website (or URL list) in the corpus + kick its first scan * (the 0.4 `registerAndSync`, fire-and-forget behind the create). */ diff --git a/services/platform/lib/knowledge/crawl-parse.test.ts b/services/platform/lib/knowledge/crawl-parse.test.ts index 19b1c9f2ce..2b024a9f43 100644 --- a/services/platform/lib/knowledge/crawl-parse.test.ts +++ b/services/platform/lib/knowledge/crawl-parse.test.ts @@ -26,11 +26,11 @@ import { normalizeListedUrl, paragraphsForHashing, parseRobots, - parseSitemapEntries, parseSitemapLocs, + plainTextCarriesRendered, + plainTextCoverage, robotsHeaderForbidsIndexing, siteHosts, - sitemapLastmodInstant, stripBoilerplate, robotsSitemapsFromStored, } from './crawl-parse'; @@ -255,134 +255,6 @@ describe('sitemap parsing', () => { }); }); -/** - * What a sitemap says about a page's last change decides whether a rescan - * requests the page at all, so the value is read toward its LATEST meaning: - * a scan that read the page before that instant cannot know it has the - * current text. The crawler read `` alone and requested every page on - * every scan, however long its sitemap said it had not changed. - */ -describe('sitemapLastmodInstant', () => { - const instant = (value: string): string | null => - sitemapLastmodInstant(value)?.toISOString() ?? null; - - it('reads a time with a zone as that instant', () => { - expect(instant('2026-10-06T14:30:00Z')).toBe('2026-10-06T14:30:00.000Z'); - expect(instant('2026-10-06T14:30:00+02:00')).toBe( - '2026-10-06T12:30:00.000Z', - ); - expect(instant('2026-10-06T14:30:00-0700')).toBe( - '2026-10-06T21:30:00.000Z', - ); - expect(instant('2026-10-06T14:30+02:00')).toBe('2026-10-06T12:30:00.000Z'); - expect(instant(' 2026-10-06T14:30:00.123456+00:00 ')).toBe( - '2026-10-06T14:30:00.000Z', - ); - }); - - // A date alone keeps its value through every edit of that day: the page - // is unchanged only once the day is over everywhere. - it('reads a date alone as the end of that day in the last time zone', () => { - expect(instant('2026-10-06')).toBe('2026-10-07T12:00:00.000Z'); - expect(instant('2024-02-29')).toBe('2024-03-01T12:00:00.000Z'); - expect(instant('2026-12-31')).toBe('2027-01-01T12:00:00.000Z'); - }); - - it('reads a time without a zone in the zone furthest behind UTC', () => { - expect(instant('2026-10-06T14:30:00')).toBe('2026-10-07T02:30:00.000Z'); - expect(instant('2026-10-06 14:30:00')).toBe('2026-10-07T02:30:00.000Z'); - }); - - it.each([ - ['a year alone', '2026'], - ['a month alone', '2026-10'], - ['a month that does not exist', '2026-13-01'], - ['a day that does not exist', '2026-02-30'], - ['an hour that does not exist', '2026-10-06T25:00:00Z'], - ['a zone that does not exist', '2026-10-06T14:30:00+19:00'], - ['another date format', '06.10.2026'], - ['an HTTP date', 'Tue, 06 Oct 2026 14:30:00 GMT'], - ['words', 'yesterday'], - ['nothing', ''], - ])('reads nothing out of %s', (_case, value) => { - expect(sitemapLastmodInstant(value)).toBeNull(); - }); -}); - -describe('parseSitemapEntries', () => { - const read = (xml: string) => - parseSitemapEntries(xml).map((entry) => [ - entry.loc, - entry.lastmod?.toISOString() ?? null, - ]); - - it("gives each page its own entry's date", () => { - const xml = ` - - - https://a.example - 2026-07-27 - monthly - - - - 2026-10-06T08:00:00+02:00 - https://a.example/en - - https://a.example/undated - - `; - expect(read(xml)).toEqual([ - ['https://a.example', '2026-07-28T12:00:00.000Z'], - ['https://a.example/en', '2026-10-06T06:00:00.000Z'], - ['https://a.example/undated', null], - ['https://a.example/cdata', '2026-01-03T12:00:00.000Z'], - ]); - }); - - it('leaves a page without a date when its entry carries one it cannot read', () => { - expect( - read( - 'https://a.example/xlast week', - ), - ).toEqual([['https://a.example/x', null]]); - }); - - // An image extension lists the image under its own tag, with its own - // address: neither is the page's. - it('reads the page address, not an image listed in its entry', () => { - expect( - read( - 'https://a.example/xhttps://a.example/x.png2026-01-02', - ), - ).toEqual([['https://a.example/x', '2026-01-03T12:00:00.000Z']]); - }); - - it.each([ - [ - 'a sitemap index', - 'https://a.example/pages.xml2026-01-02', - ], - [ - 'addresses outside any entry', - 'https://a.example/pages.xml2026-01-02', - ], - ])('lists every address of %s, without a date', (_case, xml) => { - expect(read(xml)).toEqual([['https://a.example/pages.xml', null]]); - }); - - it('lists the same addresses, in the same order, as the plain reader', () => { - const xml = `https://a.example/before - https://a.example/12026-01-02 - https://a.example/between - https://a.example/2?a=1&b=2 - https://a.example/after`; - expect(parseSitemapEntries(xml).map((entry) => entry.loc)).toEqual( - parseSitemapLocs(xml), - ); - }); -}); - describe('extractLinks', () => { it('reads single- and double-quoted hrefs', () => { expect( @@ -613,6 +485,55 @@ describe('boilerplate', () => { }); }); +/** + * One plain request can tell a scan whether a page changed only when the + * plain HTML carries what a browser shows. The crawler used to render every + * page to find that out — the browser, and every image, script and + * stylesheet the render loads, for pages that had not changed in months. + * The share is measured on the two texts of one visit. + */ +describe('plainTextCoverage', () => { + const article = + 'Tale indexes the public pages of the websites an organization added and answers questions from them.'; + + it('is whole for a page whose plain HTML already reads as the browser shows it', () => { + expect(plainTextCoverage(article, article)).toBe(1); + // Layout only moves the line breaks. + expect(plainTextCoverage(article, article.replaceAll(' ', '\n'))).toBe(1); + expect(plainTextCarriesRendered(article, article)).toBe(true); + }); + + it('counts what the scripts of a page add, and calls the page server-rendered while that stays a small part', () => { + const rendered = `${article} We use cookies.`; + // 16 of the 19 rendered words are in the plain HTML. + expect(plainTextCoverage(article, rendered)).toBeCloseTo(16 / 19, 5); + expect(plainTextCarriesRendered(article, rendered)).toBe(false); + const longer = `${article} ${article} ${article} We use cookies.`; + expect( + plainTextCarriesRendered(`${article} ${article} ${article}`, longer), + ).toBe(true); + }); + + it('is next to nothing for a page built by its JavaScript', () => { + const shell = 'Loading… You need to enable JavaScript to run this app.'; + const rendered = `${article} ${article} ${article}`; + expect(plainTextCoverage(shell, rendered)).toBeLessThan(0.1); + expect(plainTextCarriesRendered(shell, rendered)).toBe(false); + }); + + // A word the plain HTML holds once does not vouch for it shown five times. + it('counts a word as often as it occurs', () => { + expect(plainTextCoverage('alpha beta', 'alpha alpha alpha alpha')).toBe( + 0.25, + ); + }); + + it('is whole for a page that shows no text', () => { + expect(plainTextCoverage('anything', '')).toBe(1); + expect(plainTextCarriesRendered('', ' \n ')).toBe(true); + }); +}); + describe('metaDescription', () => { it('prefers name=description and falls back to og:description', () => { expect( @@ -1051,32 +972,6 @@ describe('scanning markup that never closes its tags', () => { 'CDATA openers inside one sitemap entry', () => parseSitemapLocs(`${'`), ], - [ - 'sitemap blocks with no end', - () => parseSitemapEntries(''.repeat(RUN)), - ], - [ - 'sitemap blocks whose only end is the last one', - () => parseSitemapEntries(`${'`), - ], - [ - 'sitemap dates with no end inside one block', - () => parseSitemapEntries(`${''.repeat(RUN)}`), - ], - [ - 'sitemap dates whose only end is the last one', - () => - parseSitemapEntries(`${'`), - ], - [ - 'a sitemap of many small blocks', - () => - parseSitemapEntries( - 'https://a.example/x2026-01-02'.repeat( - RUN, - ), - ), - ], ['anchors with no target', () => extractLinks(`${'`)], [ 'anchors with an unquoted target', diff --git a/services/platform/lib/knowledge/crawl-parse.ts b/services/platform/lib/knowledge/crawl-parse.ts index 78d4836f9b..9768f042c4 100644 --- a/services/platform/lib/knowledge/crawl-parse.ts +++ b/services/platform/lib/knowledge/crawl-parse.ts @@ -265,147 +265,6 @@ export function isSitemapIndex(xml: string): boolean { return /]/i.test(xml); } -/** One page a sitemap lists: its address and, when its entry states one - * that can be read, the latest instant its `` can mean - * ({@link sitemapLastmodInstant}). */ -export interface SitemapEntry { - readonly loc: string; - readonly lastmod: Date | null; -} - -const HOUR_MS = 60 * 60 * 1000; -/** How far behind UTC a clock can run: the last time zone on Earth ends its - * day twelve hours after UTC does. */ -const FURTHEST_WEST_MS = 12 * HOUR_MS; - -const LASTMOD_DATE = /^(\d{4})-(\d{2})-(\d{2})$/; -const LASTMOD_DATETIME = - /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2})(?:[.,]\d{1,9})?)?(Z|[+-]\d{2}(?::?\d{2})?)?$/i; -const LASTMOD_ZONE = /^([+-])(\d{2}):?(\d{2})?$/; - -/** The UTC epoch of these calendar fields, or null when they name no real - * moment — a thirteenth month, the 30th of February, 25 o'clock. */ -function utcEpoch( - year: number, - month: number, - day: number, - hour: number, - minute: number, - second: number, -): number | null { - const epoch = Date.UTC(year, month - 1, day, hour, minute, second); - const read = new Date(epoch); - return read.getUTCFullYear() === year && - read.getUTCMonth() === month - 1 && - read.getUTCDate() === day && - read.getUTCHours() === hour && - read.getUTCMinutes() === minute && - read.getUTCSeconds() === second - ? epoch - : null; -} - -/** - * The latest instant a sitemap `` can mean, or null for a value - * that says too little to act on. A scan leaves a page alone only when it - * read the page AFTER this instant, so every doubt is resolved toward the - * later reading and an unreadable value toward "request the page": - * - * - a date and a time with a zone (`2026-10-06T14:30:00+02:00`, `…Z`) is - * that instant; - * - a date and a time without a zone is read in the zone furthest behind - * UTC — the site's own clock is not known; - * - a date alone (`2026-10-06`, what most sitemaps carry) covers its whole - * day: the page can change at any hour of it and keep the same value, so - * it means the end of that day in the zone furthest behind UTC; - * - a month or a year alone, and anything that is not a W3C date, is null. - * - * Read with patterns of its own rather than `Date.parse`, which takes a - * date alone as UTC midnight and a time without a zone as the server's - * local time: the first would skip a page edited later that day, the second - * would make the answer depend on where the backend runs. - */ -export function sitemapLastmodInstant(value: string): Date | null { - const text = value.trim(); - const date = LASTMOD_DATE.exec(text); - if (date) { - const start = utcEpoch( - Number(date[1]), - Number(date[2]), - Number(date[3]), - 0, - 0, - 0, - ); - return start === null - ? null - : new Date(start + 24 * HOUR_MS + FURTHEST_WEST_MS); - } - const stamp = LASTMOD_DATETIME.exec(text); - if (!stamp) return null; - const local = utcEpoch( - Number(stamp[1]), - Number(stamp[2]), - Number(stamp[3]), - Number(stamp[4]), - Number(stamp[5]), - Number(stamp[6] ?? '0'), - ); - if (local === null) return null; - const zone = stamp[7]; - if (zone === undefined) return new Date(local + FURTHEST_WEST_MS); - if (zone.toUpperCase() === 'Z') return new Date(local); - const offset = LASTMOD_ZONE.exec(zone); - if (!offset) return null; - const hours = Number(offset[2]); - const minutes = Number(offset[3] ?? '0'); - if (hours > 14 || minutes > 59) return null; - const ahead = (hours * 60 + minutes) * 60 * 1000; - return new Date(offset[1] === '-' ? local + ahead : local - ahead); -} - -/** What one `` block says about its page's last change. */ -function entryLastmod(block: string): Date | null { - const match = /]*>([\s\S]*?)<\/lastmod>/i.exec( - upToLast(block, /<\/lastmod>/gi), - ); - if (!match) return null; - const raw = (match[1] ?? '').trim(); - const cdata = /^$/.exec(raw); - return sitemapLastmodInstant(cdata ? (cdata[1] ?? '') : raw); -} - -/** - * The pages a sitemap lists, each with what its entry says about the page's - * last change. The addresses are exactly the ones {@link parseSitemapLocs} - * reads, in document order: one inside a `` block takes that block's - * ``, and one outside any block — a document that is not quite a - * sitemap — has none, so its page is requested as before. - * - * The blocks are matched up to the last `` only, and each block's - * date up to its last `` (`markup-scan.ts`). - */ -export function parseSitemapEntries(xml: string): SitemapEntry[] { - const entries: SitemapEntry[] = []; - const undated = (text: string): void => { - for (const loc of parseSitemapLocs(text)) { - entries.push({ loc, lastmod: null }); - } - }; - let cursor = 0; - for (const block of upToLast(xml, /<\/url>/gi).matchAll( - /])[^<>]*>([\s\S]*?)<\/url>/gi, - )) { - undated(xml.slice(cursor, block.index)); - const body = block[1] ?? ''; - const lastmod = entryLastmod(body); - for (const loc of parseSitemapLocs(body)) entries.push({ loc, lastmod }); - cursor = block.index + block[0].length; - } - undated(xml.slice(cursor)); - return entries; -} - /** `href` targets of a page's anchors, entity-decoded — attribute values * are entity-encoded by spec, so `?a=1&b=2` in the markup means * `?a=1&b=2` to the browser and must mean the same to the crawler. */ @@ -672,6 +531,63 @@ export function siteHosts(domain: string): Set { return hosts; } +const wordsOf = (text: string): string[] => + text.split(/\s+/).filter((word) => word !== ''); + +/** + * How much of the text a browser shows for a page its plain HTML already + * carries: the share of the rendered text's words found in the text read + * out of the unrendered HTML, each word counted as often as it occurs. `1` + * for a page whose rendered text is empty — nothing it shows is missing. + */ +export function plainTextCoverage( + plainText: string, + renderedText: string, +): number { + const rendered = wordsOf(renderedText); + if (rendered.length === 0) return 1; + const carried = new Map(); + for (const word of wordsOf(plainText)) { + carried.set(word, (carried.get(word) ?? 0) + 1); + } + let found = 0; + for (const word of rendered) { + const left = carried.get(word) ?? 0; + if (left === 0) continue; + carried.set(word, left - 1); + found += 1; + } + return found / rendered.length; +} + +/** The share of a rendered page's words its plain HTML has to carry for the + * page to count as server-rendered. Measured, 2026-10-06: server-rendered + * pages sit between 0.87 and 1.00 (the rest is what their scripts add — a + * consent banner, a counter, a hydrated widget), a page built by its + * JavaScript at 0.01. Nine in ten keeps the first kind and can never admit + * the second; a page in between is rendered, which is only slower. */ +export const PLAIN_TEXT_COVERAGE_FLOOR = 0.9; + +/** + * Whether a page's plain HTML carries what a browser shows for it — the + * page is server-rendered, so one plain request can tell a later scan + * whether it changed: the same text read out of the plain HTML is the same + * page. False for a page built by its JavaScript, whose plain HTML is a + * shell that reads the same whatever the page shows; such a page has to be + * rendered to be judged. + * + * Decided per page from the two texts of one visit, never from the site, + * its framework or its markup. + */ +export function plainTextCarriesRendered( + plainText: string, + renderedText: string, +): boolean { + return ( + plainTextCoverage(plainText, renderedText) >= PLAIN_TEXT_COVERAGE_FLOOR + ); +} + /** Split page text into the paragraphs the boilerplate ledger hashes. Short * fragments (menu items, button labels) are ignored — hashing them would * blocklist ordinary words. */ diff --git a/services/platform/tests/manual/reference/automation.md b/services/platform/tests/manual/reference/automation.md index d897618a1a..4e53475ebe 100644 --- a/services/platform/tests/manual/reference/automation.md +++ b/services/platform/tests/manual/reference/automation.md @@ -387,7 +387,7 @@ Legend: ✅ fully automated · 🔶 partially automated · ⛔ manual-only (no s | [knowledge](../suites/knowledge.md) | Website sources take the knowledge write role on the server too: a read-only Member is refused `RBAC_FORBIDDEN` (403) on add, edit, delete, **Resume scanning** and **Scan now** (`POST /api/app/websites`, `PATCH` and `DELETE …/{websiteId}`, `…/{websiteId}/resume`, `…/{websiteId}/scan`) before the service is reached, where every member of the organization could call them before; Editor, Developer, Admin and Owner pass, and the list, a row, the status sync and the search fact stay open to every member | ✅ route | `backend/domains/websites/routes.test.ts` | | [knowledge](../suites/knowledge.md) | A page is stamped as visited only once it is stored and indexed, so a link cut off in between leaves it due for the resumed scan; unchanged text whose chunks were cut from other text is indexed again instead of serving the old text; a row the running scan retired (an alias it followed, a page that answered 404) is not revived by the links of the next page, while discovery still revives what an earlier scan retired | ✅ unit + real Postgres | `backend/core/knowledge/crawl_action.vectors.test.ts`, `backend/core/knowledge/crawl.test.ts`, `backend/core/knowledge/crawl_action.redirects.test.ts`, `backend/integration-check.ts` (`checkWebsitesCrawl`) | | [knowledge](../suites/knowledge.md) | A page that redirects onto another host or path of its site (`www.` onto the apex, a missing trailing slash, a removed page sent to the homepage) is retired as an alias and its target tracked instead, unless robots.txt disallows the target; a listed URL and a redirect that only rewrites the query keep their row | ✅ unit | `backend/core/knowledge/crawl_action.redirects.test.ts` | -| [knowledge](../suites/knowledge.md) | A scheduled rescan leaves alone the pages a site's sitemap dates as unchanged: a page whose stored text was read after its entry's `` is not requested — a date alone covers its whole day in any time zone, a time without a zone the same half day, and a read within the hour after the date does not count; a page without a readable date, one the sitemap stopped dating, one whose date moved, one whose last read is older than seven days, one whose last attempt failed and one whose stored text is not indexed yet are requested; each discovery replaces the dates the one before recorded; **Scan now** requests every page and records no date, while **Resume scanning** and the scans that follow a saved embedding model stay ordinary ones | ✅ unit + real Postgres | `lib/knowledge/crawl-parse.test.ts`, `backend/core/knowledge/crawl.test.ts`, `backend/core/knowledge/crawl_action.lastmod.test.ts`, `backend/domains/websites/service.scan-now.test.ts`, `backend/domains/websites/service.embedding-follow.test.ts`, `backend/integration-check.ts` (`checkWebsitesCrawl`: the `websites lastmod` lanes) | +| [knowledge](../suites/knowledge.md) | A scan asks for each page once and opens the browser only for a page that changed: the request carries the validators of the page's last settled visit (`ETag`, `Last-Modified`), and a 304 leaves the page as it is, stamped as visited with its failure record cleared; for a server that sends neither, an HTML page whose plain text reads as it did is left as it is too, although every response differs in its bytes (a token, a nonce), and a robots meta tag on it is still honoured; a changed page is rendered, and keeps what to check it by only when its plain HTML carried at least nine tenths of the words the browser showed, so a page built by its JavaScript is rendered on every scan; a row without stored text, or whose text is not indexed yet, is asked unconditionally; a document's 304 spares its download; purging a page's content clears what it was checked by; **Scan now** asks the same way as a scheduled scan | ✅ unit + real Postgres | `lib/knowledge/crawl-parse.test.ts`, `backend/core/knowledge/crawl_action.change-check.test.ts`, `backend/core/knowledge/crawl_action.redirects.test.ts`, `backend/integration-check.ts` (`checkWebsitesCrawl`: the `websites change check` lanes) | | [knowledge](../suites/knowledge.md) / [settings](../suites/settings.md) | The crawler's browser session has room to run: the `default` session profile allows 512 tasks (a headless Chromium needs 160 and more; under 128 every navigation after the first two or three died `net::ERR_ABORTED`), and a browser that dies mid-batch ends the batch with the untouched pages handed back, not charged | ✅ unit | `services/sandbox/src/session/docker-session-args.test.ts`, `backend/core/node_only/sandbox/render_fetch.test.ts` | | [settings](../suites/settings.md) | `SET-F33` (MCP endpoint) — the wire contract behind the page: envelope validation, version negotiation, batches, tool-argument validation against the advertised schema (document tools included), `params` typing, the unissued `tools/list` cursor, 405 + `Allow` on other verbs, `isError` on failure-shaped results, `get_docs` serving a reference and never the builder prompt | ✅ automated | `backend/core/automations_builder/mcp_http.test.ts`, `lib/engine/selftest/dispatch.test.ts` | | [projects](../suites/projects.md) | REST chat lifecycle: door-side model resolution (`CHAT_MODEL_UNKNOWN` / `CHAT_MODEL_AMBIGUOUS`), model facts and the default marker on `GET /models`, message `status` (a stop settles as `cancelled`) and `usage` (`costEstimateCents` included), the 202’s pre-minted `messageId` and the `queued` poll state until the worker opens the turn, streaming progress on the poll (`since` and `reasoningSince` slice the text and the reasoning independently, each with its offset/length pair and the reset-to-0 rule), `reasoningEffort` / `maxOutputTokens` on a send (a cap above the model’s ceiling refused; the cap a per-turn budget the tool rounds share — a later round gets what the earlier ones left, a round with nothing left is not run, a cut round marks the turn `finishReason: length` and its calls settle `invalid_args` unrun), `usage.costEstimateCents` rounded to a millionth of a cent, a named `locale` asking for the reply language on the system prompt and on the message (`localeFixed` through the job into the runtime directive and the wire-only notice; the app lane untouched), archive with `archivedAt` echoed from the domain’s own stamp / rename / delete / cancel routes, the stop of a send still queued (202 `cancelling` → a `cancelled` reply without a model call) and the 404’s `data.lastMessageId` / `data.lastStatus`, hidden regenerate-siblings invisible by id, blank-prompt and locale refusals | ✅ automated | `backend/rest/v1-threads.test.ts`, `backend/rest/v1-threads.contract.test.ts`, `backend/rest/v1-threads.scope.test.ts`, `backend/domains/chat/rest-turn.test.ts`, `backend/domains/chat/store.test.ts`, `lib/chat/turn.test.ts`, `lib/chat/context.test.ts`, `lib/chat/effort.test.ts`, `lib/chat/tools.test.ts`, `lib/shared/utils/language-name.test.ts`, `scripts/openapi/spec.test.ts`, the `REST resources` lane in `backend:integration` (the queued stop and the 404’s data on the real schema) | From 0d2c2fd7faa3d3bc2c21c8441a67d3be9c691288 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Tue, 6 Oct 2026 22:32:14 +0800 Subject: [PATCH 3/3] test(platform): keep the change-check scan tests out of the indexer Two tests of a rendered page let the store report changed text, which sent the scan into the indexer and its lookup of the knowledge database: green beside a developer's `.env`, red in CI. The row now already stores what the browser shows, and the tests assert the indexer is never asked. Also stops exporting the coverage floor, which only its own module reads (Knip). --- .../crawl_action.change-check.test.ts | 40 ++++++++++++++----- .../platform/lib/knowledge/crawl-parse.ts | 2 +- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/services/platform/backend/core/knowledge/crawl_action.change-check.test.ts b/services/platform/backend/core/knowledge/crawl_action.change-check.test.ts index d2e544c6ba..32e26d3452 100644 --- a/services/platform/backend/core/knowledge/crawl_action.change-check.test.ts +++ b/services/platform/backend/core/knowledge/crawl_action.change-check.test.ts @@ -14,6 +14,7 @@ import { fetchAndStorePage, scanWebsiteImpl, } from './crawl_action'; +import { assertCorpusWritable } from './index_health'; import { getKnowledgePoolForOrg } from './pool'; vi.mock('../../../lib/net/safe-fetch', async (importOriginal) => ({ @@ -29,7 +30,12 @@ vi.mock('../node_only/sandbox/render_fetch', async (importOriginal) => ({ vi.mock('./pool', async (importOriginal) => ({ ...(await importOriginal()), getKnowledgePoolForOrg: vi.fn(), + resolveOrgUrl: vi.fn(async () => 'postgresql://corpus.example/tale'), })); +// The indexer's gate on a corpus under repair: these tests store text the +// index already holds, so it must never be asked — and never reach for a +// real database when one is. +vi.mock('./index_health', () => ({ assertCorpusWritable: vi.fn() })); /** * A scan asks for each page once and opens the browser only for a page that @@ -133,7 +139,11 @@ beforeEach(() => { vi.mocked(safeFetchBinary).mockReset(); vi.mocked(renderUrlsInSandbox).mockReset(); vi.mocked(getKnowledgePoolForOrg).mockReset(); - vi.spyOn(console, 'log').mockImplementation(() => undefined); + vi.mocked(assertCorpusWritable).mockClear(); + // Cleared: a spy on a method that is already spied on keeps its calls. + vi.spyOn(console, 'log') + .mockClear() + .mockImplementation(() => undefined); vi.spyOn(console, 'warn').mockImplementation(() => undefined); }); @@ -530,9 +540,13 @@ describe('scanWebsiteImpl — a page that changed', () => { it('renders a changed page and keeps what to check it by next time', async () => { const now = body(TEXT_V2, 'b'); - const { sql, statements } = scanCorpus( - settled({ probe_hash: hashOf(body(TEXT_V1, 'a')) }), - ); + const { sql, statements } = scanCorpus({ + ...settled({ probe_hash: hashOf(body(TEXT_V1, 'a')) }), + // The row already stores what the browser is about to show (an + // earlier scan stored it and stopped before stamping the visit), so + // the store finds nothing to index. + content_hash: hashOf(now), + }); vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); answers({ body: now, headers: { etag: '"v2"' } }); // The browser shows what the plain HTML already said. @@ -541,6 +555,7 @@ describe('scanWebsiteImpl — a page that changed', () => { await scanWebsiteImpl(engineCtx(), SCAN); expect(renderUrlsInSandbox).toHaveBeenCalledTimes(1); + expect(assertCorpusWritable).not.toHaveBeenCalled(); expect(settledWrite(statements)?.params).toEqual([ DOMAIN, URL, @@ -552,18 +567,21 @@ describe('scanWebsiteImpl — a page that changed', () => { it('keeps nothing to check a page by when only the browser could read it', async () => { const shell = body('Loading…', 'a'); - const { sql, statements } = scanCorpus(settled({})); + const shown = body( + `${TEXT_V1} ${TEXT_V2} A long article that only the page's own scripts fetched and rendered for the reader.`, + 'a', + ); + const { sql, statements } = scanCorpus({ + ...settled({}), + content_hash: hashOf(shown), + }); vi.mocked(getKnowledgePoolForOrg).mockResolvedValue(sql); answers({ body: shell, headers: { etag: '"shell"' } }); - shows( - body( - `${TEXT_V1} ${TEXT_V2} A long article that only the page's own scripts fetched and rendered for the reader.`, - 'a', - ), - ); + shows(shown); await scanWebsiteImpl(engineCtx(), SCAN); + expect(assertCorpusWritable).not.toHaveBeenCalled(); expect(settledWrite(statements)?.params).toEqual([ DOMAIN, URL, diff --git a/services/platform/lib/knowledge/crawl-parse.ts b/services/platform/lib/knowledge/crawl-parse.ts index 9768f042c4..cf516eaf2e 100644 --- a/services/platform/lib/knowledge/crawl-parse.ts +++ b/services/platform/lib/knowledge/crawl-parse.ts @@ -566,7 +566,7 @@ export function plainTextCoverage( * consent banner, a counter, a hydrated widget), a page built by its * JavaScript at 0.01. Nine in ten keeps the first kind and can never admit * the second; a page in between is rendered, which is only slower. */ -export const PLAIN_TEXT_COVERAGE_FLOOR = 0.9; +const PLAIN_TEXT_COVERAGE_FLOOR = 0.9; /** * Whether a page's plain HTML carries what a browser shows for it — the