From 20acd4bbc900e0f0ae715a004f565d886e6aad6f Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Fri, 23 Oct 2026 01:18:52 +0200 Subject: [PATCH 1/2] RDoc-4148 Sync Algolia config automatically & place it inside git --- CLAUDE.md | 16 +++- scripts/deploy.ps1 | 16 +++- scripts/lib/algolia-crawler-config.js | 93 ++++++++++++++++++ scripts/lib/algolia-index-settings.json | 41 ++++++++ scripts/lib/cloudfront-common.ps1 | 2 +- scripts/sync-algolia.ps1 | 120 ++++++++++++++++++++++++ 6 files changed, 281 insertions(+), 7 deletions(-) create mode 100644 scripts/lib/algolia-crawler-config.js create mode 100644 scripts/lib/algolia-index-settings.json create mode 100644 scripts/sync-algolia.ps1 diff --git a/CLAUDE.md b/CLAUDE.md index bde1712f75..732fb04e4b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,20 +94,28 @@ rendered client-side never reaches an agent, which is also why `LanguageContent` --- -## CloudFront Configuration as Code +## Service Configuration as Code -Three pieces of CloudFront config are versioned here and do not travel with the S3 upload, so a -local build never exercises them. `deploy.ps1` phase 3 pushes all three: +CloudFront and Algolia config is versioned here and does not travel with the S3 upload, so a local +build never exercises it. `deploy.ps1` pushes it: CloudFront in phase 3, Algolia after the +invalidation. | Source | Pushed by | Identifier | |---|---|---| | `scripts/redirects.json` | `Update-CloudFrontKVS` in `deploy.ps1` | `$env:KVS_ARN` | | `scripts/handle_redirects.js` | `scripts/sync-edge-function.ps1` | `-EdgeFunctionName` | | `scripts/lib/csp-policy.js` | `scripts/sync-csp.ps1` | `-ResponseHeadersPolicyId` | +| `scripts/lib/algolia-crawler-config.js` | `scripts/sync-algolia.ps1` | `$env:ALGOLIA_CRAWLER_ID` | +| `scripts/lib/algolia-index-settings.json` | `scripts/sync-algolia.ps1` | index named in the crawler config | Both AWS APIs replace the whole config, so both sync scripts re-send it verbatim and verify afterwards: the response headers policy also holds CORS and HSTS, and the function config holds -the KVS binding. `-Check` reports drift without writing, for CI. +the KVS binding. The Algolia APIs merge top-level keys instead, so `sync-algolia.ps1` compares +only the keys the repo keeps and warns about keys only Algolia has. The crawler's indexing +`apiKey` stays in Algolia for that reason. A crawler config change starts a recrawl, since it only +applies to newly crawled records; a settings change applies at once. The crawler's +`initialIndexSettings`, which Algolia applies only to a new index, is built from the same settings +file. `-Check` reports drift without writing, for CI. --- diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1 index e80b825ea3..78956a813b 100644 --- a/scripts/deploy.ps1 +++ b/scripts/deploy.ps1 @@ -16,10 +16,12 @@ Prerequisites *What's New* for specific versions) * Environment variable **`KVS_ARN`** set (CloudFront KeyValueStore holding the redirect table) +* Optional `ALGOLIA_*` variables for the Algolia config push (see `sync-algolia.ps1`) * Project `package.json` includes `@docusaurus/core` and `@docusaurus/cli` Phase 3 also pushes the CloudFront config kept in this repo: the redirect table, -the viewer-request function and the CSP. See CLAUDE.md. +the viewer-request function and the CSP. After the invalidation it pushes the +Algolia config. See CLAUDE.md. Example ------- @@ -61,6 +63,7 @@ $PythonWhatsNewPath = Join-Path $PSScriptRoot 'build_whats_new.py' $RedirectsFilePath = Join-Path $PSScriptRoot 'redirects.json' $SyncEdgeFunctionPath = Join-Path $PSScriptRoot 'sync-edge-function.ps1' $SyncCspPath = Join-Path $PSScriptRoot 'sync-csp.ps1' +$SyncAlgoliaPath = Join-Path $PSScriptRoot 'sync-algolia.ps1' function ThrowIfEmpty { param ( @@ -226,7 +229,7 @@ if ($empty) { } if ($DryRun) { - Write-Host "Dry run mode enabled. Skipping sync to s3://$S3BucketName/, CloudFront KeyValueStore update, edge function publish, CSP push and CloudFront invalidation." -ForegroundColor Yellow + Write-Host "Dry run mode enabled. Skipping sync to s3://$S3BucketName/, CloudFront KeyValueStore update, edge function publish, CSP push, CloudFront invalidation and Algolia config push." -ForegroundColor Yellow } else { Write-Host "Syncing to s3://$S3BucketName/ ..." -ForegroundColor Cyan @@ -274,6 +277,15 @@ if ($DryRun) { if ($LASTEXITCODE) { throw 'CloudFront invalidation failed' } } + # After the invalidation, so a recrawl reads this release's HTML; errors are held like the CloudFront sync. + Write-Host 'Syncing Algolia configuration' -ForegroundColor Cyan + try { + & $SyncAlgoliaPath + } catch { + if (-not $configError) { $configError = $_ } + Write-Host " Algolia config sync failed: $_" -ForegroundColor Red + } + # Phase 4 — CloudFront now serves only the new index.html, so old hashed # bundles are no longer referenced and can be safely removed Write-Host " [4/4] Cleanup: removing stale hashed assets from S3" -ForegroundColor Gray diff --git a/scripts/lib/algolia-crawler-config.js b/scripts/lib/algolia-crawler-config.js new file mode 100644 index 0000000000..77b410caa7 --- /dev/null +++ b/scripts/lib/algolia-crawler-config.js @@ -0,0 +1,93 @@ +// Algolia Crawler config, pushed by scripts/sync-algolia.ps1; the crawler's apiKey stays in Algolia. + +const indexSettings = require("./algolia-index-settings.json"); + +const INDEX_NAME = "ProductionDocsCrawler"; + +const recordExtractor = ({ url, $, helpers }) => { + const LANDING_PAGES = ["", "/7.2", "/7.1", "/6.2", "/cloud", "/guides", "/samples", "/quill"]; + const SECTION_LABELS = { + guides: "Guides", + cloud: "Cloud", + samples: "Samples", + quill: "Quill", + }; + + const path = url.pathname.endsWith("/") ? url.pathname.slice(0, -1) : url.pathname; + if (LANDING_PAGES.includes(path)) { + return []; + } + + const lvl0 = + SECTION_LABELS[path.split("/")[1]] || + $(".menu__link.menu__link--sublist.menu__link--active, .navbar__item.navbar__link--active").last().text() || + "Documentation"; + + // The page's own breadcrumb, minus the home icon and the current page + const items = $(".theme-doc-breadcrumbs .breadcrumbs__item").not(".breadcrumbs__item--active"); + const crumbs = []; + for (let i = 0; i < items.length; i++) { + const text = items.eq(i).text().trim(); + if (text) { + crumbs.push(text); + } + } + const breadcrumb = crumbs.join(" › "); + + const records = helpers.docsearch({ + recordProps: { + lvl0: { selectors: "", defaultValue: lvl0 }, + lvl1: ["header h1", "article h1"], + lvl2: "article h2", + lvl3: "article h3", + lvl4: "article h4", + lvl5: "article h5, article td:first-child", + lvl6: "article h6", + content: "article p, article li, article td:last-child", + }, + aggregateContent: true, + recordVersion: "v3", + }); + + return records.map((record) => ({ ...record, breadcrumb })); +}; + +const CRAWLER_CONFIG = { + appId: "GYTCYX561T", + indexPrefix: "", + rateLimit: 8, + maxDepth: 10, + maxUrls: null, + schedule: "on the 8 day of the month", + startUrls: ["https://docs.ravendb.net/7.2"], + renderJavaScript: false, + sitemaps: ["https://docs.ravendb.net/sitemap.xml"], + // Older versions canonicalize to the current one; without this they would not be indexed. + ignoreCanonicalTo: true, + discoveryPatterns: ["https://docs.ravendb.net/**"], + ignoreRobotsTxtRules: true, + actions: [ + { + indexName: INDEX_NAME, + pathsToMatch: ["https://docs.ravendb.net/**"], + recordExtractor, + }, + ], + safetyChecks: { + beforeIndexPublishing: { + maxLostRecordsPercentage: 50, + }, + }, + // Applied only when Algolia creates the index; nulls are defaults, not values. + initialIndexSettings: { + [INDEX_NAME]: Object.fromEntries(Object.entries(indexSettings).filter(([, value]) => value !== null)), + }, +}; + +module.exports = { CRAWLER_CONFIG }; + +if (require.main === module) { + const toApi = (key, value) => + typeof value === "function" ? { __type: "function", source: value.toString() } : value; + console.log(JSON.stringify(CRAWLER_CONFIG, toApi, 4)); +} diff --git a/scripts/lib/algolia-index-settings.json b/scripts/lib/algolia-index-settings.json new file mode 100644 index 0000000000..e519b49d30 --- /dev/null +++ b/scripts/lib/algolia-index-settings.json @@ -0,0 +1,41 @@ +{ + "minWordSizefor1Typo": 3, + "minWordSizefor2Typos": 7, + "hitsPerPage": 20, + "maxValuesPerFacet": 100, + "minProximity": 1, + "searchableAttributes": [ + "unordered(hierarchy.lvl0)", + "unordered(hierarchy.lvl1)", + "unordered(hierarchy.lvl2)", + "unordered(hierarchy.lvl3)", + "unordered(hierarchy.lvl4)", + "unordered(hierarchy.lvl5)", + "unordered(hierarchy.lvl6)", + "content" + ], + "numericAttributesToIndex": null, + "attributesToRetrieve": ["hierarchy", "content", "anchor", "url", "url_without_anchor", "type"], + "allowTyposOnNumericTokens": false, + "ignorePlurals": true, + "camelCaseAttributes": ["hierarchy", "content"], + "advancedSyntax": true, + "attributeCriteriaComputedByMinProximity": true, + "distinct": true, + "unretrievableAttributes": null, + "optionalWords": null, + "attributesForFaceting": ["type", "lang", "language", "version", "docusaurus_tag"], + "attributesToSnippet": ["content:10"], + "attributesToHighlight": ["hierarchy", "content"], + "paginationLimitedTo": 1000, + "attributeForDistinct": "url", + "exactOnSingleWordQuery": "attribute", + "ranking": ["words", "filters", "typo", "attribute", "proximity", "exact", "custom"], + "customRanking": ["desc(weight.pageRank)", "desc(weight.level)", "asc(weight.position)"], + "separatorsToIndex": "", + "removeWordsIfNoResults": "allOptional", + "queryType": "prefixLast", + "highlightPreTag": "", + "highlightPostTag": "", + "alternativesAsExact": ["ignorePlurals", "singleWordSynonym"] +} diff --git a/scripts/lib/cloudfront-common.ps1 b/scripts/lib/cloudfront-common.ps1 index cd682b4c59..1f81227075 100644 --- a/scripts/lib/cloudfront-common.ps1 +++ b/scripts/lib/cloudfront-common.ps1 @@ -1,4 +1,4 @@ -# Shared by sync-csp.ps1 and sync-edge-function.ps1. +# Shared by sync-csp.ps1 and sync-edge-function.ps1. sync-algolia.ps1 uses Get-JsonDifference. # CloudFront limits: CSP header value, and function code size. $script:MaxCspBytes = 1783 diff --git a/scripts/sync-algolia.ps1 b/scripts/sync-algolia.ps1 new file mode 100644 index 0000000000..7253a23ed6 --- /dev/null +++ b/scripts/sync-algolia.ps1 @@ -0,0 +1,120 @@ +# Pushes the Algolia crawler config and index settings from scripts/lib when they differ; -Check reports drift. +# pwsh scripts/sync-algolia.ps1 [-CrawlerId ] [-Check] [-DryRun] +# Env: ALGOLIA_CRAWLER_ID/_USER_ID/_API_KEY (crawler), ALGOLIA_API_KEY with editSettings (settings); unset = skipped. + +[CmdletBinding()] +param( + [Parameter(HelpMessage = 'Algolia Crawler ID. Defaults to $env:ALGOLIA_CRAWLER_ID.')] + [string]$CrawlerId = $env:ALGOLIA_CRAWLER_ID, + + [Parameter(HelpMessage = 'Report drift and exit non-zero. Writes nothing.')] + [switch]$Check, + + [Parameter(HelpMessage = 'Print the diff without writing.')] + [switch]$DryRun +) + +$ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'lib/cloudfront-common.ps1') + +# node prints non-ASCII (the breadcrumb separator). +[Console]::OutputEncoding = [Text.UTF8Encoding]::new() + +$CrawlerConfigPath = Join-Path $PSScriptRoot 'lib/algolia-crawler-config.js' +$IndexSettingsPath = Join-Path $PSScriptRoot 'lib/algolia-index-settings.json' + +$crawlerJson = (& node $CrawlerConfigPath | Out-String) +if ($LASTEXITCODE -or -not $crawlerJson) { throw "Failed to render $CrawlerConfigPath" } +$crawler = $crawlerJson | ConvertFrom-Json +$settingsJson = Get-Content -Path $IndexSettingsPath -Raw +$settings = $settingsJson | ConvertFrom-Json + +$appId = $crawler.appId +$indexName = $crawler.actions[0].indexName + +# Diff over the repo's keys; a push cannot remove live-only keys (both APIs merge), so those only warn. +function Get-ManagedDrift { + param($Live, $Local, [string]$Noun, [string[]]$Unmanaged = @()) + + $localNames = @($Local.PSObject.Properties.Name) + $liveOnly = @($Live.PSObject.Properties.Name | Where-Object { $_ -notin $localNames -and $_ -notin $Unmanaged }) + if ($liveOnly.Count) { + Write-Warning "Live $Noun keys missing from the repo: $($liveOnly -join ', '). Add them to the repo copy." + } + + $managed = [ordered]@{} + foreach ($name in $localNames) { $managed[$name] = $Live.$name } + return @(Get-JsonDifference -Reference ([pscustomobject]$managed) -Candidate $Local) +} + +function Test-ShouldPush { + param([string[]]$Drift, [string]$Noun, [string]$SourcePath) + + if (-not $Drift.Count) { + Write-Host " $Noun`: in sync with the repo. Nothing to do." -ForegroundColor Green + return $false + } + foreach ($d in $Drift) { Write-Host " $d" -ForegroundColor Gray } + + if ($Check) { + Write-Host " $Noun`: live differs from $SourcePath. Run without -Check to push it." -ForegroundColor Red + $script:driftFound = $true + return $false + } + if ($DryRun) { + Write-Host " Dry run. Skipping the $Noun update." -ForegroundColor Yellow + return $false + } + return $true +} + +$driftFound = $false + +if ($CrawlerId -and $env:ALGOLIA_CRAWLER_USER_ID -and $env:ALGOLIA_CRAWLER_API_KEY) { + $crawlerUri = "https://crawler.algolia.com/api/1/crawlers/$CrawlerId" + $pair = "$($env:ALGOLIA_CRAWLER_USER_ID):$($env:ALGOLIA_CRAWLER_API_KEY)" + $crawlerHeaders = @{ Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($pair)) } + # The crawler's apiKey is not in the repo. + $getCrawlerDrift = { + $live = (Invoke-RestMethod -Uri "${crawlerUri}?withConfig=true" -Headers $crawlerHeaders).config + Get-ManagedDrift -Live $live -Local $crawler -Noun 'crawler config' -Unmanaged 'apiKey' + } + + Write-Host "Reading crawler $CrawlerId" -ForegroundColor Cyan + if (Test-ShouldPush -Drift (& $getCrawlerDrift) -Noun 'crawler config' -SourcePath $CrawlerConfigPath) { + Write-Host ' Updating the crawler config' -ForegroundColor Cyan + Invoke-RestMethod -Uri "$crawlerUri/config" -Headers $crawlerHeaders -Method Patch ` + -Body $crawlerJson -ContentType 'application/json; charset=utf-8' | Out-Null + + # Fail if the API rewrote anything, or every deploy would recrawl. + $left = @(& $getCrawlerDrift) + if ($left.Count) { throw "The crawler config does not read back as written: $($left -join '; ')" } + + Write-Host ' Starting a recrawl' -ForegroundColor Cyan + Invoke-RestMethod -Uri "$crawlerUri/reindex" -Headers $crawlerHeaders -Method Post | Out-Null + Write-Host 'Crawler config updated, recrawl started.' -ForegroundColor Green + } +} else { + Write-Host ' crawler: skipped, no ALGOLIA_CRAWLER_ID / ALGOLIA_CRAWLER_USER_ID / ALGOLIA_CRAWLER_API_KEY' -ForegroundColor Yellow +} + +if ($env:ALGOLIA_API_KEY) { + $settingsRequest = @{ + Uri = "https://$appId.algolia.net/1/indexes/$indexName/settings" + Headers = @{ 'X-Algolia-Application-Id' = $appId; 'X-Algolia-API-Key' = $env:ALGOLIA_API_KEY } + } + + Write-Host "Reading index settings for $indexName" -ForegroundColor Cyan + $live = Invoke-RestMethod @settingsRequest + # `version` is the response format, not a setting. + $drift = Get-ManagedDrift -Live $live -Local $settings -Noun 'index settings' -Unmanaged 'version' + + if (Test-ShouldPush -Drift $drift -Noun 'index settings' -SourcePath $IndexSettingsPath) { + Invoke-RestMethod @settingsRequest -Method Put -Body $settingsJson -ContentType 'application/json; charset=utf-8' | Out-Null + Write-Host 'Index settings updated.' -ForegroundColor Green + } +} else { + Write-Host ' index settings: skipped, no ALGOLIA_API_KEY' -ForegroundColor Yellow +} + +if ($driftFound) { exit 1 } From ccb9bf2be18faabfcce6ed291ce0eb9cc26b715c Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Fri, 23 Oct 2026 01:18:52 +0200 Subject: [PATCH 2/2] RDoc-4138 Search results link to random language --- scripts/lib/algolia-crawler-config.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/lib/algolia-crawler-config.js b/scripts/lib/algolia-crawler-config.js index 77b410caa7..fd5716e89a 100644 --- a/scripts/lib/algolia-crawler-config.js +++ b/scripts/lib/algolia-crawler-config.js @@ -66,6 +66,8 @@ const CRAWLER_CONFIG = { ignoreCanonicalTo: true, discoveryPatterns: ["https://docs.ravendb.net/**"], ignoreRobotsTxtRules: true, + // Language switcher ?lang= links serve the same HTML; index each page once. + ignoreQueryParams: ["lang"], actions: [ { indexName: INDEX_NAME,