diff --git a/CLAUDE.md b/CLAUDE.md
index bde1712f75..732fb04e4b 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -94,20 +94,28 @@ rendered client-side never reaches an agent, which is also why `LanguageContent`
---
-## CloudFront Configuration as Code
+## Service Configuration as Code
-Three pieces of CloudFront config are versioned here and do not travel with the S3 upload, so a
-local build never exercises them. `deploy.ps1` phase 3 pushes all three:
+CloudFront and Algolia config is versioned here and does not travel with the S3 upload, so a local
+build never exercises it. `deploy.ps1` pushes it: CloudFront in phase 3, Algolia after the
+invalidation.
| Source | Pushed by | Identifier |
|---|---|---|
| `scripts/redirects.json` | `Update-CloudFrontKVS` in `deploy.ps1` | `$env:KVS_ARN` |
| `scripts/handle_redirects.js` | `scripts/sync-edge-function.ps1` | `-EdgeFunctionName` |
| `scripts/lib/csp-policy.js` | `scripts/sync-csp.ps1` | `-ResponseHeadersPolicyId` |
+| `scripts/lib/algolia-crawler-config.js` | `scripts/sync-algolia.ps1` | `$env:ALGOLIA_CRAWLER_ID` |
+| `scripts/lib/algolia-index-settings.json` | `scripts/sync-algolia.ps1` | index named in the crawler config |
Both AWS APIs replace the whole config, so both sync scripts re-send it verbatim and verify
afterwards: the response headers policy also holds CORS and HSTS, and the function config holds
-the KVS binding. `-Check` reports drift without writing, for CI.
+the KVS binding. The Algolia APIs merge top-level keys instead, so `sync-algolia.ps1` compares
+only the keys the repo keeps and warns about keys only Algolia has. The crawler's indexing
+`apiKey` stays in Algolia for that reason. A crawler config change starts a recrawl, since it only
+applies to newly crawled records; a settings change applies at once. The crawler's
+`initialIndexSettings`, which Algolia applies only to a new index, is built from the same settings
+file. `-Check` reports drift without writing, for CI.
---
diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1
index e80b825ea3..78956a813b 100644
--- a/scripts/deploy.ps1
+++ b/scripts/deploy.ps1
@@ -16,10 +16,12 @@ Prerequisites
*What's New* for specific versions)
* Environment variable **`KVS_ARN`** set (CloudFront KeyValueStore holding the
redirect table)
+* Optional `ALGOLIA_*` variables for the Algolia config push (see `sync-algolia.ps1`)
* Project `package.json` includes `@docusaurus/core` and `@docusaurus/cli`
Phase 3 also pushes the CloudFront config kept in this repo: the redirect table,
-the viewer-request function and the CSP. See CLAUDE.md.
+the viewer-request function and the CSP. After the invalidation it pushes the
+Algolia config. See CLAUDE.md.
Example
-------
@@ -61,6 +63,7 @@ $PythonWhatsNewPath = Join-Path $PSScriptRoot 'build_whats_new.py'
$RedirectsFilePath = Join-Path $PSScriptRoot 'redirects.json'
$SyncEdgeFunctionPath = Join-Path $PSScriptRoot 'sync-edge-function.ps1'
$SyncCspPath = Join-Path $PSScriptRoot 'sync-csp.ps1'
+$SyncAlgoliaPath = Join-Path $PSScriptRoot 'sync-algolia.ps1'
function ThrowIfEmpty {
param (
@@ -226,7 +229,7 @@ if ($empty) {
}
if ($DryRun) {
- Write-Host "Dry run mode enabled. Skipping sync to s3://$S3BucketName/, CloudFront KeyValueStore update, edge function publish, CSP push and CloudFront invalidation." -ForegroundColor Yellow
+ Write-Host "Dry run mode enabled. Skipping sync to s3://$S3BucketName/, CloudFront KeyValueStore update, edge function publish, CSP push, CloudFront invalidation and Algolia config push." -ForegroundColor Yellow
} else {
Write-Host "Syncing to s3://$S3BucketName/ ..." -ForegroundColor Cyan
@@ -274,6 +277,15 @@ if ($DryRun) {
if ($LASTEXITCODE) { throw 'CloudFront invalidation failed' }
}
+ # After the invalidation, so a recrawl reads this release's HTML; errors are held like the CloudFront sync.
+ Write-Host 'Syncing Algolia configuration' -ForegroundColor Cyan
+ try {
+ & $SyncAlgoliaPath
+ } catch {
+ if (-not $configError) { $configError = $_ }
+ Write-Host " Algolia config sync failed: $_" -ForegroundColor Red
+ }
+
# Phase 4 — CloudFront now serves only the new index.html, so old hashed
# bundles are no longer referenced and can be safely removed
Write-Host " [4/4] Cleanup: removing stale hashed assets from S3" -ForegroundColor Gray
diff --git a/scripts/lib/algolia-crawler-config.js b/scripts/lib/algolia-crawler-config.js
new file mode 100644
index 0000000000..fd5716e89a
--- /dev/null
+++ b/scripts/lib/algolia-crawler-config.js
@@ -0,0 +1,95 @@
+// Algolia Crawler config, pushed by scripts/sync-algolia.ps1; the crawler's apiKey stays in Algolia.
+
+const indexSettings = require("./algolia-index-settings.json");
+
+const INDEX_NAME = "ProductionDocsCrawler";
+
+const recordExtractor = ({ url, $, helpers }) => {
+ const LANDING_PAGES = ["", "/7.2", "/7.1", "/6.2", "/cloud", "/guides", "/samples", "/quill"];
+ const SECTION_LABELS = {
+ guides: "Guides",
+ cloud: "Cloud",
+ samples: "Samples",
+ quill: "Quill",
+ };
+
+ const path = url.pathname.endsWith("/") ? url.pathname.slice(0, -1) : url.pathname;
+ if (LANDING_PAGES.includes(path)) {
+ return [];
+ }
+
+ const lvl0 =
+ SECTION_LABELS[path.split("/")[1]] ||
+ $(".menu__link.menu__link--sublist.menu__link--active, .navbar__item.navbar__link--active").last().text() ||
+ "Documentation";
+
+ // The page's own breadcrumb, minus the home icon and the current page
+ const items = $(".theme-doc-breadcrumbs .breadcrumbs__item").not(".breadcrumbs__item--active");
+ const crumbs = [];
+ for (let i = 0; i < items.length; i++) {
+ const text = items.eq(i).text().trim();
+ if (text) {
+ crumbs.push(text);
+ }
+ }
+ const breadcrumb = crumbs.join(" › ");
+
+ const records = helpers.docsearch({
+ recordProps: {
+ lvl0: { selectors: "", defaultValue: lvl0 },
+ lvl1: ["header h1", "article h1"],
+ lvl2: "article h2",
+ lvl3: "article h3",
+ lvl4: "article h4",
+ lvl5: "article h5, article td:first-child",
+ lvl6: "article h6",
+ content: "article p, article li, article td:last-child",
+ },
+ aggregateContent: true,
+ recordVersion: "v3",
+ });
+
+ return records.map((record) => ({ ...record, breadcrumb }));
+};
+
+const CRAWLER_CONFIG = {
+ appId: "GYTCYX561T",
+ indexPrefix: "",
+ rateLimit: 8,
+ maxDepth: 10,
+ maxUrls: null,
+ schedule: "on the 8 day of the month",
+ startUrls: ["https://docs.ravendb.net/7.2"],
+ renderJavaScript: false,
+ sitemaps: ["https://docs.ravendb.net/sitemap.xml"],
+ // Older versions canonicalize to the current one; without this they would not be indexed.
+ ignoreCanonicalTo: true,
+ discoveryPatterns: ["https://docs.ravendb.net/**"],
+ ignoreRobotsTxtRules: true,
+ // Language switcher ?lang= links serve the same HTML; index each page once.
+ ignoreQueryParams: ["lang"],
+ actions: [
+ {
+ indexName: INDEX_NAME,
+ pathsToMatch: ["https://docs.ravendb.net/**"],
+ recordExtractor,
+ },
+ ],
+ safetyChecks: {
+ beforeIndexPublishing: {
+ maxLostRecordsPercentage: 50,
+ },
+ },
+ // Applied only when Algolia creates the index; nulls are defaults, not values.
+ initialIndexSettings: {
+ [INDEX_NAME]: Object.fromEntries(Object.entries(indexSettings).filter(([, value]) => value !== null)),
+ },
+};
+
+module.exports = { CRAWLER_CONFIG };
+
+if (require.main === module) {
+ const toApi = (key, value) =>
+ typeof value === "function" ? { __type: "function", source: value.toString() } : value;
+ console.log(JSON.stringify(CRAWLER_CONFIG, toApi, 4));
+}
diff --git a/scripts/lib/algolia-index-settings.json b/scripts/lib/algolia-index-settings.json
new file mode 100644
index 0000000000..e519b49d30
--- /dev/null
+++ b/scripts/lib/algolia-index-settings.json
@@ -0,0 +1,41 @@
+{
+ "minWordSizefor1Typo": 3,
+ "minWordSizefor2Typos": 7,
+ "hitsPerPage": 20,
+ "maxValuesPerFacet": 100,
+ "minProximity": 1,
+ "searchableAttributes": [
+ "unordered(hierarchy.lvl0)",
+ "unordered(hierarchy.lvl1)",
+ "unordered(hierarchy.lvl2)",
+ "unordered(hierarchy.lvl3)",
+ "unordered(hierarchy.lvl4)",
+ "unordered(hierarchy.lvl5)",
+ "unordered(hierarchy.lvl6)",
+ "content"
+ ],
+ "numericAttributesToIndex": null,
+ "attributesToRetrieve": ["hierarchy", "content", "anchor", "url", "url_without_anchor", "type"],
+ "allowTyposOnNumericTokens": false,
+ "ignorePlurals": true,
+ "camelCaseAttributes": ["hierarchy", "content"],
+ "advancedSyntax": true,
+ "attributeCriteriaComputedByMinProximity": true,
+ "distinct": true,
+ "unretrievableAttributes": null,
+ "optionalWords": null,
+ "attributesForFaceting": ["type", "lang", "language", "version", "docusaurus_tag"],
+ "attributesToSnippet": ["content:10"],
+ "attributesToHighlight": ["hierarchy", "content"],
+ "paginationLimitedTo": 1000,
+ "attributeForDistinct": "url",
+ "exactOnSingleWordQuery": "attribute",
+ "ranking": ["words", "filters", "typo", "attribute", "proximity", "exact", "custom"],
+ "customRanking": ["desc(weight.pageRank)", "desc(weight.level)", "asc(weight.position)"],
+ "separatorsToIndex": "",
+ "removeWordsIfNoResults": "allOptional",
+ "queryType": "prefixLast",
+ "highlightPreTag": "",
+ "highlightPostTag": "",
+ "alternativesAsExact": ["ignorePlurals", "singleWordSynonym"]
+}
diff --git a/scripts/lib/cloudfront-common.ps1 b/scripts/lib/cloudfront-common.ps1
index cd682b4c59..1f81227075 100644
--- a/scripts/lib/cloudfront-common.ps1
+++ b/scripts/lib/cloudfront-common.ps1
@@ -1,4 +1,4 @@
-# Shared by sync-csp.ps1 and sync-edge-function.ps1.
+# Shared by sync-csp.ps1 and sync-edge-function.ps1. sync-algolia.ps1 uses Get-JsonDifference.
# CloudFront limits: CSP header value, and function code size.
$script:MaxCspBytes = 1783
diff --git a/scripts/sync-algolia.ps1 b/scripts/sync-algolia.ps1
new file mode 100644
index 0000000000..7253a23ed6
--- /dev/null
+++ b/scripts/sync-algolia.ps1
@@ -0,0 +1,120 @@
+# Pushes the Algolia crawler config and index settings from scripts/lib when they differ; -Check reports drift.
+# pwsh scripts/sync-algolia.ps1 [-CrawlerId ] [-Check] [-DryRun]
+# Env: ALGOLIA_CRAWLER_ID/_USER_ID/_API_KEY (crawler), ALGOLIA_API_KEY with editSettings (settings); unset = skipped.
+
+[CmdletBinding()]
+param(
+ [Parameter(HelpMessage = 'Algolia Crawler ID. Defaults to $env:ALGOLIA_CRAWLER_ID.')]
+ [string]$CrawlerId = $env:ALGOLIA_CRAWLER_ID,
+
+ [Parameter(HelpMessage = 'Report drift and exit non-zero. Writes nothing.')]
+ [switch]$Check,
+
+ [Parameter(HelpMessage = 'Print the diff without writing.')]
+ [switch]$DryRun
+)
+
+$ErrorActionPreference = 'Stop'
+. (Join-Path $PSScriptRoot 'lib/cloudfront-common.ps1')
+
+# node prints non-ASCII (the breadcrumb separator).
+[Console]::OutputEncoding = [Text.UTF8Encoding]::new()
+
+$CrawlerConfigPath = Join-Path $PSScriptRoot 'lib/algolia-crawler-config.js'
+$IndexSettingsPath = Join-Path $PSScriptRoot 'lib/algolia-index-settings.json'
+
+$crawlerJson = (& node $CrawlerConfigPath | Out-String)
+if ($LASTEXITCODE -or -not $crawlerJson) { throw "Failed to render $CrawlerConfigPath" }
+$crawler = $crawlerJson | ConvertFrom-Json
+$settingsJson = Get-Content -Path $IndexSettingsPath -Raw
+$settings = $settingsJson | ConvertFrom-Json
+
+$appId = $crawler.appId
+$indexName = $crawler.actions[0].indexName
+
+# Diff over the repo's keys; a push cannot remove live-only keys (both APIs merge), so those only warn.
+function Get-ManagedDrift {
+ param($Live, $Local, [string]$Noun, [string[]]$Unmanaged = @())
+
+ $localNames = @($Local.PSObject.Properties.Name)
+ $liveOnly = @($Live.PSObject.Properties.Name | Where-Object { $_ -notin $localNames -and $_ -notin $Unmanaged })
+ if ($liveOnly.Count) {
+ Write-Warning "Live $Noun keys missing from the repo: $($liveOnly -join ', '). Add them to the repo copy."
+ }
+
+ $managed = [ordered]@{}
+ foreach ($name in $localNames) { $managed[$name] = $Live.$name }
+ return @(Get-JsonDifference -Reference ([pscustomobject]$managed) -Candidate $Local)
+}
+
+function Test-ShouldPush {
+ param([string[]]$Drift, [string]$Noun, [string]$SourcePath)
+
+ if (-not $Drift.Count) {
+ Write-Host " $Noun`: in sync with the repo. Nothing to do." -ForegroundColor Green
+ return $false
+ }
+ foreach ($d in $Drift) { Write-Host " $d" -ForegroundColor Gray }
+
+ if ($Check) {
+ Write-Host " $Noun`: live differs from $SourcePath. Run without -Check to push it." -ForegroundColor Red
+ $script:driftFound = $true
+ return $false
+ }
+ if ($DryRun) {
+ Write-Host " Dry run. Skipping the $Noun update." -ForegroundColor Yellow
+ return $false
+ }
+ return $true
+}
+
+$driftFound = $false
+
+if ($CrawlerId -and $env:ALGOLIA_CRAWLER_USER_ID -and $env:ALGOLIA_CRAWLER_API_KEY) {
+ $crawlerUri = "https://crawler.algolia.com/api/1/crawlers/$CrawlerId"
+ $pair = "$($env:ALGOLIA_CRAWLER_USER_ID):$($env:ALGOLIA_CRAWLER_API_KEY)"
+ $crawlerHeaders = @{ Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($pair)) }
+ # The crawler's apiKey is not in the repo.
+ $getCrawlerDrift = {
+ $live = (Invoke-RestMethod -Uri "${crawlerUri}?withConfig=true" -Headers $crawlerHeaders).config
+ Get-ManagedDrift -Live $live -Local $crawler -Noun 'crawler config' -Unmanaged 'apiKey'
+ }
+
+ Write-Host "Reading crawler $CrawlerId" -ForegroundColor Cyan
+ if (Test-ShouldPush -Drift (& $getCrawlerDrift) -Noun 'crawler config' -SourcePath $CrawlerConfigPath) {
+ Write-Host ' Updating the crawler config' -ForegroundColor Cyan
+ Invoke-RestMethod -Uri "$crawlerUri/config" -Headers $crawlerHeaders -Method Patch `
+ -Body $crawlerJson -ContentType 'application/json; charset=utf-8' | Out-Null
+
+ # Fail if the API rewrote anything, or every deploy would recrawl.
+ $left = @(& $getCrawlerDrift)
+ if ($left.Count) { throw "The crawler config does not read back as written: $($left -join '; ')" }
+
+ Write-Host ' Starting a recrawl' -ForegroundColor Cyan
+ Invoke-RestMethod -Uri "$crawlerUri/reindex" -Headers $crawlerHeaders -Method Post | Out-Null
+ Write-Host 'Crawler config updated, recrawl started.' -ForegroundColor Green
+ }
+} else {
+ Write-Host ' crawler: skipped, no ALGOLIA_CRAWLER_ID / ALGOLIA_CRAWLER_USER_ID / ALGOLIA_CRAWLER_API_KEY' -ForegroundColor Yellow
+}
+
+if ($env:ALGOLIA_API_KEY) {
+ $settingsRequest = @{
+ Uri = "https://$appId.algolia.net/1/indexes/$indexName/settings"
+ Headers = @{ 'X-Algolia-Application-Id' = $appId; 'X-Algolia-API-Key' = $env:ALGOLIA_API_KEY }
+ }
+
+ Write-Host "Reading index settings for $indexName" -ForegroundColor Cyan
+ $live = Invoke-RestMethod @settingsRequest
+ # `version` is the response format, not a setting.
+ $drift = Get-ManagedDrift -Live $live -Local $settings -Noun 'index settings' -Unmanaged 'version'
+
+ if (Test-ShouldPush -Drift $drift -Noun 'index settings' -SourcePath $IndexSettingsPath) {
+ Invoke-RestMethod @settingsRequest -Method Put -Body $settingsJson -ContentType 'application/json; charset=utf-8' | Out-Null
+ Write-Host 'Index settings updated.' -ForegroundColor Green
+ }
+} else {
+ Write-Host ' index settings: skipped, no ALGOLIA_API_KEY' -ForegroundColor Yellow
+}
+
+if ($driftFound) { exit 1 }