SEO cleanup: sitemap, 404 metadata, coming-soon heading

app/docs/[section]/[[...slug]]/page.tsx

@@ -83,6 +83,12 @@ export default async function ArticlePage({ params }: PageProps) {
     // Use title from frontmatter if available, otherwise fall back to navigation title or section name
     const pageTitle = frontmatter.title || selectedNavItem?.title || section;
     const showInstallPrimer = section === "getting-started" && file === "index";
+    const isComingSoon = frontmatter.layout === 'coming-soon';
+    // coming-soon pages carry placeholder prose with no markdown heading, so
+    // the h1 has to come from the frontmatter title - otherwise the page ships
+    // with no heading element at all. Guarded in case a future coming-soon page
+    // does start with one.
+    const showComingSoonHeading = isComingSoon && !/^#\s/m.test(content);
     const sectionTitle = capitalCase(section)
         .replace(/documentdb/i, 'DocumentDB')
         .replace(/api/i, 'API');
@@ -184,7 +190,12 @@ export default async function ArticlePage({ params }: PageProps) {
                         </details>
 
                         {/* Coming Soon Component for coming-soon layout */}
-                        {frontmatter.layout === 'coming-soon' && <ComingSoon />}
+                        {showComingSoonHeading && (
+                            <h1 className="text-4xl font-bold text-white mb-4">
+                                {pageTitle}
+                            </h1>
+                        )}
+                        {isComingSoon && <ComingSoon />}
 
                         {showInstallPrimer && (
                             <section className="mb-8 rounded-2xl border border-blue-500/30 bg-gradient-to-br from-blue-500/10 via-neutral-900/90 to-neutral-900/90 p-6">

app/not-found.tsx

@@ -1,5 +1,19 @@
+import type { Metadata } from "next";
 import Link from "next/link";
 
+// Without this the route inherits the root layout's metadata, which hardcodes
+// robots index/follow - so the page emitted both that tag and the noindex
+// Next.js injects for not-found, and carried the homepage title verbatim.
+export const metadata: Metadata = {
+  title: "Page not found - DocumentDB",
+  description:
+    "The page you're looking for doesn't exist. Find documentation, downloads, and samples for DocumentDB.",
+  robots: {
+    index: false,
+    follow: true,
+  },
+};
+
 const suggestions = [
   {
     title: "Documentation",

articles/content.yml

@@ -12,7 +12,7 @@ landing:
       link: /docs/postgres-api
     - title: DocumentDB Local
       link: /docs/documentdb-local
-    - title: Architecture under the hood
+    - title: Architecture under the hood (Coming soon)
       link: /docs/architecture
     - title: Samples & Demos
       link: /samples

scripts/generate-sitemap.mjs

@@ -11,21 +11,41 @@ import path from 'node:path';
 const siteUrl = 'https://documentdb.io';
 const outDir = path.join(process.cwd(), 'out');
 
-// Top-level build outputs that are not HTML pages: Next.js assets, the APT/RPM
-// package repositories, images, and the not-found page (with trailingSlash the
-// export emits out/404/index.html alongside out/404.html). The packages
-// workflow adds deb/ and rpm/ after this script runs in the deploy job, but
-// they are excluded here too so local full builds behave identically. Note
-// that out/packages/ is NOT excluded: it is the exported /packages download
-// page; the workflow only adds release-info.json (not a page) next to it.
+// Top-level build outputs that are not pages at all: Next.js assets, the
+// APT/RPM package repositories, and images. The packages workflow adds deb/
+// and rpm/ after this script runs in the deploy job, but they are excluded
+// here too so local full builds behave identically. Note that out/packages/ is
+// NOT excluded: it is the exported /packages download page; the workflow only
+// adds release-info.json (not a page) next to it.
+//
+// Pages that exist but must not be advertised are handled by isNoindex()
+// rather than by name. That covers both forms the not-found route takes - with
+// trailingSlash the export writes out/404/index.html alongside out/404.html,
+// and the App Router adds out/_not-found/index.html - and anything else the
+// framework starts emitting later. Listing a noindex URL is what earns the
+// "submitted URL marked noindex" warning in Search Console, and a page already
+// states that about itself, so there is no second list to keep in sync.
 const excludedTopLevelDirectories = new Set([
   '_next',
   'deb',
   'rpm',
   'images',
-  '404',
 ]);
 
+let noindexPagesSkipped = 0;
+
+/**
+ * True when the page asks crawlers not to index it. Reads the meta tag in
+ * either attribute order and tolerates content lists such as "noindex,nofollow".
+ */
+function isNoindex(html) {
+  return (html.match(/<meta\b[^>]*>/gi) ?? []).some(
+    (tag) =>
+      /\bname=["']?robots["']?/i.test(tag) &&
+      /\bcontent=["'][^"']*\bnoindex\b/i.test(tag),
+  );
+}
+
 function xmlEscape(value) {
   return value
     .replace(/&/g, '&amp;')
@@ -45,10 +65,14 @@ function collectPages(directory, relativePath = '') {
   const indexFile = path.join(directory, 'index.html');
 
   if (fs.existsSync(indexFile)) {
-    pages.push({
-      url: relativePath === '' ? '/' : `/${relativePath}/`,
-      lastModified: fs.statSync(indexFile).mtime,
-    });
+    if (isNoindex(fs.readFileSync(indexFile, 'utf8'))) {
+      noindexPagesSkipped += 1;
+    } else {
+      pages.push({
+        url: relativePath === '' ? '/' : `/${relativePath}/`,
+        lastModified: fs.statSync(indexFile).mtime,
+      });
+    }
   }
 
   for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
@@ -113,4 +137,8 @@ if (fs.existsSync(robotsPath)) {
   fs.writeFileSync(robotsPath, `User-agent: *\nAllow: /\n\n${sitemapLine}\n`);
 }
 
-console.log(`Wrote out/sitemap.xml with ${pages.length} URLs and advertised it in out/robots.txt.`);
+console.log(
+  `Wrote out/sitemap.xml with ${pages.length} URLs (skipped ${noindexPagesSkipped} noindex ${
+    noindexPagesSkipped === 1 ? 'page' : 'pages'
+  }) and advertised it in out/robots.txt.`,
+);