Merge #0011

src/main/config/compileDb.ts

@@ -28,6 +28,19 @@ const COMPILE_DB_CONFIG = {
      */
     maxOutputChars: 8000,
 
+    /**
+     * How many source diagnostics are kept out of a failed run, one broken header
+     * can cascade into a wall of them and the first few are what get fixed.
+     */
+    maxDiagnostics: 10,
+
+    /**
+     * Matches how the build tool and the header tool report a complaint against a
+     * file, `path(line): error: message` with an optional column and an optional
+     * compiler error code, in either capitalization.
+     */
+    diagnosticPattern: /^(.*?)\((\d+)(?:,\s*\d+)?\)\s*:\s*(?:fatal\s+)?error\s*(?:[A-Z]+\d+)?\s*:\s*(.+)$/gim,
+
     /**
      * The engine-relative path of the build entry point, keyed by
      * `process.platform`.

src/main/lib/compileDb/run.ts

@@ -1,8 +1,15 @@
 import { execFile } from "node:child_process"
+import path from "node:path"
 import { promisify } from "node:util"
 import COMPILE_DB_CONFIG from "@main/config/compileDb"
+import { toPosixPath } from "@main/lib/compileDb/paths"
 import CONSTANTS from "@main/lib/constants"
-import type { CompileDbContext, CompileDbRunKind, CompileDbRunResult } from "@/main/types/compileDb"
+import type {
+    CompileDbContext,
+    CompileDbDiagnostic,
+    CompileDbRunKind,
+    CompileDbRunResult,
+} from "@/main/types/compileDb"
 
 /**
  * Promisified version of `child_process.execFile`.
@@ -190,6 +197,36 @@ function buildSteps(kind: CompileDbRunKind, context: CompileDbContext): CompileD
     return [buildSolutionStep(context), buildWorkspaceStep(context), buildClangDatabaseStep(context)]
 }
 
+/**
+ * Pulls the build tool's complaints about individual source files out of a failed
+ * run, so a failure the project's own C++ caused reads as the diagnostic it is
+ * rather than as a wall of build-tool logging.
+ * @param output The failed step's captured output.
+ * @param projectRoot The absolute repository root, trimmed off each path.
+ * @returns The diagnostics, deduplicated and capped.
+ */
+export function collectDiagnostics(output: string, projectRoot: string): CompileDbDiagnostic[] {
+    const collected = new Map<string, CompileDbDiagnostic>()
+
+    for (const match of output.matchAll(COMPILE_DB_CONFIG.diagnosticPattern)) {
+        const [, rawFile, rawLine, rawMessage] = match
+
+        // A path the build tool reports from outside the project (the engine's own
+        // headers) has no shorter form worth showing, so it is kept whole
+        const relativeFile = path.relative(projectRoot, rawFile.trim())
+        const file = !relativeFile || relativeFile.startsWith("..") ? rawFile.trim() : relativeFile
+
+        // The same complaint is routinely logged more than once in a single run
+        collected.set(`${file}:${rawLine}:${rawMessage}`, {
+            file: toPosixPath(file),
+            line: Number(rawLine),
+            message: rawMessage.trim(),
+        })
+    }
+
+    return [...collected.values()].slice(0, COMPILE_DB_CONFIG.maxDiagnostics)
+}
+
 /**
  * Runs a regeneration end to end, stopping at the first invocation that exits
  * non-zero so a broken solution step cannot be followed by a database written
@@ -220,6 +257,7 @@ export async function runRegeneration(
                 ok: false,
                 exitCode,
                 failedStep: step.label,
+                diagnostics: collectDiagnostics(output, context.projectRoot),
                 output: trimTail(outputs.join("\n\n")),
                 durationMs: Date.now() - startedAt,
             }
@@ -231,6 +269,7 @@ export async function runRegeneration(
         ok: true,
         exitCode: 0,
         failedStep: null,
+        diagnostics: [],
         output: trimTail(outputs.join("\n\n")),
         durationMs: Date.now() - startedAt,
     }

src/main/lib/compileDb/service.ts

@@ -96,6 +96,16 @@ export class CompileDbService {
         }
     }
 
+    /**
+     * Whether the last run left the database out of date, either because the build
+     * tool refused the project's sources or because it could not be run at all.
+     * @returns True when the last run did not land a database.
+     */
+    private _hasFailed(): boolean {
+        const { lastResult, lastError } = compileDbStore.get()
+        return Boolean(lastError) || lastResult?.ok === false
+    }
+
     /**
      * Resolves everything a run needs and caches it, the engine folder comes from
      * the preference override or from wherever the platform records its installs,
@@ -198,7 +208,11 @@ export class CompileDbService {
         this._knownFiles = next
         compileDbStore.set({ trackedFileCount: next.size })
 
-        if (!hasChanged) return
+        // A failed run leaves the database out of date, and what fixes the usual
+        // cause (a header still half-written when it first appeared) is an edit
+        // rather than another file appearing, so a failure is worth retrying on any
+        // change at all rather than waiting for the set to move again
+        if (!hasChanged && !this._hasFailed()) return
 
         // A build or plugin descriptor can move the solution too, which is left to
         // the user rather than rewritten under an open IDE

src/main/types/compileDb.d.ts

@@ -24,16 +24,28 @@ export type CompileDbContext = {
     targetPlatform: string
 }
 
+/**
+ * One complaint the build tool made about a source file, parsed out of its output
+ * so a failure caused by the project's own C++ can be told apart from one caused
+ * by GitGame or by the engine install, and reported as the diagnostic it is.
+ */
+export type CompileDbDiagnostic = {
+    file: string
+    line: number
+    message: string
+}
+
 /**
  * The outcome of a single regeneration, `output` is the tail of the combined
- * streams so a failure can be read without opening a terminal, and `failedStep`
- * names which of the steps in a `full` run gave up.
+ * streams, `failedStep` names which step of a `full` run gave up, and a non-empty
+ * `diagnostics` marks the failure as the project's own rather than ours.
  */
 export type CompileDbRunResult = {
     kind: CompileDbRunKind
     ok: boolean
     exitCode: number
     failedStep: string | null
+    diagnostics: CompileDbDiagnostic[]
     output: string
     durationMs: number
 }

src/renderer/components/fields/StatusCompileDb.tsx

@@ -1,6 +1,11 @@
 import batExecIcon from "@react95-icons/BatExec_16x16_4.png"
 import useCompileDbState from "@renderer/hooks/useCompileDbState"
-import { buildCompileDbFailureDetails, buildCompileDbLabel } from "@renderer/lib/utils/compileDb"
+import {
+    buildCompileDbFailureDetails,
+    buildCompileDbFailureMessage,
+    buildCompileDbFailureTitle,
+    buildCompileDbLabel,
+} from "@renderer/lib/utils/compileDb"
 import { useCallback, useEffect, useMemo, useState } from "react"
 import StatusBarFrame, { type StatusBarLabelState } from "@/renderer/components/frames/StatusBar"
 import TileProgressBar from "@/renderer/components/ui/TileProgressBar"
@@ -55,8 +60,8 @@ export default function StatusCompileDbField({ className }: StatusCompileDbField
 
         if (hasFailed) {
             window.api.dialogs.errorWithDetails(
-                "Compile database: the last regeneration failed",
-                state.lastError ?? "The engine's build tool exited with an error.",
+                buildCompileDbFailureTitle(state),
+                buildCompileDbFailureMessage(state),
                 buildCompileDbFailureDetails(state),
             )
 

src/renderer/lib/utils/compileDb.ts

@@ -1,5 +1,15 @@
-import dedent from "dedent"
-import type { CompileDbState } from "@/main/types/compileDb"
+import type { CompileDbDiagnostic, CompileDbState } from "@/main/types/compileDb"
+
+/**
+ * Whether the last failure was the project's own C++ being refused rather than
+ * GitGame or the engine install failing to run at all, which are worth telling
+ * apart because only one of them is ours to answer for.
+ * @param state The current compile-database state.
+ * @returns True when the failure carries source diagnostics.
+ */
+export function hasSourceDiagnostics(state: CompileDbState): boolean {
+    return Boolean(state.lastResult && !state.lastResult.ok && state.lastResult.diagnostics.length > 0)
+}
 
 /**
  * Builds the short label shown in the status bar chip, names the step while a run
@@ -12,28 +22,68 @@ export function buildCompileDbLabel(state: CompileDbState): string {
     if (state.isRunning) return state.runningStep ?? "Running"
     if (state.lastError) return "Error"
 
-    if (state.lastResult && !state.lastResult.ok) return "Failed"
+    if (state.lastResult && !state.lastResult.ok) {
+        return hasSourceDiagnostics(state) ? "Code error" : "Failed"
+    }
+
     if (state.isFullRegenSuggested) return "Full regen"
     if (state.lastResult) return `${(state.lastResult.durationMs / 1000).toFixed(1)}s`
 
     return `${state.trackedFileCount} files`
 }
 
 /**
- * Builds the details block shown when the chip is clicked on a failure, pairs the
- * step that gave up with the tail of the build tool's own output.
+ * Builds the title of the dialog a failure opens, naming which side of the line
+ * the failure fell on so the project's own C++ is never reported as ours.
+ * @param state The current compile-database state.
+ * @returns The dialog title.
+ */
+export function buildCompileDbFailureTitle(state: CompileDbState): string {
+    if (hasSourceDiagnostics(state)) return "Compile database: C++ errors found"
+    return "Compile database: regeneration failed"
+}
+
+/**
+ * Builds the message line of the dialog a failure opens.
+ * @param state The current compile-database state.
+ * @returns The dialog message.
+ */
+export function buildCompileDbFailureMessage(state: CompileDbState): string {
+    if (hasSourceDiagnostics(state)) {
+        const { diagnostics } = state.lastResult ?? { diagnostics: [] }
+
+        return `The engine's build tool ran but would not accept the project's sources, it reported ${diagnostics.length} problem${diagnostics.length === 1 ? "" : "s"}. The database is left as it was, and GitGame retries as soon as the sources change again.`
+    }
+
+    return state.lastError ?? "The engine's build tool exited with an error before writing the database."
+}
+
+/**
+ * Formats the build tool's complaints one per block, path and line first so the
+ * file can be opened straight from the dialog.
+ * @param diagnostics The diagnostics from the failed run.
+ * @returns The formatted block.
+ */
+function formatDiagnostics(diagnostics: CompileDbDiagnostic[]): string {
+    return diagnostics.map(diagnostic => `${diagnostic.file}(${diagnostic.line})\n  ${diagnostic.message}`).join("\n\n")
+}
+
+/**
+ * Builds the details block shown when the chip is clicked on a failure, the source
+ * diagnostics on their own when the project's C++ was the problem, and the tail of
+ * the build tool's own output when it was not.
  * @param state The current compile-database state.
  * @returns The formatted details block.
  */
 export function buildCompileDbFailureDetails(state: CompileDbState): string {
-    if (state.lastError) return state.lastError
-
-    if (!state.lastResult) return "No further details available."
+    if (!state.lastResult) return state.lastError ?? "No further details available."
 
-    return dedent`
-        Step: ${state.lastResult.failedStep ?? "unknown"}
-        Exit code: ${state.lastResult.exitCode}
+    if (hasSourceDiagnostics(state)) return formatDiagnostics(state.lastResult.diagnostics)
 
-        ${state.lastResult.output || "The build tool produced no output."}
-    `
+    return [
+        `Step: ${state.lastResult.failedStep ?? "unknown"}`,
+        `Exit code: ${state.lastResult.exitCode}`,
+        "",
+        state.lastResult.output || "The build tool produced no output.",
+    ].join("\n")
 }

tests/lib/utils/compileDb.ts

@@ -1,3 +1,4 @@
+import dedent from "dedent"
 import type { CompileDbFileKind } from "@/main/types/compileDb"
 
 /**
@@ -9,3 +10,21 @@ import type { CompileDbFileKind } from "@/main/types/compileDb"
 export function buildScan(entries: [string, CompileDbFileKind][]): Map<string, CompileDbFileKind> {
     return new Map(entries)
 }
+
+/**
+ * The tail of a real failed run, the header tool refusing a half-written header,
+ * kept verbatim so the parser is held to what the build tool really prints.
+ */
+export const HEADER_TOOL_FAILURE_OUTPUT = dedent`
+    Using bundled DotNet SDK version: 10.0 win-x64
+    Running UnrealBuildTool: dotnet "..\\..\\Engine\\Binaries\\DotNET\\UnrealBuildTool\\UnrealBuildTool.dll" -mode=GenerateClangDatabase
+    Creating target...
+    Clang compiler version 22.1.3 is newer than latest preferred version 20.1.8. Please use caution as this compiler has not been heavily tested
+    Log file: C:\\Users\\someone\\AppData\\Local\\UnrealHeaderTool\\Saved\\Logs\\UnrealHeaderTool.log
+    UHT compiled-in object format Default
+    F:\\Projects\\Cybearl\\project-windy\\Plugins\\WindyToolkit\\Source\\WindyToolkit\\Public\\Wind\\WindyWindFieldTypes.h(13): Error: Found EOF when expecting public while parsing public access modifier in class 'UWindyWindFieldTypes'
+    Unhandled 1 aggregate exceptions
+
+    Result: Failed (OtherCompilationError)
+    Total execution time: 1.79 seconds
+`

tests/unit/compileDb.test.ts

@@ -1,6 +1,7 @@
 import { classifySourcePath } from "@main/lib/compileDb/paths"
+import { collectDiagnostics } from "@main/lib/compileDb/run"
 import { diffSourceFiles } from "@main/lib/compileDb/scan"
-import { buildScan } from "@tests/lib/utils/compileDb"
+import { buildScan, HEADER_TOOL_FAILURE_OUTPUT } from "@tests/lib/utils/compileDb"
 import { describe, expect, it } from "vitest"
 
 describe("classifySourcePath", () => {
@@ -59,6 +60,57 @@ describe("classifySourcePath", () => {
     })
 })
 
+describe("collectDiagnostics", () => {
+    const projectRoot = "F:\\Projects\\Cybearl\\project-windy"
+
+    it("pulls the header tool's complaint out of a real failed run", () => {
+        expect(collectDiagnostics(HEADER_TOOL_FAILURE_OUTPUT, projectRoot)).toEqual([
+            {
+                file: "Plugins/WindyToolkit/Source/WindyToolkit/Public/Wind/WindyWindFieldTypes.h",
+                line: 13,
+                message:
+                    "Found EOF when expecting public while parsing public access modifier in class 'UWindyWindFieldTypes'",
+            },
+        ])
+    })
+
+    it("reads the compiler's own formats, with a column and with an error code", () => {
+        const output = [
+            "F:\\Projects\\Cybearl\\project-windy\\Source\\Thing.cpp(42,9): error: use of undeclared identifier 'foo'",
+            "F:\\Projects\\Cybearl\\project-windy\\Source\\Other.cpp(7): fatal error C1083: Cannot open include file",
+        ].join("\n")
+
+        expect(collectDiagnostics(output, projectRoot)).toEqual([
+            { file: "Source/Thing.cpp", line: 42, message: "use of undeclared identifier 'foo'" },
+            { file: "Source/Other.cpp", line: 7, message: "Cannot open include file" },
+        ])
+    })
+
+    it("keeps a path from outside the project whole, there being no shorter form", () => {
+        const output = "C:\\UE_5.8\\Engine\\Source\\Runtime\\Core\\Public\\Thing.h(9): Error: broken"
+
+        expect(collectDiagnostics(output, projectRoot)).toEqual([
+            { file: "C:/UE_5.8/Engine/Source/Runtime/Core/Public/Thing.h", line: 9, message: "broken" },
+        ])
+    })
+
+    it("collapses a complaint the build tool logged more than once", () => {
+        const line = "F:\\Projects\\Cybearl\\project-windy\\Source\\Thing.cpp(42): error: broken"
+
+        expect(collectDiagnostics([line, line, line].join("\n"), projectRoot)).toHaveLength(1)
+    })
+
+    it("finds nothing in output that only carries warnings or a plain summary", () => {
+        const output = [
+            "F:\\Projects\\Cybearl\\project-windy\\Source\\Thing.cpp(42): Warning: shadowed variable",
+            "Result: Succeeded",
+            "Total execution time: 1.41 seconds",
+        ].join("\n")
+
+        expect(collectDiagnostics(output, projectRoot)).toEqual([])
+    })
+})
+
 describe("diffSourceFiles", () => {
     const baseline = buildScan([
         ["Source/ProjectWindy/ProjectWindy.cpp", "source"],