From 0738ef7a65ebbd30e1760d2e334c1f39ce733c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Sch=C3=A4fer?= Date: Mon, 26 Oct 2026 17:37:02 +0100 Subject: [PATCH] Color F# and C# cells by what their names resolve to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cell editor was colored by Monaco's lexer alone, which sees a type, a function, a module and a value as the same identifier. Kernels can now classify the symbols in a cell, and the editor draws those classifications as Monaco semantic tokens, in `verso serve` and in VS Code. ISemanticTokenProvider is an optional interface beside ILanguageKernel, so no existing kernel changes. GetSemanticTokensAsync returns tokens typed by the Language Server Protocol's standard names, which Abstractions holds as constants (SemanticTokenTypes, SemanticTokenModifiers), or null when the code could not be analyzed, in which case the editor keeps the colors it shows. Positions are 0-based and relative to the cell, as for a Diagnostic, and a token's Length counts UTF-16 characters. Verso.Host serves it as kernel/getSemanticTokens, and both notebook services pass it through; a kernel that does not classify answers empty and is not started for the request. The editor asks for diagnostics and for semantic tokens of the same text moments apart, and a kernel that provides both answers them from one check of that text, so a pause in typing compiles once. The F# checker keeps its last check, keyed by the text, the compiler options and the #load'ed scripts with their write times. The C# workspace keeps the last document it built for a cell's text, and with it the compilation; a run or a new reference drops it. The F# kernel classifies with FCS's GetSemanticClassification over the cell's own lines, shifted past the cells executed before it. A dotted name is classified by FCS as a whole, by its last part, so List.map arrives as one function; each range is cut back to its last identifier, which leaves List to its own classification as a module. The C# kernel uses Roslyn's Classifier, and reports a top-level variable, which Roslyn classifies as a field of the script class, as the variable it reads as. In the editor, a document semantic tokens provider follows the diagnostics rules: nothing is asked while a cell runs, a cell is asked again after a run when its diagnostics are, and requests from all editors go one at a time, so a run in VS Code waits behind at most one. A cell whose text is unchanged since its answer gets that answer back without a request. The theme builder colors each standard token type by the syntax color the theme has for it. Verso's light, dark and high-contrast themes gain an enumMember color for enum members and union cases, and a theme without one colors them by its type color. In the VS Code webview the editor takes its syntax colors from the active VS Code theme, and the semantic token types its lexical rules do not name (class, struct, enum, interface, typeParameter, parameter, property, enumMember, event, method) would fall to the editor's text color. Each is colored as VS Code colors it in a source file: by the theme's semanticTokenColors, laid over its included themes, and the user's editor.semanticTokenColorCustomizations, where a selector names the type or the type with a modifier ("variable.mutable", "*.static"), and otherwise by the TextMate scope VS Code falls back to for that type (entity.name.type.struct, variable.other.enummember, ...). Types the lexers share keep their lexical rule, which already uses the same scopes. Selectors limited to a language are not supported. VS Code shows semantic tokens only when editor.semanticHighlighting.enabled says so or, as by default, when the theme or the user's customizations do; the notebook follows the same decision, and turns its semantic tokens off with a theme that has none. Signed-off-by: Jonathan Schäfer --- README.md | 2 +- docs/architecture/front-ends.md | 10 +- docs/extensions/extension-interfaces.md | 20 +- docs/extensions/theme-authoring.md | 9 +- .../Extensions/ISemanticTokenProvider.cs | 27 +++ .../Models/SemanticToken.cs | 74 ++++++ .../Components/Editor/MonacoEditor.razor | 19 +- .../Components/Notebook/Cell.razor | 28 +++ src/Verso.Blazor.Shared/Models/MonacoTheme.cs | 27 ++- src/Verso.Blazor.Shared/Models/ServiceDtos.cs | 14 ++ .../Services/INotebookService.cs | 12 + .../wwwroot/js/monaco-interop.js | 222 +++++++++++++++++- .../Services/RemoteNotebookService.cs | 29 +++ .../wwwroot/js/vscode-bridge.js | 4 +- .../Services/ServerNotebookService.cs | 32 +++ .../Helpers/SemanticTokenMapper.cs | 135 +++++++++++ .../Kernel/FSharpCheckerManager.cs | 63 ++++- src/Verso.FSharp/Kernel/FSharpKernel.cs | 42 +++- src/Verso.FSharp/README.md | 1 + src/Verso.Host/Dto/NotebookDto.cs | 20 ++ src/Verso.Host/Handlers/KernelHandler.cs | 39 +++ src/Verso.Host/HostSession.cs | 1 + src/Verso.Host/Protocol/MethodNames.cs | 1 + src/Verso/Extensions/Themes/VersoDarkTheme.cs | 1 + .../Themes/VersoHighContrastTheme.cs | 1 + .../Extensions/Themes/VersoLightTheme.cs | 1 + src/Verso/Kernels/CSharpKernel.cs | 14 +- src/Verso/Kernels/RoslynWorkspaceManager.cs | 109 ++++++++- tests/Verso.Blazor.Shared.Tests/CellTests.cs | 35 +++ .../Fakes/FakeNotebookService.cs | 12 + .../MonacoEditorTests.cs | 26 ++ .../MonacoThemeBuilderTests.cs | 36 +++ .../ServerNotebookServiceTests.cs | 33 +++ .../Helpers/SemanticTokenMapperTests.cs | 115 +++++++++ .../Kernel/CheckerManagerTests.cs | 92 ++++++++ .../Kernel/ExecutedCellsTests.cs | 15 ++ .../Kernel/SemanticTokensTests.cs | 130 ++++++++++ tests/Verso.Host.Tests/HandlerTests.cs | 131 ++++++++++- tests/Verso.Host.Tests/JsonRpcMessageTests.cs | 1 + .../CSharpKernelSemanticTokensTests.cs | 109 +++++++++ .../Kernels/RoslynWorkspaceManagerTests.cs | 57 +++++ vscode/src/blazor/activeColorTheme.ts | 84 +++++-- vscode/src/blazor/blazorEditorProvider.ts | 4 +- vscode/src/blazor/themeTokenRules.ts | 132 +++++++++++ vscode/test/suite/protocol.test.ts | 1 + vscode/test/suite/themeTokenRules.test.ts | 67 ++++++ 46 files changed, 1985 insertions(+), 52 deletions(-) create mode 100644 src/Verso.Abstractions/Extensions/ISemanticTokenProvider.cs create mode 100644 src/Verso.Abstractions/Models/SemanticToken.cs create mode 100644 src/Verso.FSharp/Helpers/SemanticTokenMapper.cs create mode 100644 tests/Verso.FSharp.Tests/Helpers/SemanticTokenMapperTests.cs create mode 100644 tests/Verso.FSharp.Tests/Kernel/CheckerManagerTests.cs create mode 100644 tests/Verso.FSharp.Tests/Kernel/SemanticTokensTests.cs create mode 100644 tests/Verso.Tests/Kernels/CSharpKernelSemanticTokensTests.cs create mode 100644 tests/Verso.Tests/Kernels/RoslynWorkspaceManagerTests.cs diff --git a/README.md b/README.md index 879b0887..8f46ffd6 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ The architecture is built on one principle: every feature is an extension, and e ![A C# cell with the completion list open, listing members of a user-defined list](docs/images/intellisense.png) -Every language kernel answers completions and hover from a real language service rather than by text matching: Roslyn for C#, FSharp.Compiler.Service for F#, a live runspace for PowerShell, and your own interpreter for Python. Kernels also compute diagnostics ahead of execution, which hosts and tools reach through `ILanguageKernel.GetDiagnosticsAsync`; errors from a run are reported as cell output. +Every language kernel answers completions and hover from a real language service rather than by text matching: Roslyn for C#, FSharp.Compiler.Service for F#, a live runspace for PowerShell, and your own interpreter for Python. Kernels also compute diagnostics ahead of execution, which hosts and tools reach through `ILanguageKernel.GetDiagnosticsAsync`; errors from a run are reported as cell output. The C# and F# kernels also classify the symbols in a cell, so the editor colors a type, a function and a value differently. NuGet packages are referenced inline with `#r "nuget: PackageName, Version"`, and custom package sources are supported with `#i "nuget: "`. A plain version is an exact pin; floating versions such as `1.*` or `*-*` and ranges such as `[1.0.0,2.0.0)` follow NuGet's own rules, and leaving the version out picks the latest stable release. Python uses `#!pip` for package management, and JavaScript uses `#!npm` for npm packages. State persists across cells within each kernel, and variables are shared across kernels through a central variable store. diff --git a/docs/architecture/front-ends.md b/docs/architecture/front-ends.md index 5ca5742e..da438a06 100644 --- a/docs/architecture/front-ends.md +++ b/docs/architecture/front-ends.md @@ -46,6 +46,14 @@ The refresh policy is set by the expensive kernels, where a request is a full co While any cell is running no request is issued (`NotebookCellContext.IsAnyExecuting`, passed to the editor as `DiagnosticsSuspended`). `Verso.Host` answers requests one at a time, so a request during a run would wait for the run, and some kernels answer empty while executing. A refresh asked for in that time runs once the run is over, in the same top-to-bottom order. Run All is one `execution/runAll` request, so in VS Code the cells are refreshed once it has finished, not one by one as it goes. One request is in flight per editor; a reply for text that has since changed is dropped; a failed request (null) keeps the markers that are showing, while an empty list clears them. Cells over 2,000 lines are not analyzed. +#### Editor semantic tokens + +Semantic tokens are pulled, as hover is: `monaco-interop.js` registers a document semantic tokens provider per language, and Monaco asks for a model's tokens when it is attached, after an edit (debounced by Monaco, from 300 ms up as answers take longer), when the theme changes, and when the provider raises `onDidChange`. The provider calls `MonacoEditor.GetSemanticTokens`, which goes through `Cell.razor` to `INotebookService.GetSemanticTokensAsync`; `ServerNotebookService` calls the kernel in process if it implements `ISemanticTokenProvider`, and `RemoteNotebookService` sends `kernel/getSemanticTokens` to `Verso.Host`. A kernel that does not classify answers an empty list and is not started for it. + +The provider keeps each model's last answer and the version of the text it was for. Monaco asks for every model when `onDidChange` is raised, and a model whose text has not changed since its answer gets that answer back without a request. The provider raises it when a diagnostics request goes out for text or kernel state that no semantic request has been sent for: after a run, for the cells the run refreshes, and on focus after another cell ran. Monaco passes the id of the answer it shows; a model already showing its last answer keeps it rather than having it painted again. An edit sends both requests, and the diagnostics request does not cause a second semantic one. Requests from all editors are queued and sent one at a time, so in VS Code a run waits behind at most one of them. + +While any cell is running, a request is refused with Monaco's "busy" error, which keeps the tokens shown, and is asked again when the run is over. A failed request (null) is refused the same way; an empty list clears the tokens. Cells over 2,000 lines are not analyzed. Semantic highlighting is turned on per editor (`'semanticHighlighting.enabled': true`), since standalone themes leave it off. Monaco styles a token by the theme rule whose name matches its type with the modifiers appended (`variable.mutable` falls back to `variable`); `MonacoThemeBuilder` adds a rule for each standard token type under the theme's syntax color for it. + Monaco's bundled TypeScript worker would otherwise mark JavaScript and TypeScript cells as if each were a standalone file, so its validation is turned off when Monaco loads; its completion and hover stay on. Isolated (iframe) layouts need more than the base notebook operations, so hosts also implement `IIsolatedLayoutHost` (an extension of `INotebookService` in `Verso.Blazor.Shared.Services`). It covers frame allocation, renderer-package fetch, mount and unmount lifecycle, the message pump between the frame and the kernel, and theme propagation. Blazor Server implements it in-process; Blazor WASM implements it over JSON-RPC. @@ -247,7 +255,7 @@ Kernel calls context.RequestInputAsync(prompt, isPassword, ct) | `NotebookHandler` | open, close, save, getLanguages, getCellTypes, setFilePath | | `CellHandler` | add, insert, remove, move, updateSource, changeType, changeLanguage | | `ExecutionHandler` | run, runAll, cancel | -| `KernelHandler` | restart, getCompletions, getDiagnostics, getHoverInfo | +| `KernelHandler` | restart, getCompletions, getDiagnostics, getHoverInfo, getSemanticTokens | | `OutputHandler` | clearAll | | `InteractionHandler` | interact (cell interaction dispatch) | | `LayoutHandler` | getLayouts, switch, render, getCellContainer | diff --git a/docs/extensions/extension-interfaces.md b/docs/extensions/extension-interfaces.md index b576edc9..24f0fd44 100644 --- a/docs/extensions/extension-interfaces.md +++ b/docs/extensions/extension-interfaces.md @@ -674,7 +674,7 @@ Interaction types in the `verso/` namespace are reserved by the host and are int ## Augmentation Interfaces -These interfaces are not standalone extension capabilities. They are implemented **alongside** a primary capability interface (e.g., `ILanguageKernel + IExtensionSettings` or `IDataFormatter + ICellInteractionHandler`). They do not extend `IExtension` and cannot be the sole interface on a `[VersoExtension]` class. +These interfaces are not standalone extension capabilities. They are implemented **alongside** a primary capability interface (e.g., `ILanguageKernel + IExtensionSettings`, `ILanguageKernel + ISemanticTokenProvider` or `IDataFormatter + ICellInteractionHandler`). They do not extend `IExtension` and cannot be the sole interface on a `[VersoExtension]` class. ### IExtensionSettings @@ -755,6 +755,24 @@ public sealed class MyKernel : ILanguageKernel, IExtensionSettings } ``` +### ISemanticTokenProvider + +Classifies the symbols in a cell, so the editor can color a type, a function and a value differently where its lexer sees the same identifier. Implemented alongside `ILanguageKernel`; the C# and F# kernels implement it. + +| Member | Type | Description | +|---|---|---| +| `GetSemanticTokensAsync(string)` | `Task?>` | Classifies the cell's source. `null` when the code could not be analyzed. | + +A `SemanticToken` is `(Line, StartColumn, Length, Type, Modifiers)`. Positions are 0-based and relative to the cell's source, as for a `Diagnostic`; `Length` counts UTF-16 characters, as .NET strings and the editor do, and a token lies on one line. Tokens may come in any order; the editor sorts them and drops a token that overlaps the one before it, one past the end of its line, and one of a type it does not know. + +`Type` is one of the `SemanticTokenTypes` constants, which are the Language Server Protocol's standard names: `namespace`, `type`, `class`, `struct`, `enum`, `interface`, `typeParameter`, `parameter`, `variable`, `property`, `enumMember`, `event`, `function`, `method`, `keyword` and `operator`. `Modifiers` holds `SemanticTokenModifiers` (`mutable`, `static`) or is `null`. The theme colors each type by one of its syntax colors (see [Theme Authoring](theme-authoring.md#syntax-color-mapping)). + +Report what the lexer cannot know and leave out what it already colors. A keyword, a string or a number gains nothing from a token, and a token whose type the theme does not color is drawn in the editor's foreground color, which can be worse than the lexer's guess. A computation expression builder in F# (`async`, `seq`) is the exception: the lexer reads it as an identifier, and a `keyword` token gives it the keyword color. + +The editor asks when a cell is first shown, after the user pauses typing, and after a run, for the cell that ran and every cell below it that shows kernel markers; it asks again on focus when another cell has run since the last answer. It does not ask while any cell in the notebook is running, and the colors already shown stay until the next answer. A request goes out from one editor at a time. Like diagnostics, the request is advisory: `null`, an exception or a kernel that cannot start keeps the colors shown, and an empty list clears them. Leading `#!` magic lines are blanked before the code reaches the kernel. + +A classification usually falls out of the same type check as `GetDiagnosticsAsync`, and the editor asks for both after an edit. A kernel that implements both should answer them from one check of the same text, so a pause in typing compiles once; the C# and F# kernels do. + ### ICellInteractionHandler Handles bidirectional interactions from rendered cell content. Implement alongside a formatter or renderer to support interactive outputs (e.g., paginated tables, inline forms, drill-down panels). diff --git a/docs/extensions/theme-authoring.md b/docs/extensions/theme-authoring.md index e23f1b4f..060df52f 100644 --- a/docs/extensions/theme-authoring.md +++ b/docs/extensions/theme-authoring.md @@ -248,7 +248,7 @@ How much work the shadow has to do depends on the palette. Dark themes separate ## Syntax Color Mapping -Themes provide syntax highlighting via `GetSyntaxColors()`, which returns a `SyntaxColorMap`. The platform recognizes 12 standard token types: +Themes provide syntax highlighting via `GetSyntaxColors()`, which returns a `SyntaxColorMap`. The platform recognizes 13 standard token types: | Token Type | Description | Light Default | Dark Default | |------------|-------------|--------------|-------------| @@ -263,9 +263,12 @@ Themes provide syntax highlighting via `GetSyntaxColors()`, which returns a `Syn | `punctuation` | Brackets, semicolons | `#000000` | `#D4D4D4` | | `preprocessor` | Preprocessor directives | `#808080` | `#C586C0` | | `attribute` | Attributes and decorators | `#267F99` | `#4EC9B0` | -| `namespace` | Namespace references | `#267F99` | `#4EC9B0` | +| `namespace` | Namespace and module references | `#267F99` | `#4EC9B0` | +| `enumMember` | Enum members, union cases and named constants | `#0070C1` | `#4FC1FF` | -Always provide all 12 tokens. Renderers may fall back to `EditorForeground` for missing tokens. +Always provide all 13 tokens. Renderers may fall back to `EditorForeground` for missing tokens. + +The same colors apply to the semantic tokens a kernel reports (see [`ISemanticTokenProvider`](extension-interfaces.md#isemantictokenprovider)). A class, struct, enum, interface or type parameter takes the `type` color; a method takes `function`; a parameter, property or event takes `variable`. A theme without `enumMember` colors enum members and union cases by its `type` color. ## Overriding Defaults diff --git a/src/Verso.Abstractions/Extensions/ISemanticTokenProvider.cs b/src/Verso.Abstractions/Extensions/ISemanticTokenProvider.cs new file mode 100644 index 00000000..49d18923 --- /dev/null +++ b/src/Verso.Abstractions/Extensions/ISemanticTokenProvider.cs @@ -0,0 +1,27 @@ +namespace Verso.Abstractions; + +/// +/// Augmentation interface for language kernels that can classify the symbols in a cell, so the +/// editor can color a type, a function and a value differently where a lexer alone sees the same +/// identifier. A kernel implements this alongside . Implementing +/// alone is not a valid extension capability. +/// +/// +/// The editor asks after the user pauses typing and when the cell's kernel state may have changed +/// (its run finished, the kernel restarted), and keeps the colors it already shows while a cell is +/// running. A classification is usually a by-product of the type check that +/// runs on the same text, and the editor asks for +/// both after an edit, so a kernel answers both from one check of that text. +/// +public interface ISemanticTokenProvider +{ + /// + /// Classifies the symbols in . + /// + /// The full source text of the cell. + /// + /// The cell's tokens, in any order, or null when the code could not be analyzed. The + /// editor keeps the colors it already shows on null; an empty list clears them. + /// + Task?> GetSemanticTokensAsync(string code); +} diff --git a/src/Verso.Abstractions/Models/SemanticToken.cs b/src/Verso.Abstractions/Models/SemanticToken.cs new file mode 100644 index 00000000..0cfd8478 --- /dev/null +++ b/src/Verso.Abstractions/Models/SemanticToken.cs @@ -0,0 +1,74 @@ +namespace Verso.Abstractions; + +/// +/// One classified span of a cell's source, returned by . +/// A token lies on a single line. +/// +/// The zero-based line of the token, relative to the cell's source as for a . +/// The zero-based column where the token begins, in UTF-16 characters. +/// The token's length in UTF-16 characters (as counts them). +/// +/// What the token names, one of . The editor ignores a token of a +/// type it does not know. +/// +/// +/// Refinements of , from , or null +/// for none. The editor ignores a modifier it does not know. +/// +public sealed record SemanticToken( + int Line, + int StartColumn, + int Length, + string Type, + IReadOnlyList? Modifiers = null); + +/// +/// The token types a may carry. The names are the standard ones of the +/// Language Server Protocol, which the editor's themes color by. +/// +public static class SemanticTokenTypes +{ + /// A namespace, or a module that groups declarations the way one does (an F# module). + public const string Namespace = "namespace"; + /// A type with no more specific kind below, or a type abbreviation. + public const string Type = "type"; + /// A class, record class, exception or other reference type. + public const string Class = "class"; + /// A struct or other value type. + public const string Struct = "struct"; + /// An enumeration. + public const string Enum = "enum"; + /// An interface. + public const string Interface = "interface"; + /// A generic type parameter. + public const string TypeParameter = "typeParameter"; + /// A function or method parameter. + public const string Parameter = "parameter"; + /// A variable or value. + public const string Variable = "variable"; + /// A property or field, including a record field. + public const string Property = "property"; + /// An enumeration member, a union case or a named constant. + public const string EnumMember = "enumMember"; + /// An event. + public const string Event = "event"; + /// A function. + public const string Function = "function"; + /// A method. + public const string Method = "method"; + /// A keyword that a lexer cannot recognize as one (an F# computation expression builder). + public const string Keyword = "keyword"; + /// An operator. + public const string Operator = "operator"; +} + +/// +/// The modifiers a may carry. +/// +public static class SemanticTokenModifiers +{ + /// A value or field that can be reassigned. + public const string Mutable = "mutable"; + /// A member that belongs to its type rather than to an instance. + public const string Static = "static"; +} diff --git a/src/Verso.Blazor.Shared/Components/Editor/MonacoEditor.razor b/src/Verso.Blazor.Shared/Components/Editor/MonacoEditor.razor index c9f418cb..fc50beaf 100644 --- a/src/Verso.Blazor.Shared/Components/Editor/MonacoEditor.razor +++ b/src/Verso.Blazor.Shared/Components/Editor/MonacoEditor.razor @@ -12,6 +12,7 @@ [Parameter] public Func>? OnGetHoverInfo { get; set; } [Parameter] public Func>? OnGetCompletions { get; set; } [Parameter] public Func>? OnGetDiagnostics { get; set; } + [Parameter] public Func>? OnGetSemanticTokens { get; set; } /// /// When true, the editor issues no diagnostics requests. A refresh asked for while suspended @@ -92,9 +93,9 @@ } /// - /// The cell has finished running: ask again for this editor's diagnostics and for every - /// editor below it that is showing kernel markers, nearest first, since the run may have - /// changed what their code refers to. + /// The cell has finished running: ask again for this editor's diagnostics and semantic + /// tokens, and for every editor below it that is showing kernel markers, nearest first, + /// since the run may have changed what their code refers to. /// public async Task RefreshAfterRunAsync() { @@ -171,6 +172,18 @@ catch { return null; } } + [JSInvokable] + public async Task GetSemanticTokens(string code) + { + try + { + return OnGetSemanticTokens is not null + ? await OnGetSemanticTokens(code) + : null; + } + catch { return null; } + } + public async ValueTask DisposeAsync() { if (_initialized) diff --git a/src/Verso.Blazor.Shared/Components/Notebook/Cell.razor b/src/Verso.Blazor.Shared/Components/Notebook/Cell.razor index 5712b2e3..1723e3ae 100644 --- a/src/Verso.Blazor.Shared/Components/Notebook/Cell.razor +++ b/src/Verso.Blazor.Shared/Components/Notebook/Cell.razor @@ -193,6 +193,7 @@ OnGetHoverInfo="HandleGetHoverInfo" OnGetCompletions="HandleGetCompletions" OnGetDiagnostics="HandleGetDiagnostics" + OnGetSemanticTokens="HandleGetSemanticTokens" DiagnosticsSuspended="IsNotebookExecuting" OnLineLimitChanged="HandleLineLimitChanged" /> @if (_lineLimit is { Clipped: true } limit) @@ -736,6 +737,33 @@ }; } + private async Task HandleGetSemanticTokens(string code) + { + // Markdown has no kernel to ask; an empty set leaves the editor's own coloring. + if (string.Equals(CellData.Type, "markdown", StringComparison.OrdinalIgnoreCase)) + return new { items = Array.Empty() }; + + // Masked as for diagnostics, so a leading magic line is not classified as code and + // positions still line up. + var result = await Service.GetSemanticTokensAsync( + CellData.Id, DiagnosticsSource.MaskLeadingMagicCommands(code)); + + // Null means the request could not be answered; the editor keeps the colors it shows. + if (result is null) return null; + + return new + { + items = result.Items.Select(t => new + { + line = t.Line, + startColumn = t.StartColumn, + length = t.Length, + type = t.Type, + modifiers = t.Modifiers + }).ToArray() + }; + } + private void ToggleTypeDropdown() { _showTypeDropdown = !_showTypeDropdown; diff --git a/src/Verso.Blazor.Shared/Models/MonacoTheme.cs b/src/Verso.Blazor.Shared/Models/MonacoTheme.cs index 6110ae05..e1dde660 100644 --- a/src/Verso.Blazor.Shared/Models/MonacoTheme.cs +++ b/src/Verso.Blazor.Shared/Models/MonacoTheme.cs @@ -30,21 +30,31 @@ public static class MonacoThemeBuilder { // A theme names the kind of thing being colored; the editor's tokenizers have their own // names for the same things, and often more than one. A name matches every token that - // starts with it, so "keyword" also covers "keyword.cs". + // starts with it, so "keyword" also covers "keyword.cs". The kernels' semantic tokens arrive + // under the standard semantic token names, with any modifiers appended ("variable.mutable"), + // and are colored by the same rules: a class as a type, a method as a function. private static readonly (string SyntaxKey, string[] Tokens)[] TokenMap = { ("keyword", new[] { "keyword" }), ("comment", new[] { "comment" }), ("string", new[] { "string" }), ("number", new[] { "number" }), - ("type", new[] { "type" }), - ("function", new[] { "function", "support.function" }), - ("variable", new[] { "variable" }), + ("type", new[] { "type", "class", "struct", "enum", "interface", "typeParameter" }), + ("function", new[] { "function", "support.function", "method" }), + ("variable", new[] { "variable", "parameter", "property", "event" }), ("operator", new[] { "operator", "operators" }), ("punctuation", new[] { "delimiter" }), ("preprocessor", new[] { "keyword.preprocessor", "namespace.cpp", "directive", "metatag" }), ("attribute", new[] { "annotation", "attribute.name" }), ("namespace", new[] { "namespace" }), + ("enumMember", new[] { "enumMember" }), + }; + + // Keys added after themes were written, and the key a theme without them is colored by. An + // enum member or union case is used more like the type it belongs to than like a variable. + private static readonly Dictionary Fallbacks = new(StringComparer.Ordinal) + { + ["enumMember"] = "type", }; /// @@ -85,7 +95,8 @@ private static readonly (string SyntaxKey, string[] Tokens)[] TokenMap = { foreach (var (key, tokens) in TokenMap) { - if (!syntax.TryGetValue(key, out var color) || string.IsNullOrWhiteSpace(color)) + if (!TryGetColor(syntax, key, out var color) + && !(Fallbacks.TryGetValue(key, out var fallback) && TryGetColor(syntax, fallback, out color))) continue; foreach (var token in tokens) rules.Add(new MonacoTokenRule(token, color)); @@ -95,6 +106,12 @@ private static readonly (string SyntaxKey, string[] Tokens)[] TokenMap = return new MonacoThemeSpec(BaseFor(kind, c.EditorBackground), colors, rules); } + private static bool TryGetColor(IReadOnlyDictionary syntax, string key, out string color) + { + color = syntax.TryGetValue(key, out var value) ? value : ""; + return !string.IsNullOrWhiteSpace(color); + } + /// /// Picks the stock theme to inherit from. High contrast comes in a dark and a light /// form, and does not say which, so the editor background does. diff --git a/src/Verso.Blazor.Shared/Models/ServiceDtos.cs b/src/Verso.Blazor.Shared/Models/ServiceDtos.cs index 44509e0d..1e558b81 100644 --- a/src/Verso.Blazor.Shared/Models/ServiceDtos.cs +++ b/src/Verso.Blazor.Shared/Models/ServiceDtos.cs @@ -272,6 +272,20 @@ public sealed record DiagnosticItemDto( int EndColumn, string? Code); +/// The kernel's classification of one cell's source. +public sealed record SemanticTokensResultDto(IReadOnlyList Items); + +/// +/// A single classified span. Positions are 0-based and relative to the cell's source; the type and +/// modifiers are names from SemanticTokenTypes and SemanticTokenModifiers. +/// +public sealed record SemanticTokenItemDto( + int Line, + int StartColumn, + int Length, + string Type, + IReadOnlyList? Modifiers); + /// Result of a cell execution. public sealed record ExecutionResultDto( Guid CellId, diff --git a/src/Verso.Blazor.Shared/Services/INotebookService.cs b/src/Verso.Blazor.Shared/Services/INotebookService.cs index a82b3be0..1986e07d 100644 --- a/src/Verso.Blazor.Shared/Services/INotebookService.cs +++ b/src/Verso.Blazor.Shared/Services/INotebookService.cs @@ -302,6 +302,18 @@ Task OutputChannelMessageAsync( Task GetDiagnosticsAsync(Guid cellId, string code) => Task.FromResult(null); + /// + /// Get the kernel's semantic classification of a cell's source, which the editor colors by. + /// Returns null when the request could not be answered, which the editor treats as "keep the + /// colors shown"; an empty list clears them. + /// + /// + /// Defaulted to null so a service without semantic token support leaves the editor's own + /// coloring. + /// + Task GetSemanticTokensAsync(Guid cellId, string code) + => Task.FromResult(null); + // ── Layout & theme switching ─────────────────────────────────────── /// diff --git a/src/Verso.Blazor.Shared/wwwroot/js/monaco-interop.js b/src/Verso.Blazor.Shared/wwwroot/js/monaco-interop.js index 2c52a6ad..349c274a 100644 --- a/src/Verso.Blazor.Shared/wwwroot/js/monaco-interop.js +++ b/src/Verso.Blazor.Shared/wwwroot/js/monaco-interop.js @@ -141,6 +141,8 @@ window.versoMonaco = (function () { } } }); + + monaco.languages.registerDocumentSemanticTokensProvider(language, semanticTokensProvider()); } // ── Kernel diagnostics ────────────────────────────────────────────── @@ -283,6 +285,7 @@ window.versoMonaco = (function () { const generation = state.generation; const epoch = state.staleEpoch; state.inFlight = true; + refreshSemanticTokens(model, epoch); return ref.invokeMethodAsync('GetDiagnostics', model.getValue()) .then(function (result) { @@ -305,13 +308,13 @@ window.versoMonaco = (function () { // ── Refresh after a run ───────────────────────────────────────────── // A run changes what the cells after it can refer to, so when a cell's run ends it is asked - // again, and so is every cell below it that is showing kernel markers, since those are the - // ones a run can visibly fix. For C# and F# each request is a full compilation, so the other - // cells below are left to their next focus, as are the cells above. The cells go top to - // bottom and one at a time: the cells nearest the run update first, and a run started in the - // meantime waits behind at most one request. Nothing goes while a cell runs; the walk resumes - // when the run is over. A cell whose answer still holds for its text and kernel state is - // skipped. + // again (diagnostics, and with them semantic tokens), and so is every cell below it that is + // showing kernel markers, since those are the ones a run can visibly fix. For C# and F# each + // request is a full compilation, so the other cells below are left to their next focus, as are + // the cells above. The cells go top to bottom and one at a time: the cells nearest the run + // update first, and a run started in the meantime waits behind at most one request. Nothing + // goes while a cell runs; the walk resumes when the run is over. A cell whose answer still + // holds for its text and kernel state is skipped. const refreshAfterRunQueue = new Set(); // elementIds still to ask let refreshAfterRunActive = false; @@ -390,6 +393,182 @@ window.versoMonaco = (function () { if (editor) setKernelMarkers(editor.getModel(), []); } + // ── Kernel semantic tokens ────────────────────────────────────────── + // Monaco pulls semantic tokens: it asks for a model's tokens when the model is attached, after + // an edit (debounced by Monaco itself), when the theme changes, and whenever the provider says + // they changed. A kernel answers from a type check, so the asking follows the diagnostics rules + // above. Nothing is asked while a cell runs, and the colors shown stay. A model is asked again + // without an edit only when its diagnostics are (its own run ended, or it gained focus after + // kernel state changed). Any other model whose text is unchanged since its answer gets that + // answer back without asking. Requests from every editor go one at a time, so a run in VS Code, + // where the host answers one request at a time, waits behind at most one of them. + const SEMANTIC_TOKEN_TYPES = [ + 'namespace', 'type', 'class', 'struct', 'enum', 'interface', 'typeParameter', 'parameter', + 'variable', 'property', 'enumMember', 'event', 'function', 'method', 'keyword', 'operator' + ]; + const SEMANTIC_TOKEN_MODIFIERS = ['mutable', 'static']; + const semanticTypeIndex = {}; + SEMANTIC_TOKEN_TYPES.forEach(function (t, i) { semanticTypeIndex[t] = i; }); + const semanticModifierBit = {}; + SEMANTIC_TOKEN_MODIFIERS.forEach(function (m, i) { semanticModifierBit[m] = 1 << i; }); + + const modelElements = {}; // model URI → elementId of its cell editor + // model URI → { data, version, resultId, refresh, askedVersion, askedEpoch }: the last answer, + // the model version it was for and the id it was last handed to Monaco under, whether a fresh + // answer is wanted for an unchanged version, and the version and diagnostics epoch the last + // request was sent for. + const semanticState = {}; + let semanticProvider = null; + let semanticChanged = null; + let semanticQueue = Promise.resolve(); + let semanticResultId = 0; + + function semanticStateFor(uri) { + let state = semanticState[uri]; + if (!state) { + state = { data: null, version: null, resultId: null, refresh: false, askedVersion: null, askedEpoch: -1 }; + semanticState[uri] = state; + } + return state; + } + + // Monaco keeps the tokens it shows when a request fails with "busy", and clears them on null. + function semanticBusy() { + return new Error('busy'); + } + + // Asks Monaco for every model's tokens. Models that need no new answer get their last one. + function announceSemanticTokensChanged() { + if (semanticChanged) semanticChanged.fire(); + } + + // Called as a diagnostics request for model goes out. Unless a semantic request has already + // gone out for the same text and kernel state (the edit both reacted to), ask for one too. + function refreshSemanticTokens(model, epoch) { + const state = semanticState[model.uri.toString()]; + if (!state) return; + if (state.askedVersion === model.getVersionId() && state.askedEpoch === epoch) return; + state.refresh = true; + announceSemanticTokensChanged(); + } + + // Kernel positions are 0-based and cell-relative, measured on the text that was sent. A token + // past the end of its line, of a type Monaco was not told about, or overlapping the one before + // it is dropped; the rest are delta-encoded as Monaco expects. + function encodeSemanticTokens(lineLengths, items) { + const tokens = []; + (items || []).forEach(function (item) { + const type = semanticTypeIndex[item.type]; + if (type === undefined) return; + const line = Number(item.line); + const start = Number(item.startColumn); + const length = Number(item.length); + if (!(line >= 0 && line < lineLengths.length) || !(start >= 0) || !(length > 0)) return; + if (start >= lineLengths[line]) return; + let modifiers = 0; + (item.modifiers || []).forEach(function (m) { + if (semanticModifierBit[m] !== undefined) modifiers |= semanticModifierBit[m]; + }); + tokens.push([line, start, Math.min(length, lineLengths[line] - start), type, modifiers]); + }); + tokens.sort(function (a, b) { return a[0] - b[0] || a[1] - b[1]; }); + + const data = []; + let prevLine = 0, prevStart = 0, lastLine = -1, lastEnd = 0; + tokens.forEach(function (t) { + if (t[0] === lastLine && t[1] < lastEnd) return; + data.push(t[0] - prevLine, t[0] === prevLine ? t[1] - prevStart : t[1], t[2], t[3], t[4]); + prevLine = t[0]; + prevStart = t[1]; + lastLine = t[0]; + lastEnd = t[1] + t[2]; + }); + return new Uint32Array(data); + } + + function semanticAnswer(state) { + state.resultId = String(++semanticResultId); + return { resultId: state.resultId, data: state.data }; + } + + function requestSemanticTokens(model, uri, elementId, state) { + if (model.isDisposed()) return Promise.resolve(null); + const diagnostics = diagnosticsState[elementId]; + if (diagnostics && diagnostics.suspended) { + state.refresh = true; + return Promise.reject(semanticBusy()); + } + const ref = dotnetRefs[uri]; + if (!ref) return Promise.resolve(null); + + const version = model.getVersionId(); + state.refresh = false; + state.askedVersion = version; + state.askedEpoch = diagnostics ? diagnostics.staleEpoch : 0; + + if (model.getLineCount() > DIAGNOSTICS_MAX_LINES) { + state.data = null; + state.version = version; + return Promise.resolve(null); + } + + const lineLengths = model.getLinesContent().map(function (line) { return line.length; }); + return ref.invokeMethodAsync('GetSemanticTokens', model.getValue()).then(function (result) { + // A failure keeps what is shown, until the next edit or refresh asks again. + state.version = version; + if (!result) { + state.data = null; + throw semanticBusy(); + } + state.data = encodeSemanticTokens(lineLengths, result.items); + return semanticAnswer(state); + }, function () { + state.version = version; + state.data = null; + throw semanticBusy(); + }); + } + + // lastResultId is the id of the answer Monaco shows, or null when it shows none (a new + // model, or one whose tokens a theme or language change cleared). + function provideSemanticTokens(model, lastResultId) { + const uri = model.uri.toString(); + const elementId = modelElements[uri]; + if (!elementId || !dotnetRefs[uri]) return null; + + const state = semanticStateFor(uri); + if (!state.refresh && state.version === model.getVersionId()) { + // Already showing the last answer: keep it rather than paint it again. + if (!state.data || (lastResultId && lastResultId === state.resultId)) { + return Promise.reject(semanticBusy()); + } + return semanticAnswer(state); + } + + return new Promise(function (resolve, reject) { + semanticQueue = semanticQueue.then(function () { + return requestSemanticTokens(model, uri, elementId, state).then(resolve, reject); + }); + }); + } + + function semanticTokensProvider() { + if (!semanticProvider) { + semanticChanged = new monaco.Emitter(); + semanticProvider = { + onDidChange: semanticChanged.event, + getLegend: function () { + return { tokenTypes: SEMANTIC_TOKEN_TYPES, tokenModifiers: SEMANTIC_TOKEN_MODIFIERS }; + }, + provideDocumentSemanticTokens: function (model, lastResultId) { + return provideSemanticTokens(model, lastResultId); + }, + releaseDocumentSemanticTokens: function () { } + }; + } + return semanticProvider; + } + // Extend the built-in C# monarch tokenizer to highlight #i "nuget: ..." directives // the same way Monaco highlights #r directives (as preprocessor + string). // The theme last handed to applyTheme(), kept so it can be defined as soon as Monaco @@ -397,6 +576,17 @@ window.versoMonaco = (function () { const CUSTOM_THEME = 'verso-active'; const BASE_THEMES = ['vs', 'vs-dark', 'hc-black', 'hc-light']; let _themeSpec = null; + // Whether the editors draw the kernels' semantic tokens; a theme description can turn + // them off, as a VS Code theme without semantic highlighting does. + let _semanticHighlighting = true; + + function applySemanticHighlighting(enabled) { + if (enabled === _semanticHighlighting) return; + _semanticHighlighting = enabled; + Object.values(editors).forEach(function (ed) { + ed.updateOptions({ 'semanticHighlighting.enabled': enabled }); + }); + } // Monaco reads theme colors as hex only, and paints anything else bright red rather // than ignoring it. Theme tokens and CSS variables can hold rgb()/rgba() as well, so @@ -578,6 +768,8 @@ window.versoMonaco = (function () { lineNumbersMinChars: 3, renderLineHighlight: 'line', automaticLayout: true, + // Standalone themes leave semantic highlighting off; the kernels supply the tokens. + 'semanticHighlighting.enabled': _semanticHighlighting, fontSize: _editorSettings.fontSize, fontFamily: _editorSettings.fontFamily, fontLigatures: _editorSettings.fontLigatures, @@ -750,6 +942,7 @@ window.versoMonaco = (function () { const modelUri = editor.getModel().uri.toString(); if (dotnetRef) { dotnetRefs[modelUri] = dotnetRef; + modelElements[modelUri] = elementId; } registerProviders(options.language || 'csharp'); }); @@ -764,6 +957,8 @@ window.versoMonaco = (function () { const modelUri = editor.getModel()?.uri?.toString(); if (modelUri) { delete dotnetRefs[modelUri]; + delete modelElements[modelUri]; + delete semanticState[modelUri]; } editor.dispose(); delete editors[elementId]; @@ -918,8 +1113,10 @@ window.versoMonaco = (function () { if (editor) { const model = editor.getModel(); if (model && monaco.editor.getModel(model.uri)) { - // The previous kernel's diagnostics say nothing about the new language. + // The previous kernel's diagnostics and tokens say nothing about the new language. clearDiagnostics(elementId); + const semantic = semanticState[model.uri.toString()]; + if (semantic) semantic.refresh = true; monaco.editor.setModelLanguage(model, language); registerProviders(language); } @@ -954,6 +1151,13 @@ window.versoMonaco = (function () { refreshAfterRunQueue.add(elementId); } if (!state.suspended) runRefreshAfterRunQueue(); + // Tokens an edit asked for during the run. + const editor = editors[elementId]; + const model = editor && editor.getModel(); + const semantic = model && semanticState[model.uri.toString()]; + if (!state.suspended && semantic && semantic.refresh) { + announceSemanticTokensChanged(); + } }, setTheme: function (theme) { @@ -971,10 +1175,12 @@ window.versoMonaco = (function () { // syntax rules, for a description whose rules are complete // colors Monaco color id -> CSS color ('editor.background', ...) // rules [{ token, foreground, fontStyle }] in Monaco's token names + // semanticHighlighting false to draw the kernels' semantic tokens not at all // Calling it again replaces the previous description. applyTheme: function (spec) { if (!spec) return; _themeSpec = spec; + applySemanticHighlighting(spec.semanticHighlighting !== false); _currentTheme = CUSTOM_THEME; if (monacoReady) { defineCustomTheme(); diff --git a/src/Verso.Blazor.Wasm/Services/RemoteNotebookService.cs b/src/Verso.Blazor.Wasm/Services/RemoteNotebookService.cs index b9a7415c..d94a5ec9 100644 --- a/src/Verso.Blazor.Wasm/Services/RemoteNotebookService.cs +++ b/src/Verso.Blazor.Wasm/Services/RemoteNotebookService.cs @@ -784,6 +784,21 @@ result.Range is not null .ToList()); } + public async Task GetSemanticTokensAsync(Guid cellId, string code) + { + var result = await _bridge.RequestAsync( + "kernel/getSemanticTokens", + new { cellId = cellId.ToString(), code }); + + // Null when the kernel could not analyze the code; the editor keeps the colors it shows. + if (result is null) return null; + + return new SemanticTokensResultDto( + (result.Items ?? new List()) + .Select(t => new SemanticTokenItemDto(t.Line, t.StartColumn, t.Length, t.Type, t.Modifiers)) + .ToList()); + } + // ── Layout & theme switching ──────────────────────────────────────── public Task SwitchLayoutAsync(string layoutId) @@ -2721,6 +2736,20 @@ private sealed class DiagnosticItem public string? Code { get; set; } } + private sealed class SemanticTokensResponse + { + public List? Items { get; set; } + } + + private sealed class SemanticTokenItem + { + public int Line { get; set; } + public int StartColumn { get; set; } + public int Length { get; set; } + public string Type { get; set; } = ""; + public List? Modifiers { get; set; } + } + private sealed class SettingsDefinitionsResponse { public List? Extensions { get; set; } diff --git a/src/Verso.Blazor.Wasm/wwwroot/js/vscode-bridge.js b/src/Verso.Blazor.Wasm/wwwroot/js/vscode-bridge.js index bca34d13..8f2fe24a 100644 --- a/src/Verso.Blazor.Wasm/wwwroot/js/vscode-bridge.js +++ b/src/Verso.Blazor.Wasm/wwwroot/js/vscode-bridge.js @@ -186,7 +186,9 @@ base: getBaseTheme(), inherit: rules.length === 0, colors: colors, - rules: rules + rules: rules, + // VS Code shows semantic tokens only with a theme (or setting) that asks for them. + semanticHighlighting: !active || active.semanticHighlighting !== false }; // The page is watched for changes of any kind, most of which are not the theme. diff --git a/src/Verso.Blazor/Services/ServerNotebookService.cs b/src/Verso.Blazor/Services/ServerNotebookService.cs index 5fe22217..fe324dd0 100644 --- a/src/Verso.Blazor/Services/ServerNotebookService.cs +++ b/src/Verso.Blazor/Services/ServerNotebookService.cs @@ -996,6 +996,38 @@ info.Range is { } r d.StartLine, d.StartColumn, d.EndLine, d.EndColumn, d.Code)).ToList()); } + public async Task GetSemanticTokensAsync(Guid cellId, string code) + { + if (_scaffold is null) return null; + + if (_scaffold.GetCell(cellId) is null) return null; + + // A kernel that does not classify leaves the editor's own coloring, and is not started + // for a request it cannot answer. + if (_scaffold.ResolveKernelForCell(cellId) is not ISemanticTokenProvider provider) + return new SemanticTokensResultDto(Array.Empty()); + + // Advisory, as diagnostics are: a failure answers null, and the editor keeps its colors. + IReadOnlyList? tokens; + try + { + await _scaffold.WarmUpKernelAsync(((ILanguageKernel)provider).LanguageId); + + // The same semantic pass as diagnostics; kept off the circuit's dispatcher. + tokens = await Task.Run(() => provider.GetSemanticTokensAsync(code)); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return null; + } + + if (tokens is null) return null; + + return new SemanticTokensResultDto( + tokens.Select(t => new SemanticTokenItemDto( + t.Line, t.StartColumn, t.Length, t.Type, t.Modifiers)).ToList()); + } + // ── Layout & theme switching ─────────────────────────────────────── public async Task RenderActiveLayoutAsync() diff --git a/src/Verso.FSharp/Helpers/SemanticTokenMapper.cs b/src/Verso.FSharp/Helpers/SemanticTokenMapper.cs new file mode 100644 index 00000000..e2aeb194 --- /dev/null +++ b/src/Verso.FSharp/Helpers/SemanticTokenMapper.cs @@ -0,0 +1,135 @@ +using FSharp.Compiler.EditorServices; +using Verso.Abstractions; + +namespace Verso.FSharp.Helpers; + +/// +/// Maps the F# compiler's semantic classifications to Verso semantic tokens. +/// +internal static class SemanticTokenMapper +{ + private static readonly IReadOnlyList Mutable = new[] { SemanticTokenModifiers.Mutable }; + + /// + /// Converts the classifications of a virtual document into tokens of , + /// the cell that begins after lines of it. Classifications in + /// the prefix, and any that span lines, are dropped. + /// + /// + /// The compiler classifies a long identifier by its last part over the whole of its range, so + /// List.map arrives as one function and List as a module beneath it. Each range is + /// cut back to its last identifier, which leaves List to its own classification. Where one + /// span is still classified more than once the last classification is kept, and a span that + /// overlaps the one before it is dropped. + /// + public static IReadOnlyList MapClassifications( + IEnumerable items, int prefixLineCount, string cellCode) + { + var lines = cellCode.Split('\n'); + var bySpan = new Dictionary<(int Line, int Column, int Length), SemanticToken>(); + foreach (var item in items) + { + var range = item.Range; + if (range.StartLine != range.EndLine) continue; + + // FCS lines are 1-based; cell lines are 0-based. + var line = range.StartLine - 1 - prefixLineCount; + if (line < 0 || line >= lines.Length) continue; + + var lineText = lines[line].TrimEnd('\r'); + var start = range.StartColumn; + var end = Math.Min(range.EndColumn, lineText.Length); + if (end <= start) continue; + + var mapped = Map(item.Type); + if (mapped is null) continue; + + start = LastIdentifierStart(lineText, start, end); + + var (type, modifiers) = mapped.Value; + bySpan[(line, start, end - start)] = new SemanticToken(line, start, end - start, type, modifiers); + } + + var tokens = new List(bySpan.Count); + foreach (var token in bySpan.Values.OrderBy(t => t.Line).ThenBy(t => t.StartColumn)) + { + if (tokens.Count > 0 && tokens[^1] is var previous + && previous.Line == token.Line && previous.StartColumn + previous.Length > token.StartColumn) + { + continue; + } + tokens.Add(token); + } + + return tokens; + } + + // The start of the identifier after the last '.' of a dotted name in [start, end), or start. + // A ``quoted identifier`` may hold a '.' of its own and is taken whole. + private static int LastIdentifierStart(string lineText, int start, int end) + { + var span = lineText.AsSpan(start, end - start); + if (span.EndsWith("``")) + { + var open = span[..^2].LastIndexOf("``"); + if (open <= 0) return start; + return span[open - 1] == '.' ? start + open : start; + } + + var dot = span.LastIndexOf('.'); + return dot >= 0 && dot < span.Length - 1 ? start + dot + 1 : start; + } + + /// + /// Returns the token type and modifiers for a classification, or null for one the + /// editor already colors from the source text alone (a format specifier inside a string) or + /// that carries no meaning of its own. + /// + internal static (string Type, IReadOnlyList? Modifiers)? Map(SemanticClassificationType type) => type switch + { + SemanticClassificationType.Namespace => (SemanticTokenTypes.Namespace, null), + SemanticClassificationType.Module => (SemanticTokenTypes.Namespace, null), + + SemanticClassificationType.ReferenceType => (SemanticTokenTypes.Class, null), + SemanticClassificationType.ConstructorForReferenceType => (SemanticTokenTypes.Class, null), + SemanticClassificationType.DisposableType => (SemanticTokenTypes.Class, null), + SemanticClassificationType.Exception => (SemanticTokenTypes.Class, null), + SemanticClassificationType.ValueType => (SemanticTokenTypes.Struct, null), + SemanticClassificationType.ConstructorForValueType => (SemanticTokenTypes.Struct, null), + SemanticClassificationType.Enumeration => (SemanticTokenTypes.Enum, null), + SemanticClassificationType.Interface => (SemanticTokenTypes.Interface, null), + SemanticClassificationType.Delegate => (SemanticTokenTypes.Type, null), + SemanticClassificationType.Type => (SemanticTokenTypes.Type, null), + SemanticClassificationType.TypeDef => (SemanticTokenTypes.Type, null), + SemanticClassificationType.TypeArgument => (SemanticTokenTypes.TypeParameter, null), + + SemanticClassificationType.UnionCase => (SemanticTokenTypes.EnumMember, null), + SemanticClassificationType.Literal => (SemanticTokenTypes.EnumMember, null), + + SemanticClassificationType.Function => (SemanticTokenTypes.Function, null), + SemanticClassificationType.IntrinsicFunction => (SemanticTokenTypes.Function, null), + SemanticClassificationType.Method => (SemanticTokenTypes.Method, null), + SemanticClassificationType.ExtensionMethod => (SemanticTokenTypes.Method, null), + + SemanticClassificationType.Property => (SemanticTokenTypes.Property, null), + SemanticClassificationType.RecordField => (SemanticTokenTypes.Property, null), + SemanticClassificationType.RecordFieldAsFunction => (SemanticTokenTypes.Property, null), + SemanticClassificationType.UnionCaseField => (SemanticTokenTypes.Property, null), + SemanticClassificationType.Field => (SemanticTokenTypes.Property, null), + SemanticClassificationType.MutableRecordField => (SemanticTokenTypes.Property, Mutable), + SemanticClassificationType.Event => (SemanticTokenTypes.Event, null), + + SemanticClassificationType.Value => (SemanticTokenTypes.Variable, null), + SemanticClassificationType.LocalValue => (SemanticTokenTypes.Variable, null), + SemanticClassificationType.DisposableTopLevelValue => (SemanticTokenTypes.Variable, null), + SemanticClassificationType.DisposableLocalValue => (SemanticTokenTypes.Variable, null), + SemanticClassificationType.MutableVar => (SemanticTokenTypes.Variable, Mutable), + SemanticClassificationType.NamedArgument => (SemanticTokenTypes.Parameter, null), + + SemanticClassificationType.ComputationExpression => (SemanticTokenTypes.Keyword, null), + SemanticClassificationType.Operator => (SemanticTokenTypes.Operator, null), + + // Printf (a %d inside a format string) and Plaintext. + _ => null, + }; +} diff --git a/src/Verso.FSharp/Kernel/FSharpCheckerManager.cs b/src/Verso.FSharp/Kernel/FSharpCheckerManager.cs index c0cf9b80..7d6d8ecc 100644 --- a/src/Verso.FSharp/Kernel/FSharpCheckerManager.cs +++ b/src/Verso.FSharp/Kernel/FSharpCheckerManager.cs @@ -13,6 +13,14 @@ internal sealed class FSharpCheckerManager : IDisposable { private FSharpChecker? _checker; + // The last check asked for, by everything it depends on. The editor asks for diagnostics and + // for semantic tokens of the same text moments apart, and a check of a cell re-checks every + // cell executed before it, so the second request shares the first one's check rather than + // running its own. A request for different text replaces it. + private readonly object _lastCheckLock = new(); + private string? _lastCheckKey; + private Task<(FSharpParseFileResults ParseResults, FSharpCheckFileResults CheckResults)?>? _lastCheck; + /// /// Creates and configures the F# checker instance. /// @@ -44,14 +52,56 @@ public void Initialize() /// A tuple of (FSharpParseFileResults, FSharpCheckFileResults) or null /// if the check was aborted. /// - public async Task<(FSharpParseFileResults ParseResults, FSharpCheckFileResults CheckResults)?> ParseAndCheckAsync( + /// + /// A call with the same file, text, compiler options and loaded scripts, unchanged on disk, as + /// the previous one returns that call's results, or joins it while it is still running. An aborted or failed check is not + /// kept. + /// + public Task<(FSharpParseFileResults ParseResults, FSharpCheckFileResults CheckResults)?> ParseAndCheckAsync( string fileName, string sourceText, FSharpProjectOptions options, CancellationToken ct = default) { - if (_checker is null) - throw new InvalidOperationException("Checker has not been initialized."); + var checker = _checker + ?? throw new InvalidOperationException("Checker has not been initialized."); + + // A cancellable request cannot be shared: cancelling it would cancel every caller joined to it. + if (ct.CanBeCanceled) + return CheckAsync(checker, fileName, sourceText, options, ct); + var key = CheckKey(fileName, sourceText, options); + lock (_lastCheckLock) + { + if (_lastCheck is not null && _lastCheckKey == key && !_lastCheck.IsFaulted && !_lastCheck.IsCanceled + && !(_lastCheck.IsCompletedSuccessfully && _lastCheck.Result is null)) + { + return _lastCheck; + } + + _lastCheckKey = key; + _lastCheck = CheckAsync(checker, fileName, sourceText, options, ct); + return _lastCheck; + } + } + + // The other source files are #load'ed scripts, which the checker reads from disk, so a key + // holds each one's write time as well as its path. + private static string CheckKey(string fileName, string sourceText, FSharpProjectOptions options) + { + var key = new System.Text.StringBuilder(); + foreach (var option in options.OtherOptions) + key.Append(option).Append('\0'); + foreach (var file in options.SourceFiles) + { + if (string.Equals(file, fileName, StringComparison.Ordinal)) continue; + key.Append(file).Append('\u0001').Append(File.GetLastWriteTimeUtc(file).Ticks).Append('\0'); + } + return key.Append(fileName).Append('\0').Append(sourceText).ToString(); + } + + private static async Task<(FSharpParseFileResults ParseResults, FSharpCheckFileResults CheckResults)?> CheckAsync( + FSharpChecker checker, string fileName, string sourceText, FSharpProjectOptions options, CancellationToken ct) + { var source = SourceText.ofString(sourceText); - var asyncOp = _checker.ParseAndCheckFileInProject( + var asyncOp = checker.ParseAndCheckFileInProject( fileName, 0, source, options, userOpName: FSharpOption.None); @@ -115,6 +165,11 @@ public void TriggerBackgroundCheck(string fileName, string sourceText, FSharpPro public void Dispose() { _checker = null; + lock (_lastCheckLock) + { + _lastCheckKey = null; + _lastCheck = null; + } } /// diff --git a/src/Verso.FSharp/Kernel/FSharpKernel.cs b/src/Verso.FSharp/Kernel/FSharpKernel.cs index b1b2058c..4866da98 100644 --- a/src/Verso.FSharp/Kernel/FSharpKernel.cs +++ b/src/Verso.FSharp/Kernel/FSharpKernel.cs @@ -19,7 +19,7 @@ namespace Verso.FSharp.Kernel; /// Powered by FSharp.Compiler.Service (FsiEvaluationSession). /// [VersoExtension] -public sealed class FSharpKernel : ILanguageKernel, IExtensionSettings +public sealed class FSharpKernel : ILanguageKernel, IExtensionSettings, ISemanticTokenProvider { private const string VirtualFileName = "/verso/notebook.fsx"; @@ -600,6 +600,46 @@ public async Task> GetDiagnosticsAsync(string code) } } + public async Task?> GetSemanticTokensAsync(string code) + { + ThrowIfDisposed(); + + if (!_initialized || _projectContext is null || _checkerManager is null) + return null; + + if (string.IsNullOrWhiteSpace(code)) + return Array.Empty(); + + try + { + var (sourceText, prefixLineCount, options) = _projectContext.BuildDocument(code); + int cellLineCount = CountLines(code); + + // The same check diagnostics run on this text, so the two requests an edit makes + // share one (see FSharpCheckerManager). + var result = await _checkerManager.ParseAndCheckAsync(VirtualFileName, sourceText, options) + .ConfigureAwait(false); + if (result is null) return null; + + var (_, checkResults) = result.Value; + + // Classify the cell's own lines only; the prefix is every cell executed before it. + var lastLineLength = code.Length - (code.LastIndexOf('\n') + 1); + var cellRange = global::FSharp.Compiler.Text.RangeModule.mkRange( + VirtualFileName, + global::FSharp.Compiler.Text.PositionModule.mkPos(prefixLineCount + 1, 0), + global::FSharp.Compiler.Text.PositionModule.mkPos(prefixLineCount + cellLineCount, lastLineLength)); + + var items = checkResults.GetSemanticClassification( + FSharpOption.Some(cellRange)); + return SemanticTokenMapper.MapClassifications(items, prefixLineCount, code); + } + catch + { + return null; + } + } + public ValueTask DisposeAsync() { if (_disposed) return ValueTask.CompletedTask; diff --git a/src/Verso.FSharp/README.md b/src/Verso.FSharp/README.md index cf819dfa..b4170734 100644 --- a/src/Verso.FSharp/README.md +++ b/src/Verso.FSharp/README.md @@ -10,6 +10,7 @@ Full F# scripting powered by FSharp.Compiler.Service. Latest F# language version - **F# Interactive session** with configurable warning level, language version, and default opens - **IntelliSense** with dot-completion, error diagnostics, and hover info (type signatures and documentation) +- **Semantic highlighting**: modules, types, functions, values and union cases colored by what the compiler resolves them to - **NuGet references** via `#r "nuget: PackageName"` with dual-path resolution (FSI built-in or Verso fallback) - **Script directives** (`#r`, `#load`, `#I`, `#nowarn`, and `#time`) - **Bidirectional variable sharing** between F# and other kernels diff --git a/src/Verso.Host/Dto/NotebookDto.cs b/src/Verso.Host/Dto/NotebookDto.cs index 3342a693..c1a06dc0 100644 --- a/src/Verso.Host/Dto/NotebookDto.cs +++ b/src/Verso.Host/Dto/NotebookDto.cs @@ -281,6 +281,26 @@ public sealed class HoverResult public RangeDto? Range { get; set; } } +public sealed class SemanticTokensParams +{ + public string CellId { get; set; } = ""; + public string Code { get; set; } = ""; +} + +public sealed class SemanticTokensResult +{ + public List Items { get; set; } = new(); +} + +public sealed class SemanticTokenDto +{ + public int Line { get; set; } + public int StartColumn { get; set; } + public int Length { get; set; } + public string Type { get; set; } = ""; + public List? Modifiers { get; set; } +} + public sealed class RangeDto { public int StartLine { get; set; } diff --git a/src/Verso.Host/Handlers/KernelHandler.cs b/src/Verso.Host/Handlers/KernelHandler.cs index 0d69e8a8..fa24c3b7 100644 --- a/src/Verso.Host/Handlers/KernelHandler.cs +++ b/src/Verso.Host/Handlers/KernelHandler.cs @@ -110,6 +110,45 @@ public static async Task HandleGetCompletionsAsync(NotebookSe }; } + public static async Task HandleGetSemanticTokensAsync(NotebookSession ns, JsonElement? @params) + { + var p = @params?.Deserialize(JsonRpcMessage.SerializerOptions) + ?? throw new JsonException("Missing params for kernel/getSemanticTokens"); + + // A kernel that does not classify leaves the editor's own coloring. It is not started for + // a request it cannot answer. + if (ResolveKernelForCell(ns, p.CellId) is not ISemanticTokenProvider provider) + return new SemanticTokensResult(); + + // Advisory, as diagnostics are: a kernel that cannot start or fails mid-analysis answers + // null, and the editor keeps the colors it already shows. + IReadOnlyList? tokens; + try + { + await ns.Scaffold.WarmUpKernelAsync(((ILanguageKernel)provider).LanguageId); + tokens = await provider.GetSemanticTokensAsync(p.Code); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return null; + } + + if (tokens is null) + return null; + + return new SemanticTokensResult + { + Items = tokens.Select(t => new SemanticTokenDto + { + Line = t.Line, + StartColumn = t.StartColumn, + Length = t.Length, + Type = t.Type, + Modifiers = t.Modifiers is { Count: > 0 } m ? m.ToList() : null + }).ToList() + }; + } + private static ILanguageKernel? ResolveKernelForCell(NotebookSession ns, string cellId) { return ns.Scaffold.ResolveKernelForCell(Guid.Parse(cellId)); diff --git a/src/Verso.Host/HostSession.cs b/src/Verso.Host/HostSession.cs index 3299f3df..723664ec 100644 --- a/src/Verso.Host/HostSession.cs +++ b/src/Verso.Host/HostSession.cs @@ -416,6 +416,7 @@ public async Task DispatchAsync(object id, string method, JsonElement? @ MethodNames.KernelGetCompletions => await KernelHandler.HandleGetCompletionsAsync(ns, @params), MethodNames.KernelGetDiagnostics => await KernelHandler.HandleGetDiagnosticsAsync(ns, @params), MethodNames.KernelGetHoverInfo => await KernelHandler.HandleGetHoverInfoAsync(ns, @params), + MethodNames.KernelGetSemanticTokens => await KernelHandler.HandleGetSemanticTokensAsync(ns, @params), MethodNames.OutputClearAll => OutputHandler.HandleClearAll(ns), MethodNames.CellInteract => await InteractionHandler.HandleInteractAsync(ns, @params), MethodNames.ChannelReady => await OutputChannelHandler.HandleReadyAsync(ns, @params), diff --git a/src/Verso.Host/Protocol/MethodNames.cs b/src/Verso.Host/Protocol/MethodNames.cs index 0c97ee8d..6b6287e0 100644 --- a/src/Verso.Host/Protocol/MethodNames.cs +++ b/src/Verso.Host/Protocol/MethodNames.cs @@ -46,6 +46,7 @@ public static class MethodNames public const string KernelGetCompletions = "kernel/getCompletions"; public const string KernelGetDiagnostics = "kernel/getDiagnostics"; public const string KernelGetHoverInfo = "kernel/getHoverInfo"; + public const string KernelGetSemanticTokens = "kernel/getSemanticTokens"; // Output public const string OutputClearAll = "output/clearAll"; diff --git a/src/Verso/Extensions/Themes/VersoDarkTheme.cs b/src/Verso/Extensions/Themes/VersoDarkTheme.cs index bc47974d..b96da5a4 100644 --- a/src/Verso/Extensions/Themes/VersoDarkTheme.cs +++ b/src/Verso/Extensions/Themes/VersoDarkTheme.cs @@ -129,6 +129,7 @@ public SyntaxColorMap GetSyntaxColors() map.Set("preprocessor", "#C586C0"); map.Set("attribute", "#4EC9B0"); map.Set("namespace", "#4EC9B0"); + map.Set("enumMember", "#4FC1FF"); return map; } } diff --git a/src/Verso/Extensions/Themes/VersoHighContrastTheme.cs b/src/Verso/Extensions/Themes/VersoHighContrastTheme.cs index 8f1a93be..d9f0a552 100644 --- a/src/Verso/Extensions/Themes/VersoHighContrastTheme.cs +++ b/src/Verso/Extensions/Themes/VersoHighContrastTheme.cs @@ -150,6 +150,7 @@ public SyntaxColorMap GetSyntaxColors() map.Set("preprocessor", "#DA70D6"); // Orchid, 5.5:1 map.Set("attribute", "#6FC3DF"); // Cyan, 9.0:1 map.Set("namespace", "#6FC3DF"); // Cyan, 9.0:1 + map.Set("enumMember", "#C5E478"); // Yellow green, 14.7:1 return map; } } diff --git a/src/Verso/Extensions/Themes/VersoLightTheme.cs b/src/Verso/Extensions/Themes/VersoLightTheme.cs index 8e152b0c..32f45fe3 100644 --- a/src/Verso/Extensions/Themes/VersoLightTheme.cs +++ b/src/Verso/Extensions/Themes/VersoLightTheme.cs @@ -47,6 +47,7 @@ public SyntaxColorMap GetSyntaxColors() map.Set("preprocessor", "#808080"); map.Set("attribute", "#267F99"); map.Set("namespace", "#267F99"); + map.Set("enumMember", "#0070C1"); return map; } } diff --git a/src/Verso/Kernels/CSharpKernel.cs b/src/Verso/Kernels/CSharpKernel.cs index f0e39727..89768cd9 100644 --- a/src/Verso/Kernels/CSharpKernel.cs +++ b/src/Verso/Kernels/CSharpKernel.cs @@ -14,7 +14,7 @@ namespace Verso.Kernels; /// with chained state, code completions, diagnostics, and hover information. /// [VersoExtension] -public sealed class CSharpKernel : ILanguageKernel +public sealed class CSharpKernel : ILanguageKernel, ISemanticTokenProvider { private static readonly Regex NuGetReferenceRegex = new( @"^#r\s+""nuget:\s*([^""]+)""", @@ -399,6 +399,18 @@ private static string BlankKeepingLineBreaks(Match match) return await _workspaceManager!.GetHoverInfoAsync(code, cursorPosition).ConfigureAwait(false); } + public async Task?> GetSemanticTokensAsync(string code) + { + ThrowIfDisposed(); + EnsureInitialized(); + + // Blanked as for diagnostics, so Roslyn does not read the rest of a #i line as a directive. + code = NuGetSourceRegex.Replace(code, BlankKeepingLineBreaks); + code = NuGetReferenceRegex.Replace(code, BlankKeepingLineBreaks); + + return await _workspaceManager!.GetSemanticTokensAsync(code).ConfigureAwait(false); + } + public async ValueTask DisposeAsync() { if (_disposed) return; diff --git a/src/Verso/Kernels/RoslynWorkspaceManager.cs b/src/Verso/Kernels/RoslynWorkspaceManager.cs index 88a62757..8a727b5b 100644 --- a/src/Verso/Kernels/RoslynWorkspaceManager.cs +++ b/src/Verso/Kernels/RoslynWorkspaceManager.cs @@ -1,6 +1,7 @@ using System.Composition.Hosting; using System.Reflection; using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Classification; using Microsoft.CodeAnalysis.Completion; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.Host; @@ -29,6 +30,13 @@ internal sealed class RoslynWorkspaceManager : IDisposable private ProjectId? _lastSubmissionId; private int _submissionCount; + // The last document built, for the cell text it holds. The editor asks for diagnostics and for + // semantic tokens of the same text moments apart; a document keeps the compilation it makes, so + // the second request shares the first one's rather than compiling the cell again. A run or a new + // reference changes what the cell compiles against and drops it. + private string? _lastDocumentCode; + private Document? _lastDocument; + public RoslynWorkspaceManager(IReadOnlyList defaultImports, IEnumerable references) { var assemblies = MefHostServices.DefaultAssemblies @@ -67,6 +75,7 @@ public void AddReferences(IEnumerable assemblyPaths) lock (_gate) { _references.AddRange(refs); + _lastDocument = null; } } @@ -80,21 +89,28 @@ public void AppendExecutedCode(string code) var submission = CreateSubmission(code, out _); _workspace.AddProject(submission); _lastSubmissionId = submission.Id; + _lastDocument = null; } } /// /// Builds a document holding only the current cell, as a submission that follows every - /// executed cell. Positions in the document are positions in the cell. + /// executed cell. Positions in the document are positions in the cell. The same text gets the + /// same document back until the workspace changes, so requests for it share one compilation. /// - private Document BuildDocument(string currentCode) + internal Document BuildDocument(string currentCode) { lock (_gate) { + if (_lastDocument is not null && string.Equals(_lastDocumentCode, currentCode, StringComparison.Ordinal)) + return _lastDocument; + // The submission is added to a fork of the solution and never applied, so an // in-progress cell leaves nothing behind in the workspace. var submission = CreateSubmission(currentCode, out var documentId); - return _workspace.CurrentSolution.AddProject(submission).GetDocument(documentId)!; + _lastDocumentCode = currentCode; + _lastDocument = _workspace.CurrentSolution.AddProject(submission).GetDocument(documentId)!; + return _lastDocument; } } @@ -261,6 +277,93 @@ public async Task> GetDiagnosticsAsync(string cod return new HoverInfo(description, Range: range); } + /// + /// Classifies the symbols in the given code. Roslyn classifies a span once by what it names + /// and once more if it names a static member; the second becomes a modifier. Spans the editor + /// colors from the text alone (keywords, literals, punctuation) are left out. + /// + /// + /// A variable declared at the top of a cell is a field of the cell's script class, and Roslyn + /// classifies it as one. It is reported as the variable it reads as. + /// + public async Task> GetSemanticTokensAsync(string code) + { + var document = BuildDocument(code); + var text = await document.GetTextAsync().ConfigureAwait(false); + var spans = await Classifier.GetClassifiedSpansAsync(document, new TextSpan(0, text.Length)) + .ConfigureAwait(false); + + var types = new Dictionary(); + var statics = new HashSet(); + var fields = new List(); + foreach (var span in spans) + { + if (span.ClassificationType == ClassificationTypeNames.StaticSymbol) + statics.Add(span.TextSpan); + else if (MapClassification(span.ClassificationType) is { } type) + types[span.TextSpan] = type; + + if (span.ClassificationType == ClassificationTypeNames.FieldName) + fields.Add(span.TextSpan); + } + + if (fields.Count > 0) + { + var semanticModel = await document.GetSemanticModelAsync().ConfigureAwait(false); + var root = await document.GetSyntaxRootAsync().ConfigureAwait(false); + if (semanticModel is not null && root is not null) + { + foreach (var span in fields) + { + var node = root.FindToken(span.Start).Parent; + if (node is null) continue; + var symbol = semanticModel.GetSymbolInfo(node).Symbol ?? semanticModel.GetDeclaredSymbol(node); + if (symbol?.ContainingType is { IsScriptClass: true }) + types[span] = SemanticTokenTypes.Variable; + } + } + } + + var tokens = new List(types.Count); + foreach (var (span, type) in types.OrderBy(t => t.Key.Start)) + { + var lineSpan = text.Lines.GetLinePositionSpan(span); + if (lineSpan.Start.Line != lineSpan.End.Line || span.Length == 0) continue; + + tokens.Add(new SemanticToken( + lineSpan.Start.Line, lineSpan.Start.Character, span.Length, type, + statics.Contains(span) ? StaticModifier : null)); + } + + return tokens; + } + + private static readonly IReadOnlyList StaticModifier = new[] { SemanticTokenModifiers.Static }; + + internal static string? MapClassification(string classification) => classification switch + { + ClassificationTypeNames.NamespaceName => SemanticTokenTypes.Namespace, + ClassificationTypeNames.ClassName => SemanticTokenTypes.Class, + ClassificationTypeNames.RecordClassName => SemanticTokenTypes.Class, + ClassificationTypeNames.StructName => SemanticTokenTypes.Struct, + ClassificationTypeNames.RecordStructName => SemanticTokenTypes.Struct, + ClassificationTypeNames.InterfaceName => SemanticTokenTypes.Interface, + ClassificationTypeNames.EnumName => SemanticTokenTypes.Enum, + ClassificationTypeNames.DelegateName => SemanticTokenTypes.Type, + ClassificationTypeNames.ModuleName => SemanticTokenTypes.Type, + ClassificationTypeNames.TypeParameterName => SemanticTokenTypes.TypeParameter, + ClassificationTypeNames.MethodName => SemanticTokenTypes.Method, + ClassificationTypeNames.ExtensionMethodName => SemanticTokenTypes.Method, + ClassificationTypeNames.PropertyName => SemanticTokenTypes.Property, + ClassificationTypeNames.FieldName => SemanticTokenTypes.Property, + ClassificationTypeNames.EventName => SemanticTokenTypes.Event, + ClassificationTypeNames.EnumMemberName => SemanticTokenTypes.EnumMember, + ClassificationTypeNames.ConstantName => SemanticTokenTypes.EnumMember, + ClassificationTypeNames.LocalName => SemanticTokenTypes.Variable, + ClassificationTypeNames.ParameterName => SemanticTokenTypes.Parameter, + _ => null, + }; + private static string MapCompletionKind(CompletionItem item) { if (item.Tags.Length == 0) return "Text"; diff --git a/tests/Verso.Blazor.Shared.Tests/CellTests.cs b/tests/Verso.Blazor.Shared.Tests/CellTests.cs index 780a7886..1d8e2c71 100644 --- a/tests/Verso.Blazor.Shared.Tests/CellTests.cs +++ b/tests/Verso.Blazor.Shared.Tests/CellTests.cs @@ -495,6 +495,41 @@ public async Task Diagnostics_MasksLeadingMagicLines_AndMapsResult() StringAssert.Contains(json, "\"code\":\"CS0029\""); } + [TestMethod] + public async Task SemanticTokens_MasksLeadingMagicLines_AndMapsResult() + { + _service.SemanticTokensResult = new SemanticTokensResultDto(new[] + { + new SemanticTokenItemDto(2, 4, 5, "variable", new[] { "mutable" }), + new SemanticTokenItemDto(2, 12, 1, "function", null), + }); + var cell = CreateCodeCell(""); + var cut = RenderCell(cell); + var editor = cut.FindComponent().Instance; + + var result = await cut.InvokeAsync(() => editor.GetSemanticTokens("#!time\n\nlet mutable count = f 1")); + + Assert.AreEqual(1, _service.SemanticTokensRequests.Count); + Assert.AreEqual(cell.Id, _service.SemanticTokensRequests[0].CellId); + Assert.AreEqual("\n\nlet mutable count = f 1", _service.SemanticTokensRequests[0].Code); + + var json = System.Text.Json.JsonSerializer.Serialize(result); + StringAssert.Contains(json, "{\"line\":2,\"startColumn\":4,\"length\":5,\"type\":\"variable\",\"modifiers\":[\"mutable\"]}"); + StringAssert.Contains(json, "\"type\":\"function\""); + } + + [TestMethod] + public async Task SemanticTokens_ServiceReturnsNull_ReturnsNull() + { + _service.SemanticTokensResult = null; + var cut = RenderCell(CreateCodeCell("")); + var editor = cut.FindComponent().Instance; + + var result = await cut.InvokeAsync(() => editor.GetSemanticTokens("let x = 1")); + + Assert.IsNull(result, "A failed request keeps the editor's colors, so it must stay null."); + } + [TestMethod] public async Task Diagnostics_ServiceReturnsNull_ReturnsNull() { diff --git a/tests/Verso.Blazor.Shared.Tests/Fakes/FakeNotebookService.cs b/tests/Verso.Blazor.Shared.Tests/Fakes/FakeNotebookService.cs index d2495d99..b201d7cb 100644 --- a/tests/Verso.Blazor.Shared.Tests/Fakes/FakeNotebookService.cs +++ b/tests/Verso.Blazor.Shared.Tests/Fakes/FakeNotebookService.cs @@ -300,6 +300,18 @@ public Task ExecuteActionAsync(string actionId, IReadOnlyList selectedCell return Task.FromResult(DiagnosticsResult); } + /// The result answers with; null by default. + public SemanticTokensResultDto? SemanticTokensResult { get; set; } + + /// Every (cell, code) pair passed to , in order. + public List<(Guid CellId, string Code)> SemanticTokensRequests { get; } = new(); + + public Task GetSemanticTokensAsync(Guid cellId, string code) + { + SemanticTokensRequests.Add((cellId, code)); + return Task.FromResult(SemanticTokensResult); + } + // ── Layout & theme switching ─────────────────────────────────────── public int RenderActiveLayoutCallCount { get; private set; } diff --git a/tests/Verso.Blazor.Shared.Tests/MonacoEditorTests.cs b/tests/Verso.Blazor.Shared.Tests/MonacoEditorTests.cs index 784cd8fd..cae4129e 100644 --- a/tests/Verso.Blazor.Shared.Tests/MonacoEditorTests.cs +++ b/tests/Verso.Blazor.Shared.Tests/MonacoEditorTests.cs @@ -70,6 +70,32 @@ public async Task GetDiagnostics_ReturnsNull_WhenCallbackThrows() Assert.IsNull(await cut.Instance.GetDiagnostics("var y = 2;")); } + [TestMethod] + public async Task GetSemanticTokens_ReturnsCallbackResult() + { + TestContext!.JSInterop.Mode = JSRuntimeMode.Loose; + string? received = null; + var expected = new { items = Array.Empty() }; + var cut = TestContext.RenderComponent(p => p + .Add(e => e.OnGetSemanticTokens, code => { received = code; return Task.FromResult(expected); })); + + var result = await cut.Instance.GetSemanticTokens("let y = 2"); + + Assert.AreEqual("let y = 2", received); + Assert.AreSame(expected, result); + } + + [TestMethod] + public async Task GetSemanticTokens_ReturnsNull_WhenCallbackThrows() + { + // Null tells the editor to keep the colors it has. + TestContext!.JSInterop.Mode = JSRuntimeMode.Loose; + var cut = TestContext.RenderComponent(p => p + .Add(e => e.OnGetSemanticTokens, _ => throw new InvalidOperationException("host gone"))); + + Assert.IsNull(await cut.Instance.GetSemanticTokens("let y = 2")); + } + [TestMethod] public async Task GetDiagnostics_ReturnsNull_WithoutCallback() { diff --git a/tests/Verso.Blazor.Shared.Tests/MonacoThemeBuilderTests.cs b/tests/Verso.Blazor.Shared.Tests/MonacoThemeBuilderTests.cs index 249addcc..d1f7e9f1 100644 --- a/tests/Verso.Blazor.Shared.Tests/MonacoThemeBuilderTests.cs +++ b/tests/Verso.Blazor.Shared.Tests/MonacoThemeBuilderTests.cs @@ -52,6 +52,42 @@ public void Build_MapsSyntaxColorsOntoEditorTokens() Assert.AreEqual("#E5C890", spec.Rules.Single(r => r.Token == "keyword.preprocessor").Foreground); } + [TestMethod] + public void Build_ColorsSemanticTokensByTheirThemeKey() + { + var syntax = new Dictionary + { + ["type"] = "#4EC9B0", + ["function"] = "#DCDCAA", + ["variable"] = "#9CDCFE", + ["enumMember"] = "#4FC1FF", + }; + + var spec = MonacoThemeBuilder.Build(ThemeKind.Dark, MakeThemeData(syntax: syntax))!; + + // The kernels' semantic tokens use the standard names, which a theme does not spell out. + foreach (var token in new[] { "class", "struct", "enum", "interface", "typeParameter" }) + Assert.AreEqual("#4EC9B0", spec.Rules.Single(r => r.Token == token).Foreground, token); + Assert.AreEqual("#DCDCAA", spec.Rules.Single(r => r.Token == "method").Foreground); + foreach (var token in new[] { "parameter", "property", "event" }) + Assert.AreEqual("#9CDCFE", spec.Rules.Single(r => r.Token == token).Foreground, token); + Assert.AreEqual("#4FC1FF", spec.Rules.Single(r => r.Token == "enumMember").Foreground); + } + + [TestMethod] + public void Build_ThemeWithoutEnumMember_ColorsEnumMembersAsTypes() + { + var syntax = new Dictionary + { + ["type"] = "#4EC9B0", + ["variable"] = "#9CDCFE", + }; + + var spec = MonacoThemeBuilder.Build(ThemeKind.Dark, MakeThemeData(syntax: syntax))!; + + Assert.AreEqual("#4EC9B0", spec.Rules.Single(r => r.Token == "enumMember").Foreground); + } + [TestMethod] public void Build_WithoutSyntaxColors_LeavesTheRulesToTheBaseTheme() { diff --git a/tests/Verso.Blazor.Shared.Tests/ServerNotebookServiceTests.cs b/tests/Verso.Blazor.Shared.Tests/ServerNotebookServiceTests.cs index f72148d7..073d8771 100644 --- a/tests/Verso.Blazor.Shared.Tests/ServerNotebookServiceTests.cs +++ b/tests/Verso.Blazor.Shared.Tests/ServerNotebookServiceTests.cs @@ -211,6 +211,39 @@ public async Task GetDiagnostics_MarkdownCell_ReturnsEmptyList() Assert.AreEqual(0, result!.Items.Count); } + [TestMethod] + public async Task GetSemanticTokens_KernelThatDoesNotClassify_ReturnsEmptyList() + { + await using var service = await CreateServiceAsync(); + var cell = await AddPowerShellCellAsync(service, ""); + + var result = await service.GetSemanticTokensAsync(cell.Id, "Write-Host 'ok'"); + + // Empty, not null: the editor keeps its own coloring rather than waiting for an answer. + Assert.IsNotNull(result); + Assert.AreEqual(0, result!.Items.Count); + } + + [TestMethod] + public async Task GetSemanticTokens_UnknownCell_ReturnsNull() + { + await using var service = await CreateServiceAsync(); + + Assert.IsNull(await service.GetSemanticTokensAsync(Guid.NewGuid(), "x")); + } + + [TestMethod] + public async Task GetSemanticTokens_MarkdownCell_ReturnsEmptyList() + { + await using var service = await CreateServiceAsync(); + var cell = await service.AddCellAsync("markdown"); + + var result = await service.GetSemanticTokensAsync(cell.Id, "# Title"); + + Assert.IsNotNull(result); + Assert.AreEqual(0, result!.Items.Count); + } + [TestMethod] public async Task GetDiagnostics_KernelThatFails_ReturnsNull() { diff --git a/tests/Verso.FSharp.Tests/Helpers/SemanticTokenMapperTests.cs b/tests/Verso.FSharp.Tests/Helpers/SemanticTokenMapperTests.cs new file mode 100644 index 00000000..74f959ae --- /dev/null +++ b/tests/Verso.FSharp.Tests/Helpers/SemanticTokenMapperTests.cs @@ -0,0 +1,115 @@ +using FSharp.Compiler.EditorServices; +using FSharp.Compiler.Text; +using Verso.Abstractions; +using Verso.FSharp.Helpers; + +namespace Verso.FSharp.Tests.Helpers; + +[TestClass] +public class SemanticTokenMapperTests +{ + private const string File = "/verso/notebook.fsx"; + + // A classification of a virtual-document range: 1-based lines, 0-based columns. + private static SemanticClassificationItem Item( + int startLine, int startColumn, int endLine, int endColumn, SemanticClassificationType type) => + new(Tuple.Create( + RangeModule.mkRange(File, PositionModule.mkPos(startLine, startColumn), PositionModule.mkPos(endLine, endColumn)), + type)); + + [TestMethod] + public void MapClassifications_ShiftsByThePrefix_AndDropsThePrefix() + { + var items = new[] + { + Item(1, 4, 1, 5, SemanticClassificationType.Value), // in the prefix + Item(4, 4, 4, 5, SemanticClassificationType.Function), // cell line 1 + }; + + var tokens = SemanticTokenMapper.MapClassifications(items, prefixLineCount: 2, "let a = 1\nlet f x = x"); + + Assert.AreEqual(1, tokens.Count); + Assert.AreEqual(new SemanticToken(1, 4, 1, SemanticTokenTypes.Function), tokens[0]); + } + + [TestMethod] + public void MapClassifications_DottedName_KeepsTheLastIdentifier() + { + var code = "Geometry.area 2.0"; + var items = new[] + { + Item(1, 0, 1, 13, SemanticClassificationType.Function), + Item(1, 0, 1, 8, SemanticClassificationType.Module), + }; + + var tokens = SemanticTokenMapper.MapClassifications(items, 0, code); + + Assert.AreEqual(2, tokens.Count); + Assert.AreEqual(new SemanticToken(0, 0, 8, SemanticTokenTypes.Namespace), tokens[0]); + Assert.AreEqual(new SemanticToken(0, 9, 4, SemanticTokenTypes.Function), tokens[1]); + } + + [TestMethod] + public void MapClassifications_QuotedIdentifierWithADot_IsTakenWhole() + { + var code = "let ``a.b`` = 1"; + var items = new[] { Item(1, 4, 1, 11, SemanticClassificationType.Value) }; + + var tokens = SemanticTokenMapper.MapClassifications(items, 0, code); + + Assert.AreEqual(new SemanticToken(0, 4, 7, SemanticTokenTypes.Variable), tokens.Single()); + } + + [TestMethod] + public void MapClassifications_DropsMultiLineAndUnmappedAndOverlapping() + { + var code = "printfn \"%d\" x\nlet y = x"; + var items = new[] + { + Item(1, 0, 2, 3, SemanticClassificationType.Value), // spans lines + Item(1, 9, 1, 11, SemanticClassificationType.Printf), // inside a string + Item(2, 4, 2, 9, SemanticClassificationType.Value), // "y = x" + Item(2, 8, 2, 9, SemanticClassificationType.Value), // overlaps the one before + }; + + var tokens = SemanticTokenMapper.MapClassifications(items, 0, code); + + Assert.AreEqual(1, tokens.Count); + Assert.AreEqual(new SemanticToken(1, 4, 5, SemanticTokenTypes.Variable), tokens[0]); + } + + [TestMethod] + public void MapClassifications_SameSpanTwice_KeepsTheLast() + { + var code = "let x = 1"; + var items = new[] + { + Item(1, 4, 1, 5, SemanticClassificationType.Value), + Item(1, 4, 1, 5, SemanticClassificationType.MutableVar), + }; + + var token = SemanticTokenMapper.MapClassifications(items, 0, code).Single(); + + Assert.AreEqual(SemanticTokenTypes.Variable, token.Type); + CollectionAssert.AreEqual(new[] { SemanticTokenModifiers.Mutable }, token.Modifiers!.ToArray()); + } + + [TestMethod] + public void Map_EveryClassification_HasAStandardTypeOrIsDropped() + { + var standard = typeof(SemanticTokenTypes).GetFields().Select(f => (string)f.GetValue(null)!).ToHashSet(); + foreach (SemanticClassificationType type in typeof(SemanticClassificationType) + .GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Static) + .Select(f => f.GetValue(null)!)) + { + var mapped = SemanticTokenMapper.Map(type); + if (mapped is null) + { + Assert.IsTrue(type is SemanticClassificationType.Printf or SemanticClassificationType.Plaintext, + $"{type} is dropped."); + continue; + } + Assert.IsTrue(standard.Contains(mapped.Value.Type), $"{type} maps to '{mapped.Value.Type}'."); + } + } +} diff --git a/tests/Verso.FSharp.Tests/Kernel/CheckerManagerTests.cs b/tests/Verso.FSharp.Tests/Kernel/CheckerManagerTests.cs new file mode 100644 index 00000000..96643644 --- /dev/null +++ b/tests/Verso.FSharp.Tests/Kernel/CheckerManagerTests.cs @@ -0,0 +1,92 @@ +using Verso.FSharp.Kernel; + +namespace Verso.FSharp.Tests.Kernel; + +[TestClass] +public class CheckerManagerTests +{ + private const string File = "/verso/notebook.fsx"; + + [TestMethod] + public async Task ParseAndCheck_SameDocument_SharesOneCheck() + { + using var manager = new FSharpCheckerManager(); + manager.Initialize(); + var context = new FSharpProjectContext(Array.Empty(), Array.Empty()); + + var (source, _, options) = context.BuildDocument("let x = 1"); + var first = manager.ParseAndCheckAsync(File, source, options); + // Built again, as each editor request builds it, with a fresh load time. + var (sameSource, _, sameOptions) = context.BuildDocument("let x = 1"); + var second = manager.ParseAndCheckAsync(File, sameSource, sameOptions); + + Assert.AreSame(first, second); + await first; + } + + [TestMethod] + public async Task ParseAndCheck_DifferentText_ChecksAgain() + { + using var manager = new FSharpCheckerManager(); + manager.Initialize(); + var context = new FSharpProjectContext(Array.Empty(), Array.Empty()); + + var (source, _, options) = context.BuildDocument("let x = 1"); + var first = manager.ParseAndCheckAsync(File, source, options); + var (other, _, otherOptions) = context.BuildDocument("let x = 2"); + var second = manager.ParseAndCheckAsync(File, other, otherOptions); + + Assert.AreNotSame(first, second); + await Task.WhenAll(first, second); + } + + [TestMethod] + public async Task ParseAndCheck_Cancellable_IsNotShared() + { + using var manager = new FSharpCheckerManager(); + manager.Initialize(); + var context = new FSharpProjectContext(Array.Empty(), Array.Empty()); + using var cts = new CancellationTokenSource(); + + var (source, _, options) = context.BuildDocument("let x = 1"); + var shared = manager.ParseAndCheckAsync(File, source, options); + var own = manager.ParseAndCheckAsync(File, source, options, cts.Token); + + Assert.AreNotSame(shared, own); + await Task.WhenAll(shared, own); + } + + [TestMethod] + public async Task ParseAndCheck_LoadedScriptChanged_ChecksAgain() + { + using var manager = new FSharpCheckerManager(); + manager.Initialize(); + var context = new FSharpProjectContext(Array.Empty(), Array.Empty()); + var script = Path.Combine(Path.GetTempPath(), $"verso-load-{Guid.NewGuid():N}.fsx"); + try + { + System.IO.File.WriteAllText(script, "module Lib\nlet triple x = 3 * x\n"); + + var (source, _, options) = context.BuildDocument("let x = 1"); + var unloaded = manager.ParseAndCheckAsync(File, source, options); + + // Asked without waiting: a check that comes back empty is not kept, and would be asked + // again whatever the key. + context.AddLoadedFiles(new[] { script }); + (source, _, options) = context.BuildDocument("let x = 1"); + var loaded = manager.ParseAndCheckAsync(File, source, options); + Assert.AreNotSame(unloaded, loaded, "a newly loaded script changes the check"); + + System.IO.File.SetLastWriteTimeUtc(script, DateTime.UtcNow.AddMinutes(1)); + (source, _, options) = context.BuildDocument("let x = 1"); + var rewritten = manager.ParseAndCheckAsync(File, source, options); + Assert.AreNotSame(loaded, rewritten, "a rewritten loaded script changes the check"); + + await Task.WhenAll(unloaded, loaded, rewritten); + } + finally + { + System.IO.File.Delete(script); + } + } +} diff --git a/tests/Verso.FSharp.Tests/Kernel/ExecutedCellsTests.cs b/tests/Verso.FSharp.Tests/Kernel/ExecutedCellsTests.cs index cd149bc4..e7343902 100644 --- a/tests/Verso.FSharp.Tests/Kernel/ExecutedCellsTests.cs +++ b/tests/Verso.FSharp.Tests/Kernel/ExecutedCellsTests.cs @@ -44,6 +44,21 @@ public async Task RunCell_CheckedAgain_HasNoDuplicateDefinitions() Assert.AreEqual(0, diagnostics.Count(d => d.Severity == DiagnosticSeverity.Error), Describe(diagnostics)); } + [TestMethod] + public async Task RunCell_CheckedAgain_KeepsItsSemanticTokens() + { + var code = "type Shape = Circle of float\nlet shape = Circle 1.0"; + await _kernel.ExecuteAsync(code, _context); + + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + Assert.IsTrue(tokens.Any(t => t.Line == 0 && t.StartColumn == 5 && t.Type == SemanticTokenTypes.Type), + "the type's own declaration"); + Assert.IsTrue(tokens.Any(t => t.Line == 1 && t.StartColumn == 12 && t.Type == SemanticTokenTypes.EnumMember), + "the union case"); + } + [TestMethod] public async Task RunCell_RedefinedByAnotherCell_TheLaterDefinitionWins() { diff --git a/tests/Verso.FSharp.Tests/Kernel/SemanticTokensTests.cs b/tests/Verso.FSharp.Tests/Kernel/SemanticTokensTests.cs new file mode 100644 index 00000000..56a6715e --- /dev/null +++ b/tests/Verso.FSharp.Tests/Kernel/SemanticTokensTests.cs @@ -0,0 +1,130 @@ +using Verso.Abstractions; +using Verso.FSharp.Kernel; +using Verso.Testing.Stubs; + +namespace Verso.FSharp.Tests.Kernel; + +[TestClass] +public class SemanticTokensTests +{ + private FSharpKernel _kernel = null!; + private StubExecutionContext _context = null!; + + [TestInitialize] + public async Task Setup() + { + _kernel = new FSharpKernel(); + await _kernel.InitializeAsync(); + _context = new StubExecutionContext(); + } + + [TestCleanup] + public async Task Cleanup() + { + await _kernel.DisposeAsync(); + } + + // The token whose text in code is name, the first time it occurs on the given line. + private static SemanticToken TokenFor(IReadOnlyList tokens, string code, int line, string name) + { + var column = code.Split('\n')[line].IndexOf(name, StringComparison.Ordinal); + Assert.IsTrue(column >= 0, $"'{name}' is not on line {line}."); + var token = tokens.FirstOrDefault(t => t.Line == line && t.StartColumn == column); + Assert.IsNotNull(token, $"No token for '{name}' at {line}:{column}. Tokens: " + + string.Join(", ", tokens.Select(t => $"{t.Line}:{t.StartColumn}+{t.Length} {t.Type}"))); + Assert.AreEqual(name.Length, token.Length, $"Length of the token for '{name}'."); + return token; + } + + [TestMethod] + public async Task SemanticTokens_ModuleTypeFunctionValueAndUnionCase_AreDistinct() + { + var code = string.Join('\n', + "module Geometry =", + " let area r = System.Math.PI * r * r", + "type Shape = Circle of float", + "let shape = Circle (Geometry.area 2.0)"); + + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + Assert.AreEqual(SemanticTokenTypes.Namespace, TokenFor(tokens, code, 3, "Geometry").Type); + Assert.AreEqual(SemanticTokenTypes.Function, TokenFor(tokens, code, 3, "area").Type); + Assert.AreEqual(SemanticTokenTypes.EnumMember, TokenFor(tokens, code, 3, "Circle").Type); + Assert.AreEqual(SemanticTokenTypes.Variable, TokenFor(tokens, code, 3, "shape").Type); + Assert.AreEqual(SemanticTokenTypes.Type, TokenFor(tokens, code, 2, "Shape").Type); + Assert.AreEqual(SemanticTokenTypes.Struct, TokenFor(tokens, code, 2, "float").Type); + } + + [TestMethod] + public async Task SemanticTokens_AfterExecution_ArePositionedInTheCell() + { + // The kernel checks a cell after every cell executed before it; a token left at the + // position it has in that document would land several lines below the cell. + await _kernel.ExecuteAsync("type Shape = Circle of float\nlet unit = 1.0", _context); + + var code = "let shape = Circle unit"; + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + Assert.IsTrue(tokens.All(t => t.Line == 0), "Every token is on the cell's only line."); + Assert.AreEqual(SemanticTokenTypes.EnumMember, TokenFor(tokens, code, 0, "Circle").Type); + Assert.AreEqual(SemanticTokenTypes.Variable, TokenFor(tokens, code, 0, "unit").Type); + } + + [TestMethod] + public async Task SemanticTokens_DottedName_ClassifiesEachPart() + { + var code = "let xs = List.map string [1; 2]"; + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + Assert.AreEqual(SemanticTokenTypes.Namespace, TokenFor(tokens, code, 0, "List").Type); + Assert.AreEqual(SemanticTokenTypes.Function, TokenFor(tokens, code, 0, "map").Type); + } + + [TestMethod] + public async Task SemanticTokens_MutableValue_CarriesModifier() + { + var code = "let mutable count = 0\ncount <- count + 1"; + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + var token = TokenFor(tokens, code, 1, "count"); + Assert.AreEqual(SemanticTokenTypes.Variable, token.Type); + CollectionAssert.Contains(token.Modifiers!.ToList(), SemanticTokenModifiers.Mutable); + } + + [TestMethod] + public async Task SemanticTokens_AreOrderedAndDoNotOverlap() + { + var code = "open System.Text\nlet sb = StringBuilder()\nlet n = sb.Append(\"a\").Length\nprintfn \"%d\" n"; + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + Assert.IsTrue(tokens.Count > 0); + for (var i = 1; i < tokens.Count; i++) + { + var (a, b) = (tokens[i - 1], tokens[i]); + Assert.IsTrue(a.Line < b.Line || (a.Line == b.Line && a.StartColumn + a.Length <= b.StartColumn), + $"Token {i} at {b.Line}:{b.StartColumn} overlaps or precedes {a.Line}:{a.StartColumn}+{a.Length}."); + } + } + + [TestMethod] + public async Task SemanticTokens_EmptyCode_ReturnsEmpty() + { + var tokens = await _kernel.GetSemanticTokensAsync(" "); + + Assert.IsNotNull(tokens); + Assert.AreEqual(0, tokens.Count); + } + + [TestMethod] + public async Task SemanticTokens_BeforeInitialize_ReturnsNull() + { + await using var kernel = new FSharpKernel(); + + Assert.IsNull(await kernel.GetSemanticTokensAsync("let x = 1")); + } +} diff --git a/tests/Verso.Host.Tests/HandlerTests.cs b/tests/Verso.Host.Tests/HandlerTests.cs index 99db29f4..95937955 100644 --- a/tests/Verso.Host.Tests/HandlerTests.cs +++ b/tests/Verso.Host.Tests/HandlerTests.cs @@ -210,7 +210,131 @@ public async Task GetDiagnostics_MarkdownCell_ReturnsEmpty() Assert.AreEqual(0, result!.Items.Count, "nothing executes a Markdown cell, so nothing checks it"); } - private sealed class FailingStartKernel : ILanguageKernel + private static JsonElement SemanticTokensParams(Guid cellId, string code) => + JsonSerializer.SerializeToElement( + new SemanticTokensParams { CellId = cellId.ToString(), Code = code }, + JsonRpcMessage.SerializerOptions); + + [TestMethod] + public async Task GetSemanticTokens_ReturnsTheKernelsTokens() + { + var (session, notebookId) = await CreateOpenSession(); + var ns = GetNs(session, notebookId); + var kernel = new ClassifyingKernel(new[] + { + new SemanticToken(0, 4, 5, SemanticTokenTypes.Variable, new[] { SemanticTokenModifiers.Mutable }), + new SemanticToken(1, 0, 3, SemanticTokenTypes.Function), + }); + ns.Scaffold.RegisterKernel(kernel); + var cell = ns.Scaffold.AddCell("code", ClassifyingKernel.Id); + + var result = await KernelHandler.HandleGetSemanticTokensAsync( + ns, SemanticTokensParams(cell.Id, "let count = 0\nrun ()")); + + Assert.IsNotNull(result); + Assert.AreEqual("let count = 0\nrun ()", kernel.LastCode); + Assert.AreEqual(2, result!.Items.Count); + var first = result.Items[0]; + Assert.AreEqual((0, 4, 5, "variable"), (first.Line, first.StartColumn, first.Length, first.Type)); + CollectionAssert.AreEqual(new[] { "mutable" }, first.Modifiers); + Assert.IsNull(result.Items[1].Modifiers, "a token with no modifiers sends none"); + } + + [TestMethod] + public async Task GetSemanticTokens_KernelThatDoesNotClassify_ReturnsEmpty() + { + var (session, notebookId) = await CreateOpenSession(); + var ns = GetNs(session, notebookId); + ns.Scaffold.RegisterKernel(new Verso.Testing.Fakes.FakeLanguageKernel("plain")); + var cell = ns.Scaffold.AddCell("code", "plain"); + + var result = await KernelHandler.HandleGetSemanticTokensAsync(ns, SemanticTokensParams(cell.Id, "x")); + + Assert.IsNotNull(result); + Assert.AreEqual(0, result!.Items.Count); + } + + [TestMethod] + public async Task GetSemanticTokens_KernelThatCannotAnalyze_ReturnsNull() + { + // Null keeps the colors the editor shows, where an empty list would clear them. + var (session, notebookId) = await CreateOpenSession(); + var ns = GetNs(session, notebookId); + ns.Scaffold.RegisterKernel(new ClassifyingKernel(null)); + var cell = ns.Scaffold.AddCell("code", ClassifyingKernel.Id); + + var result = await KernelHandler.HandleGetSemanticTokensAsync(ns, SemanticTokensParams(cell.Id, "x")); + + Assert.IsNull(result); + } + + [TestMethod] + public async Task GetSemanticTokens_KernelThatCannotStart_ReturnsNull() + { + var (session, notebookId) = await CreateOpenSession(); + var ns = GetNs(session, notebookId); + ns.Scaffold.RegisterKernel(new FailingStartKernel()); + var cell = ns.Scaffold.AddCell("code", FailingStartKernel.Id); + + var result = await KernelHandler.HandleGetSemanticTokensAsync(ns, SemanticTokensParams(cell.Id, "x")); + + Assert.IsNull(result); + } + + [TestMethod] + public async Task GetSemanticTokens_UnknownCell_ReturnsEmpty() + { + var (session, notebookId) = await CreateOpenSession(); + var ns = GetNs(session, notebookId); + + var result = await KernelHandler.HandleGetSemanticTokensAsync( + ns, SemanticTokensParams(Guid.NewGuid(), "x")); + + Assert.IsNotNull(result); + Assert.AreEqual(0, result!.Items.Count); + } + + private sealed class ClassifyingKernel : ILanguageKernel, ISemanticTokenProvider + { + public const string Id = "classifying"; + private readonly IReadOnlyList? _tokens; + + public ClassifyingKernel(IReadOnlyList? tokens) => _tokens = tokens; + + public string? LastCode { get; private set; } + + public string ExtensionId => "com.test.classifying"; + public string Name => "Classifying"; + public string Version => "1.0.0"; + public string? Author => null; + public string? Description => null; + public string LanguageId => Id; + public string DisplayName => "Classifying"; + public IReadOnlyList FileExtensions => Array.Empty(); + + public Task OnLoadedAsync(IExtensionHostContext context) => Task.CompletedTask; + public Task OnUnloadedAsync() => Task.CompletedTask; + public Task InitializeAsync() => Task.CompletedTask; + + public Task> ExecuteAsync(string code, IExecutionContext context) + => Task.FromResult>(Array.Empty()); + public Task> GetCompletionsAsync(string code, int cursorPosition) + => Task.FromResult>(Array.Empty()); + public Task> GetDiagnosticsAsync(string code) + => Task.FromResult>(Array.Empty()); + public Task GetHoverInfoAsync(string code, int cursorPosition) + => Task.FromResult(null); + + public Task?> GetSemanticTokensAsync(string code) + { + LastCode = code; + return Task.FromResult(_tokens); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class FailingStartKernel : ILanguageKernel, ISemanticTokenProvider { public const string Id = "failing-start"; @@ -238,6 +362,11 @@ public Task> GetDiagnosticsAsync(string code) }); public Task GetHoverInfoAsync(string code, int cursorPosition) => Task.FromResult(null); + public Task?> GetSemanticTokensAsync(string code) + => Task.FromResult?>(new[] + { + new SemanticToken(0, 0, 1, SemanticTokenTypes.Variable) + }); public ValueTask DisposeAsync() => ValueTask.CompletedTask; } diff --git a/tests/Verso.Host.Tests/JsonRpcMessageTests.cs b/tests/Verso.Host.Tests/JsonRpcMessageTests.cs index 36f2c51c..d892c844 100644 --- a/tests/Verso.Host.Tests/JsonRpcMessageTests.cs +++ b/tests/Verso.Host.Tests/JsonRpcMessageTests.cs @@ -114,6 +114,7 @@ public void MethodNames_AreCorrectStrings() Assert.AreEqual("kernel/getCompletions", MethodNames.KernelGetCompletions); Assert.AreEqual("kernel/getDiagnostics", MethodNames.KernelGetDiagnostics); Assert.AreEqual("kernel/getHoverInfo", MethodNames.KernelGetHoverInfo); + Assert.AreEqual("kernel/getSemanticTokens", MethodNames.KernelGetSemanticTokens); Assert.AreEqual("output/clearAll", MethodNames.OutputClearAll); } } diff --git a/tests/Verso.Tests/Kernels/CSharpKernelSemanticTokensTests.cs b/tests/Verso.Tests/Kernels/CSharpKernelSemanticTokensTests.cs new file mode 100644 index 00000000..125e43e2 --- /dev/null +++ b/tests/Verso.Tests/Kernels/CSharpKernelSemanticTokensTests.cs @@ -0,0 +1,109 @@ +using Verso.Abstractions; +using Verso.Contexts; +using Verso.Kernels; +using Verso.Stubs; + +namespace Verso.Tests.Kernels; + +[TestClass] +public sealed class CSharpKernelSemanticTokensTests +{ + private CSharpKernel _kernel = null!; + + [TestInitialize] + public async Task Setup() + { + _kernel = new CSharpKernel(); + await _kernel.InitializeAsync(); + } + + [TestCleanup] + public async Task Cleanup() + { + await _kernel.DisposeAsync(); + } + + private static SemanticToken TokenFor(IReadOnlyList tokens, string code, int line, string name) + { + var column = code.Split('\n')[line].IndexOf(name, StringComparison.Ordinal); + Assert.IsTrue(column >= 0, $"'{name}' is not on line {line}."); + var token = tokens.FirstOrDefault(t => t.Line == line && t.StartColumn == column); + Assert.IsNotNull(token, $"No token for '{name}' at {line}:{column}. Tokens: " + + string.Join(", ", tokens.Select(t => $"{t.Line}:{t.StartColumn}+{t.Length} {t.Type}"))); + Assert.AreEqual(name.Length, token.Length, $"Length of the token for '{name}'."); + return token; + } + + [TestMethod] + public async Task SemanticTokens_ClassifiesNamespaceTypeMethodLocalAndEnumMember() + { + var code = string.Join('\n', + "var sb = new System.Text.StringBuilder();", + "sb.Append(DayOfWeek.Monday);", + "int Twice(int value) => value * 2;"); + + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + Assert.AreEqual(SemanticTokenTypes.Variable, TokenFor(tokens, code, 0, "sb").Type); + Assert.AreEqual(SemanticTokenTypes.Namespace, TokenFor(tokens, code, 0, "System").Type); + Assert.AreEqual(SemanticTokenTypes.Class, TokenFor(tokens, code, 0, "StringBuilder").Type); + Assert.AreEqual(SemanticTokenTypes.Method, TokenFor(tokens, code, 1, "Append").Type); + Assert.AreEqual(SemanticTokenTypes.Enum, TokenFor(tokens, code, 1, "DayOfWeek").Type); + Assert.AreEqual(SemanticTokenTypes.EnumMember, TokenFor(tokens, code, 1, "Monday").Type); + Assert.AreEqual(SemanticTokenTypes.Parameter, TokenFor(tokens, code, 2, "value").Type); + } + + [TestMethod] + public async Task SemanticTokens_StaticMember_CarriesModifier() + { + var code = "var now = DateTime.Now;"; + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + var token = TokenFor(tokens, code, 0, "Now"); + Assert.AreEqual(SemanticTokenTypes.Property, token.Type); + CollectionAssert.Contains(token.Modifiers!.ToList(), SemanticTokenModifiers.Static); + } + + [TestMethod] + public async Task SemanticTokens_NameFromAnExecutedCell_IsClassified() + { + var context = new Verso.Contexts.ExecutionContext( + Guid.NewGuid(), 1, new VariableStore(), CancellationToken.None, + new StubThemeContext(), LayoutCapabilities.None, + new StubExtensionHostContext(() => Array.Empty()), + new NotebookMetadataContext(new NotebookModel()), + new Verso.Stubs.StubNotebookOperations(), + writeOutput: _ => Task.CompletedTask, + display: _ => Task.CompletedTask); + await _kernel.ExecuteAsync("record Point(double X, double Y);", context); + + var code = "var p = new Point(1, 2);\nvar x = p.X;"; + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + Assert.AreEqual(SemanticTokenTypes.Class, TokenFor(tokens, code, 0, "Point").Type); + Assert.AreEqual(SemanticTokenTypes.Property, TokenFor(tokens, code, 1, "X").Type); + } + + [TestMethod] + public async Task SemanticTokens_NuGetDirective_DoesNotShiftLaterTokens() + { + var code = "#r \"nuget: Humanizer, 2.14.1\"\nvar n = 1;"; + var tokens = await _kernel.GetSemanticTokensAsync(code); + + Assert.IsNotNull(tokens); + Assert.AreEqual(SemanticTokenTypes.Variable, TokenFor(tokens, code, 1, "n").Type); + } + + [TestMethod] + public async Task SemanticTokens_KeywordsAndLiterals_AreLeftToTheEditor() + { + var tokens = await _kernel.GetSemanticTokensAsync("var s = \"text\" + 1;"); + + Assert.IsNotNull(tokens); + Assert.AreEqual(1, tokens.Count, "only the local 's'"); + Assert.AreEqual(SemanticTokenTypes.Variable, tokens[0].Type); + } +} diff --git a/tests/Verso.Tests/Kernels/RoslynWorkspaceManagerTests.cs b/tests/Verso.Tests/Kernels/RoslynWorkspaceManagerTests.cs new file mode 100644 index 00000000..78a51d11 --- /dev/null +++ b/tests/Verso.Tests/Kernels/RoslynWorkspaceManagerTests.cs @@ -0,0 +1,57 @@ +using Microsoft.CodeAnalysis; +using Verso.Kernels; + +namespace Verso.Tests.Kernels; + +[TestClass] +public sealed class RoslynWorkspaceManagerTests +{ + private static RoslynWorkspaceManager CreateManager() => + new(new[] { "System" }, new[] { MetadataReference.CreateFromFile(typeof(object).Assembly.Location) }); + + [TestMethod] + public void BuildDocument_SameText_SharesOneDocument() + { + using var manager = CreateManager(); + + var first = manager.BuildDocument("var x = 1;"); + var second = manager.BuildDocument("var x = 1;"); + + Assert.AreSame(first, second); + } + + [TestMethod] + public void BuildDocument_DifferentText_BuildsAgain() + { + using var manager = CreateManager(); + + var first = manager.BuildDocument("var x = 1;"); + var second = manager.BuildDocument("var x = 2;"); + + Assert.AreNotSame(first, second); + } + + [TestMethod] + public void BuildDocument_AfterRun_BuildsAgain() + { + using var manager = CreateManager(); + + var before = manager.BuildDocument("var y = x;"); + manager.AppendExecutedCode("var x = 1;"); + var after = manager.BuildDocument("var y = x;"); + + Assert.AreNotSame(before, after); + } + + [TestMethod] + public void BuildDocument_AfterReference_BuildsAgain() + { + using var manager = CreateManager(); + + var before = manager.BuildDocument("var x = 1;"); + manager.AddReferences(new[] { typeof(Enumerable).Assembly.Location }); + var after = manager.BuildDocument("var x = 1;"); + + Assert.AreNotSame(before, after); + } +} diff --git a/vscode/src/blazor/activeColorTheme.ts b/vscode/src/blazor/activeColorTheme.ts index 64d0c9f8..ae5aa903 100644 --- a/vscode/src/blazor/activeColorTheme.ts +++ b/vscode/src/blazor/activeColorTheme.ts @@ -2,10 +2,13 @@ import * as vscode from "vscode"; import { log } from "../log"; import { MonacoTokenRule, + SemanticTokenStyle, TextMateRule, buildMonacoRules, + buildSemanticRules, customizationRules, parseJsonc, + semanticCustomization, } from "./themeTokenRules"; export type MonacoBaseTheme = "vs" | "vs-dark" | "hc-black" | "hc-light"; @@ -14,6 +17,15 @@ export type MonacoBaseTheme = "vs" | "vs-dark" | "hc-black" | "hc-light"; export interface MonacoTokenTheme { base: MonacoBaseTheme; rules: MonacoTokenRule[]; + /** Whether VS Code would show semantic tokens with this theme, so the notebook does too. */ + semanticHighlighting: boolean; +} + +/** What a theme file and the files it includes say about syntax colors. */ +interface ThemeSyntax { + tokenColors: TextMateRule[]; + semanticTokenColors: Record; + semanticHighlighting?: boolean; } interface ThemeContribution { @@ -88,31 +100,56 @@ function locateTheme(base: MonacoBaseTheme): LocatedTheme | undefined { return fallback; } -async function readTokenColors(file: vscode.Uri, depth: number): Promise { +async function readThemeSyntax(file: vscode.Uri, depth: number): Promise { + const syntax: ThemeSyntax = { tokenColors: [], semanticTokenColors: {} }; if (depth > MAX_INCLUDE_DEPTH || !file.path.toLowerCase().endsWith(".json")) { // The other format a theme may use is a TextMate property list, which is rare now // and not worth a parser; such a theme keeps the stock syntax colors. - return []; + return syntax; } const bytes = await vscode.workspace.fs.readFile(file); const theme = parseJsonc(new TextDecoder("utf-8").decode(bytes)) as { include?: string; tokenColors?: TextMateRule[] | string; + semanticTokenColors?: Record; + semanticHighlighting?: boolean; } | null; - if (!theme || typeof theme !== "object") return []; + if (!theme || typeof theme !== "object") return syntax; + // An included theme comes first, and what this file says is laid over it. const dir = vscode.Uri.joinPath(file, ".."); - const rules: TextMateRule[] = []; + const merge = (other: ThemeSyntax): void => { + syntax.tokenColors.push(...other.tokenColors); + Object.assign(syntax.semanticTokenColors, other.semanticTokenColors); + if (other.semanticHighlighting !== undefined) syntax.semanticHighlighting = other.semanticHighlighting; + }; if (typeof theme.include === "string") { - rules.push(...(await readTokenColors(vscode.Uri.joinPath(dir, theme.include), depth + 1))); + merge(await readThemeSyntax(vscode.Uri.joinPath(dir, theme.include), depth + 1)); } if (typeof theme.tokenColors === "string") { - rules.push(...(await readTokenColors(vscode.Uri.joinPath(dir, theme.tokenColors), depth + 1))); + merge(await readThemeSyntax(vscode.Uri.joinPath(dir, theme.tokenColors), depth + 1)); } else if (Array.isArray(theme.tokenColors)) { - rules.push(...theme.tokenColors); + syntax.tokenColors.push(...theme.tokenColors); + } + if (theme.semanticTokenColors && typeof theme.semanticTokenColors === "object") { + Object.assign(syntax.semanticTokenColors, theme.semanticTokenColors); } - return rules; + if (typeof theme.semanticHighlighting === "boolean") { + syntax.semanticHighlighting = theme.semanticHighlighting; + } + return syntax; +} + +/** + * Whether VS Code shows semantic tokens: "editor.semanticHighlighting.enabled" decides + * when it is a boolean, and otherwise ("configuredByTheme") the user's semantic token + * customizations or the theme do. + */ +function semanticHighlightingEnabled(fromCustomizations: boolean | undefined, fromTheme: boolean | undefined): boolean { + const setting = vscode.workspace.getConfiguration("editor").get("semanticHighlighting.enabled"); + if (typeof setting === "boolean") return setting; + return fromCustomizations ?? fromTheme ?? false; } /** @@ -126,22 +163,33 @@ async function readTokenColors(file: vscode.Uri, depth: number): Promise { const base = activeBaseTheme(); + // Without the theme's file nothing says what VS Code would do, and the stock theme the + // editor then keeps colors semantic tokens by its own rules. + const unread: MonacoTokenTheme = { base, rules: [], semanticHighlighting: semanticHighlightingEnabled(undefined, true) }; try { const located = locateTheme(base); - if (!located) return { base, rules: [] }; - - const tokenColors = await readTokenColors(located.file, 0); - if (tokenColors.length === 0) return { base, rules: [] }; - - const custom = vscode.workspace - .getConfiguration("editor") - .get("tokenColorCustomizations"); + if (!located) return unread; + + const syntax = await readThemeSyntax(located.file, 0); + if (syntax.tokenColors.length === 0) return unread; + + const editor = vscode.workspace.getConfiguration("editor"); + const tokenColors = [ + ...syntax.tokenColors, + ...customizationRules(editor.get("tokenColorCustomizations"), located.name), + ]; + const semantic = semanticCustomization( + editor.get("semanticTokenColorCustomizations"), located.name); return { base, - rules: buildMonacoRules([...tokenColors, ...customizationRules(custom, located.name)]), + rules: [ + ...buildMonacoRules(tokenColors), + ...buildSemanticRules(tokenColors, { ...syntax.semanticTokenColors, ...semantic.rules }), + ], + semanticHighlighting: semanticHighlightingEnabled(semantic.enabled, syntax.semanticHighlighting), }; } catch (err) { log.warn(`Could not read the active color theme's syntax colors: ${err}`); - return { base, rules: [] }; + return unread; } } diff --git a/vscode/src/blazor/blazorEditorProvider.ts b/vscode/src/blazor/blazorEditorProvider.ts index 68cf0336..212a5b31 100644 --- a/vscode/src/blazor/blazorEditorProvider.ts +++ b/vscode/src/blazor/blazorEditorProvider.ts @@ -107,7 +107,9 @@ export class BlazorEditorProvider context.subscriptions.push( vscode.window.onDidChangeActiveColorTheme(() => void this.pushTokenTheme()), vscode.workspace.onDidChangeConfiguration((e) => { - if (e.affectsConfiguration("editor.tokenColorCustomizations")) { + if (e.affectsConfiguration("editor.tokenColorCustomizations") || + e.affectsConfiguration("editor.semanticTokenColorCustomizations") || + e.affectsConfiguration("editor.semanticHighlighting")) { void this.pushTokenTheme(); } }) diff --git a/vscode/src/blazor/themeTokenRules.ts b/vscode/src/blazor/themeTokenRules.ts index 54d9d4be..07cd9b8c 100644 --- a/vscode/src/blazor/themeTokenRules.ts +++ b/vscode/src/blazor/themeTokenRules.ts @@ -292,3 +292,135 @@ export function buildMonacoRules(tokenColors: readonly TextMateRule[]): MonacoTo } return rules; } + +/** + * One value of a theme's `semanticTokenColors`, or of the user's + * `editor.semanticTokenColorCustomizations` rules: a color, or a style. + */ +export type SemanticTokenStyle = + | string + | { + foreground?: string; + fontStyle?: string; + bold?: boolean; + italic?: boolean; + underline?: boolean; + strikethrough?: boolean; + }; + +// The semantic token types the kernels report, with the TextMate scopes VS Code colors each +// by when the theme's semanticTokenColors does not name it (its token classification +// registry). Each scope is one alternative, tried in order. +const SEMANTIC_TOKEN_SCOPES: ReadonlyArray = [ + ["namespace", ["entity.name.namespace"]], + ["type", ["entity.name.type", "support.type"]], + ["class", ["entity.name.type.class", "support.class"]], + ["struct", ["entity.name.type.struct"]], + ["enum", ["entity.name.type.enum"]], + ["interface", ["entity.name.type.interface"]], + ["typeParameter", ["entity.name.type.parameter"]], + ["parameter", ["variable.parameter"]], + ["variable", ["variable.other.readwrite", "entity.name.variable"]], + ["property", ["variable.other.property"]], + ["enumMember", ["variable.other.enummember"]], + ["event", ["variable.other.event"]], + ["function", ["entity.name.function", "support.function"]], + ["method", ["entity.name.function.member", "support.function"]], + ["keyword", ["keyword.control"]], + ["operator", ["keyword.operator"]], +]; + +const SEMANTIC_TOKEN_MODIFIERS: readonly string[] = ["mutable", "static"]; + +function semanticStyle(value: SemanticTokenStyle | undefined): Resolved | undefined { + if (typeof value === "string") return { foreground: value }; + if (!value || typeof value !== "object") return undefined; + + const resolved: Resolved = {}; + if (typeof value.foreground === "string") resolved.foreground = value.foreground; + if (typeof value.fontStyle === "string") { + resolved.fontStyle = value.fontStyle; + } else { + const flags = (["bold", "italic", "underline", "strikethrough"] as const) + .filter((flag) => value[flag] === true); + if (flags.length > 0) resolved.fontStyle = flags.join(" "); + } + return resolved.foreground !== undefined || resolved.fontStyle !== undefined ? resolved : undefined; +} + +/** + * Builds the Monaco rules for the kernels' semantic tokens, colored as VS Code colors the + * same tokens in a source file: by the theme's `semanticTokenColors` where a selector names + * the token's type, or its type and a modifier ("variable.mutable", "*.static"), and by the + * TextMate scope VS Code falls back to otherwise. Monaco looks a token up by its type with + * its modifiers appended, so "variable.mutable" falls back to the "variable" rule. + * + * Types the Monaco tokenizers share ("variable", "function", "type", "namespace", + * "keyword", "operator") already have a rule from {@link buildMonacoRules} with the same + * scopes, and get one here only when `semanticTokenColors` names them. Selectors limited to + * a language ("variable:fsharp") are not supported and are passed over. + */ +export function buildSemanticRules( + tokenColors: readonly TextMateRule[], + semanticTokenColors: Readonly> +): MonacoTokenRule[] { + const lexical = new Set(TOKEN_SCOPES.map(([token]) => token)); + const rules: MonacoTokenRule[] = []; + const push = (token: string, style: Resolved | undefined): void => { + if (!style) return; + const rule: MonacoTokenRule = { token }; + if (style.foreground !== undefined) rule.foreground = style.foreground; + if (style.fontStyle !== undefined) rule.fontStyle = style.fontStyle; + rules.push(rule); + }; + + for (const [type, scopes] of SEMANTIC_TOKEN_SCOPES) { + const named = semanticStyle(semanticTokenColors[type]); + if (named) { + push(type, named); + } else if (!lexical.has(type)) { + let hit: Resolved = {}; + for (const scope of scopes) { + hit = resolveScope(tokenColors, scope); + if (hit.foreground !== undefined) break; + } + push(type, hit.foreground !== undefined || hit.fontStyle !== undefined ? hit : undefined); + } + + for (const modifier of SEMANTIC_TOKEN_MODIFIERS) { + push(`${type}.${modifier}`, + semanticStyle(semanticTokenColors[`${type}.${modifier}`]) + ?? semanticStyle(semanticTokenColors[`*.${modifier}`])); + } + } + return rules; +} + +/** + * Reads the user's "editor.semanticTokenColorCustomizations": its rules, laid over the + * theme's semanticTokenColors, and whether it turns semantic highlighting on or off. A block + * scoped to the active theme ("[Theme Name]") is laid over the general one. + */ +export function semanticCustomization( + customizations: unknown, + themeName: string +): { rules: Record; enabled?: boolean } { + const result: { rules: Record; enabled?: boolean } = { rules: {} }; + if (!customizations || typeof customizations !== "object") return result; + + const apply = (block: Record): void => { + if (typeof block.enabled === "boolean") result.enabled = block.enabled; + if (block.rules && typeof block.rules === "object") { + Object.assign(result.rules, block.rules as Record); + } + }; + const all = customizations as Record; + apply(all); + for (const [key, value] of Object.entries(all)) { + if (key.startsWith("[") && value && typeof value === "object" && themeKeyMatches(key, themeName)) { + apply(value as Record); + } + } + return result; +} + diff --git a/vscode/test/suite/protocol.test.ts b/vscode/test/suite/protocol.test.ts index a252d85b..4d08e29e 100644 --- a/vscode/test/suite/protocol.test.ts +++ b/vscode/test/suite/protocol.test.ts @@ -148,6 +148,7 @@ suite("Protocol Message Types", () => { "kernel/getCompletions", "kernel/getDiagnostics", "kernel/getHoverInfo", + "kernel/getSemanticTokens", "kernel/restart", ]; for (const m of methods) { diff --git a/vscode/test/suite/themeTokenRules.test.ts b/vscode/test/suite/themeTokenRules.test.ts index b86b9eb8..bce0d34f 100644 --- a/vscode/test/suite/themeTokenRules.test.ts +++ b/vscode/test/suite/themeTokenRules.test.ts @@ -2,9 +2,11 @@ import * as assert from "assert"; import { TextMateRule, buildMonacoRules, + buildSemanticRules, customizationRules, parseJsonc, resolveScope, + semanticCustomization, } from "../../src/blazor/themeTokenRules"; suite("Theme token rules", () => { @@ -111,4 +113,69 @@ suite("Theme token rules", () => { assert.deepStrictEqual(customizationRules(undefined, "Any"), []); assert.deepStrictEqual(customizationRules("nonsense", "Any"), []); }); + // A slice of Dark 2026 and the themes it includes. + const dark2026: TextMateRule[] = [ + { scope: "entity.name.type", settings: { foreground: "#4EC9B0" } }, + { scope: "entity.name", settings: { foreground: "#FFA657" } }, + { scope: "entity.name.function", settings: { foreground: "#D2A8FF" } }, + { scope: "variable.other", settings: { foreground: "#C9D1D9" } }, + { scope: ["variable.other.enummember", "variable.other.constant"], settings: { foreground: "#79C0FF" } }, + { scope: "variable.parameter", settings: { foreground: "#9CDCFE" } }, + ]; + + test("semantic tokens take the scope VS Code falls back to", () => { + const rules = buildSemanticRules(dark2026, {}); + + // entity.name.type.struct and .class fall to entity.name.type, the most specific match. + assert.strictEqual(ruleFor(rules, "struct")?.foreground, "#4EC9B0"); + assert.strictEqual(ruleFor(rules, "class")?.foreground, "#4EC9B0"); + assert.strictEqual(ruleFor(rules, "enumMember")?.foreground, "#79C0FF"); + assert.strictEqual(ruleFor(rules, "method")?.foreground, "#D2A8FF"); + assert.strictEqual(ruleFor(rules, "property")?.foreground, "#C9D1D9"); + assert.strictEqual(ruleFor(rules, "parameter")?.foreground, "#9CDCFE"); + }); + + test("semantic types the Monaco tokenizers share keep the lexical rule", () => { + const rules = buildSemanticRules(dark2026, {}); + + for (const token of ["variable", "function", "type", "namespace", "keyword", "operator"]) { + assert.strictEqual(ruleFor(rules, token), undefined, token); + } + }); + + test("semanticTokenColors win over the fallback scope, and name modifiers", () => { + const rules = buildSemanticRules(dark2026, { + enumMember: "#FF0000", + variable: { foreground: "#00FF00", italic: true }, + "variable.mutable": { underline: true }, + "*.static": { bold: true }, + "class:fsharp": "#123456", + }); + + assert.strictEqual(ruleFor(rules, "enumMember")?.foreground, "#FF0000"); + assert.deepStrictEqual(ruleFor(rules, "variable"), { token: "variable", foreground: "#00FF00", fontStyle: "italic" }); + assert.deepStrictEqual(ruleFor(rules, "variable.mutable"), { token: "variable.mutable", fontStyle: "underline" }); + assert.strictEqual(ruleFor(rules, "method.static")?.fontStyle, "bold"); + assert.strictEqual(ruleFor(rules, "class")?.foreground, "#4EC9B0", "a language-scoped selector is passed over"); + }); + + test("a semantic token the theme says nothing about gets no rule", () => { + assert.deepStrictEqual(buildSemanticRules([], {}), []); + }); + + test("semantic token customizations apply generally and for the active theme", () => { + const custom = semanticCustomization( + { + enabled: false, + rules: { variable: "#111111" }, + "[Dark 2026]": { enabled: true, rules: { property: "#222222" } }, + "[Light 2026]": { rules: { variable: "#333333" } }, + }, + "Dark 2026" + ); + + assert.strictEqual(custom.enabled, true); + assert.deepStrictEqual(custom.rules, { variable: "#111111", property: "#222222" }); + assert.deepStrictEqual(semanticCustomization(undefined, "Any"), { rules: {} }); + }); });