Highlight diff hunks using the file lines above them

Summary:
The comparison view tokenizes each hunk on its own, starting from the grammar's initial state. When a hunk starts inside a multi-line construct, such as a template literal, triple-quoted string, or block comment, the closing delimiter is read as an opening one. Everything after it in the hunk then flips: code is colored as a string, and strings as code.

The fix fetches the lines of the "before" file above the last hunk, using the same `requestComparisonContextLines` message that expanding context already uses. The worker then tokenizes the unchanged lines between hunks and carries the grammar state into each hunk, on both sides. Lines between hunks are unchanged, so the same lines advance both sides' state.

- `tokenizeLinesFromState` in `shared/textmate-lib/tokenize.ts` starts from a given state and returns the state after the last line; `tokenizeLines` calls it with `INITIAL`.
- `SplitDiffTable` fetches up to 10,000 context lines when the view provides `fetchAdditionalLines`. Hunks are highlighted without context first, as before, and highlighted again once the lines arrive. Views without `fetchAdditionalLines`, files above the cap, and failed fetches (an error result or a rejected promise) keep the current behavior.
- `comparisonContextLines` now echoes the request's `start` and `numLines`, and the client matches on them. Before this, the client matched only on path, so two requests for the same file in flight at once (now more likely, since the view fetches context on open) could both resolve with the first response. Responses from older servers without these fields still match on path alone.

Reviewed By: cfirimar

Differential Revision: D125599812

fbshipit-source-id: 150b27f7493edbb929c128c51c17a2bed951fa2e

Author: cm-mm3-support

addons/isl-server/src/ServerToClientAPI.ts

@@ -645,13 +645,17 @@ export default class ServerToClientAPI {
               type: 'comparisonContextLines',
               lines: {value: content.split('\n').slice(start - 1, start - 1 + numLines)},
               path: relativePath,
+              start,
+              numLines,
             }),
           )
           .catch((error: Error) =>
             this.postMessage({
               type: 'comparisonContextLines',
               lines: {error},
               path: relativePath,
+              start,
+              numLines,
             }),
           );
         break;

addons/isl/src/ComparisonView/ComparisonView.tsx

@@ -628,7 +628,10 @@ function ComparisonViewFile({
 
       const result = await serverAPI.nextMessageMatching(
         'comparisonContextLines',
-        msg => msg.path === id.path,
+        msg =>
+          msg.path === id.path &&
+          // Older servers don't say which lines they sent.
+          (msg.start == null || (msg.start === start && msg.numLines === numLines)),
       );
 
       return result.lines;

addons/isl/src/ComparisonView/SplitDiffView/SplitDiffHunk.tsx

@@ -26,6 +26,7 @@ import {useTableColumnSelection} from './copyFromSelectedColumn';
 import {useTokenizedContents, useTokenizedHunks} from './syntaxHighlighting';
 
 const MAX_INPUT_LENGTH_FOR_INTRALINE_DIFF = 300;
+const MAX_CONTEXT_LINES_FOR_SYNTAX_HIGHLIGHTING = 10000;
 
 export type SplitDiffTableProps = {
   ctx: Context;
@@ -49,7 +50,13 @@ export const SplitDiffTable = React.memo(
     );
 
     const t = ctx.t ?? (s => s);
-    const tokenization = useTokenizedHunks(patch.newFileName ?? '', patch.hunks, ctx.useThemeHook);
+    const contextLinesBefore = useContextLinesBeforeHunks(ctx, patch.hunks);
+    const tokenization = useTokenizedHunks(
+      patch.newFileName ?? '',
+      patch.hunks,
+      ctx.useThemeHook,
+      contextLinesBefore,
+    );
 
     const {className: tableSelectionClassName, ...tableSelectionProps} = useTableColumnSelection();
 
@@ -587,3 +594,50 @@ function useFetchLines(ctx: Context, numLines: number, start: number) {
 
   return fetchedLines;
 }
+
+/**
+ * Fetches the lines of the "before" file above the last hunk, so syntax highlighting can
+ * start each hunk in the right state, for example inside a multi-line string or comment.
+ */
+function useContextLinesBeforeHunks(
+  ctx: Context,
+  hunks: ParsedDiff['hunks'],
+): Array<string> | undefined {
+  const [fetched, setFetched] = useState<{key: string; lines: Array<string>} | undefined>(
+    undefined,
+  );
+
+  const lastHunk = hunks.at(-1);
+  // A hunk that only adds lines comes after line `oldStart`, so that line is context too.
+  const numLines =
+    lastHunk == null ? 0 : lastHunk.oldLines === 0 ? lastHunk.oldStart : lastHunk.oldStart - 1;
+  const shouldFetch = numLines > 0 && numLines <= MAX_CONTEXT_LINES_FOR_SYNTAX_HIGHLIGHTING;
+
+  // Use-case controlled key that allows invalidating the fetched lines.
+  const invalidationKey = ctx.useComparisonInvalidationKeyHook?.();
+
+  const comparisonKey = comparisonStringKey(ctx.id.comparison);
+  // Lines fetched for an earlier comparison or file must not be used for this one.
+  const key = `${invalidationKey}:${comparisonKey}:${ctx.id.path}:${numLines}`;
+  useEffect(() => {
+    if (!shouldFetch) {
+      return;
+    }
+    let isCancelled = false;
+    ctx
+      .fetchAdditionalLines?.(ctx.id, 1, numLines)
+      .then(result => {
+        if (!isCancelled && result.value != null) {
+          setFetched({key, lines: result.value});
+        }
+      })
+      // Without the lines, hunks keep their per-hunk highlighting.
+      .catch(() => undefined);
+    return () => {
+      isCancelled = true;
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [key, shouldFetch]);
+
+  return shouldFetch && fetched?.key === key ? fetched.lines : undefined;
+}

addons/isl/src/ComparisonView/SplitDiffView/syntaxHighlighting.tsx

@@ -110,13 +110,14 @@ function getWorker(): Promise<WorkerApi<SyntaxWorkerRequest, SyntaxWorkerRespons
  * Given a set of hunks from a diff view,
  * asynchronously provide syntax highlighting by tokenizing.
  *
- * Note that we reconstruct the file contents from the diff,
- * so syntax highlighting can be inaccurate since it's missing full context.
+ * Note that we reconstruct the file contents from the diff. Without `contextLinesBefore`,
+ * syntax highlighting can be inaccurate since it's missing the lines above each hunk.
  */
 export function useTokenizedHunks(
   path: string,
   hunks: ParsedDiff['hunks'],
   useThemeHook: () => ThemeColor,
+  contextLinesBefore?: Array<string>,
 ): TokenizedDiffHunks | undefined {
   const theme = useThemeHook();
 
@@ -125,14 +126,16 @@ export function useTokenizedHunks(
   useEffect(() => {
     const token = newTrackedCancellationToken();
     getWorker().then(worker =>
-      worker.request({type: 'tokenizeHunks', theme, path, hunks}, token).then(result => {
-        if (!token.isCancelled) {
-          setTokenized(result.result);
-        }
-      }),
+      worker
+        .request({type: 'tokenizeHunks', theme, path, hunks, contextLinesBefore}, token)
+        .then(result => {
+          if (!token.isCancelled) {
+            setTokenized(result.result);
+          }
+        }),
     );
     return () => token.cancel();
-  }, [theme, path, hunks]);
+  }, [theme, path, hunks, contextLinesBefore]);
   return tokenized;
 }
 

addons/isl/src/ComparisonView/SplitDiffView/syntaxHighlightingTypes.ts

@@ -42,6 +42,11 @@ export type SyntaxWorkerRequest =
       theme: ThemeColor;
       path: string;
       hunks: ParsedDiff['hunks'];
+      /**
+       * Lines of the "before" file, starting at line 1, that come before the last hunk.
+       * Lets each hunk start from the grammar state the lines above it leave, for example inside a multi-line string.
+       */
+      contextLinesBefore?: Array<string>;
     };
 
 export type SyntaxWorkerResponse =

addons/isl/src/ComparisonView/SplitDiffView/syntaxHighlightingWorker.ts

@@ -5,6 +5,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+import type {ParsedDiff} from 'shared/patch/types';
 import type {IGrammar} from 'vscode-textmate';
 import type {ThemeColor} from '../../theme';
 import type {
@@ -16,8 +17,9 @@ import type {
 
 import {CancellationToken} from 'shared/CancellationToken';
 import FilepathClassifier from 'shared/textmate-lib/FilepathClassifier';
-import {tokenizeLines} from 'shared/textmate-lib/tokenize';
+import {tokenizeLines, tokenizeLinesFromState} from 'shared/textmate-lib/tokenize';
 import {loadWASM} from 'vscode-oniguruma';
+import {INITIAL} from 'vscode-textmate';
 import {grammars, languages} from '../../generated/textmate/TextMateGrammarManifest';
 import {getGrammar, getGrammarStore} from './grammar';
 
@@ -151,7 +153,7 @@ export function handleMessage(
     case 'tokenizeHunks': {
       workQueue.push(async () => {
         const grammar = await loadGrammar(data.theme, data.path, postMessage);
-        const result = tokenizeHunks(grammar, data.hunks, token);
+        const result = tokenizeHunks(grammar, data.hunks, data.contextLinesBefore, token);
         postMessage({type: data.type, id: data.id, result});
         cancellationTokenForId.delete(data.id);
       });
@@ -172,7 +174,8 @@ if (typeof self.document === 'undefined') {
 
 function tokenizeHunks(
   grammar: IGrammar | undefined,
-  hunks: Array<{lines: Array<string>}>,
+  hunks: ParsedDiff['hunks'],
+  contextLinesBefore: Array<string> | undefined,
   cancellationToken: CancellationToken,
 ): TokenizedDiffHunks | undefined {
   if (grammar == null) {
@@ -184,11 +187,32 @@ function tokenizeHunks(
     return undefined;
   }
 
-  const tokenizedPatches: TokenizedDiffHunks = hunks
-    .map(hunk => recoverFileContentsFromPatchLines(hunk.lines))
-    .map(([before, after]) => [tokenizeLines(before, grammar), tokenizeLines(after, grammar)]);
+  if (contextLinesBefore == null) {
+    return hunks
+      .map(hunk => recoverFileContentsFromPatchLines(hunk.lines))
+      .map(([before, after]) => [tokenizeLines(before, grammar), tokenizeLines(after, grammar)]);
+  }
 
-  return tokenizedPatches;
+  let beforeState = INITIAL;
+  let afterState = INITIAL;
+  // 1-indexed line number in the "before" file of the first line not yet tokenized.
+  let nextLine = 1;
+  return hunks.map(hunk => {
+    // A hunk that only adds lines comes after line `oldStart`, rather than starting at it.
+    const hunkStart = hunk.oldLines === 0 ? hunk.oldStart + 1 : hunk.oldStart;
+    // Lines between hunks are unchanged, so they are the same on both sides.
+    const unchangedLines = contextLinesBefore.slice(nextLine - 1, hunkStart - 1);
+    beforeState = tokenizeLinesFromState(unchangedLines, grammar, beforeState).state;
+    afterState = tokenizeLinesFromState(unchangedLines, grammar, afterState).state;
+    nextLine = hunkStart + hunk.oldLines;
+
+    const [before, after] = recoverFileContentsFromPatchLines(hunk.lines);
+    const tokenizedBefore = tokenizeLinesFromState(before, grammar, beforeState);
+    const tokenizedAfter = tokenizeLinesFromState(after, grammar, afterState);
+    beforeState = tokenizedBefore.state;
+    afterState = tokenizedAfter.state;
+    return [tokenizedBefore.tokens, tokenizedAfter.tokens];
+  });
 }
 
 function tokenizeContent(