diff --git a/src/main/columnPattern.ts b/src/main/columnPattern.ts index f9a39cb..b570da7 100644 --- a/src/main/columnPattern.ts +++ b/src/main/columnPattern.ts @@ -67,12 +67,42 @@ function classifyToken(s: string, isLast: boolean): string { // Split a painted chunk into { pre, body, post }: leading/trailing bracket/quote borders // kept LITERAL, the middle generalised to its format class via classifyToken(). const PAINT_BORDER = '[\\[\\](){}<>"\'`]'; -function inferPaintField(chunk: string, isLast: boolean): { pre: string; body: string; post: string } { +// Longest common prefix (fromEnd=false) or suffix (fromEnd=true) across the strings — +// used to peel a column's CONSTANT WRAPPER (/…/, xx…xx, …blah…) detected from many samples. +function commonAffix(strings: string[], fromEnd: boolean): string { + const vals = strings.filter(s => s.length > 0); + if (vals.length < 2) return ''; + const ref = vals[0]; + let len = ref.length; + for (let i = 1; i < vals.length && len > 0; i++) { + const s = vals[i]; + let k = 0; + const max = Math.min(len, s.length); + while (k < max && (fromEnd ? ref[ref.length - 1 - k] === s[s.length - 1 - k] : ref[k] === s[k])) k++; + len = k; + } + return fromEnd ? ref.slice(ref.length - len) : ref.slice(0, len); +} + +// Turn a painted chunk into { pre, body, post }: an outer DATA-DRIVEN wrapper (constant +// prefix/suffix shared across `samples` — the column's values on other lines) is kept +// literal, then the single-sample bracket/quote borders, then the value's format class. +// Arbitrary wrappers (/…/, xx…xx) need `samples`; brackets are guessable from one line. +function inferPaintField(chunk: string, isLast: boolean, samples?: string[]): { pre: string; body: string; post: string } { let core = chunk, pre = '', post = ''; + if (samples && samples.length >= 2) { + const withChunk = [chunk, ...samples]; + let cp = commonAffix(withChunk, false); + if (cp.length >= core.length) cp = ''; + if (cp) { pre = staticToRegex(cp); core = core.slice(cp.length); } // staticToRegex → \s+ so + let cs = commonAffix(withChunk, true); // a " ->" border matches + if (cs.length >= core.length) cs = ''; // any whitespace + "->". + if (cs) { post = staticToRegex(cs); core = core.slice(0, core.length - cs.length); } + } const lead = core.match(new RegExp(`^(?:${PAINT_BORDER})+`)); - if (lead && lead[0].length < core.length) { pre = escapeRegex(lead[0]); core = core.slice(lead[0].length); } + if (lead && lead[0].length < core.length) { pre += escapeRegex(lead[0]); core = core.slice(lead[0].length); } const trail = core.match(new RegExp(`(?:${PAINT_BORDER})+$`)); - if (trail && trail[0].length < core.length) { post = escapeRegex(trail[0]); core = core.slice(0, core.length - trail[0].length); } + if (trail && trail[0].length < core.length) { post = escapeRegex(trail[0]) + post; core = core.slice(0, core.length - trail[0].length); } return { pre, body: classifyToken(core, isLast), post }; } @@ -134,7 +164,9 @@ function compileGrok(pattern: string, flags: string): CompiledColumnPattern { return { regex: parts.map(p => p.body).join(''), flags, fields, named: true }; } -function compilePaint(sample: string, spans: PaintSpan[], flags: string): CompiledColumnPattern { +// fieldSamples (optional) = the values each column takes across sample lines, in span order — +// passed by the "refine from data" step so inferPaintField can peel constant wrappers. +function compilePaint(sample: string, spans: PaintSpan[], flags: string, fieldSamples?: string[][]): CompiledColumnPattern { const clean = (spans || []) .filter(s => s && s.end > s.start && s.start >= 0 && s.end <= sample.length) .sort((a, b) => a.start - b.start); @@ -142,6 +174,7 @@ function compilePaint(sample: string, spans: PaintSpan[], flags: string): Compil const fields: string[] = []; let out = ''; let cursor = 0; + let fieldIdx = 0; for (let i = 0; i < clean.length; i++) { const sp = clean[i]; if (sp.start < cursor) continue; // skip overlaps @@ -150,15 +183,26 @@ function compilePaint(sample: string, spans: PaintSpan[], flags: string): Compil fields.push(name); const chunk = sample.slice(sp.start, sp.end); const isLast = i === clean.length - 1 && sp.end >= sample.length; - // Format-aware: infer the token's shape (date/int/word…), keep bracket/quote borders literal. - const { pre, body, post } = inferPaintField(chunk, isLast); + // Format-aware: infer the token's shape (date/int/word…), keep bracket/quote borders literal, + // and — when we have sample values for this column — peel its constant data-driven wrapper. + const { pre, body, post } = inferPaintField(chunk, isLast, fieldSamples ? fieldSamples[fieldIdx] : undefined); out += pre + `(?<${name}>${body})` + post; cursor = sp.end; + fieldIdx++; } out += staticToRegex(sample.slice(cursor)); return { regex: out, flags, fields, named: true }; } +/** Re-compile a paint spec using per-field sample values (the values each column takes across + * lines, in span order) so constant wrappers are peeled off — the "refine from data" step. */ +export function refinePaintPattern(spec: ColumnPatternSpec, fieldSamples: string[][]): CompiledColumnPattern { + if (spec.mode !== 'paint' || !spec.sample || !spec.spans || spec.spans.length === 0) { + return compileColumnPattern(spec); + } + return compilePaint(spec.sample, spec.spans, spec.flags || '', fieldSamples); +} + function compileRawRegex(pattern: string, flags: string): CompiledColumnPattern { new RegExp(pattern, flags); // throws on invalid — caller surfaces the message const names: string[] = []; diff --git a/src/main/index.ts b/src/main/index.ts index 541c9e1..f9474bd 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -39,7 +39,7 @@ import { getPatternLog, clearPatternLog, logPattern, PatternLogEntry, flushPatte import { compilePattern, CompileInput } from './compilePattern'; import { parseTimestampFast } from './timestampParse'; import { carryForwardTimestamps, buildOriginTags, formatWallClock } from './mergeTimeline'; -import { compileColumnPattern, makeColumnExtractor, ColumnPatternSpec } from './columnPattern'; +import { compileColumnPattern, makeColumnExtractor, refinePaintPattern, CompiledColumnPattern, ColumnPatternSpec } from './columnPattern'; import { parseVtraceToFile } from './vtraceParse'; import { runTrendJob } from './trendWorkerClient'; import { resolveScope, isWholeFile, scopeInfo, forEachScopeLine, ScopeResolverContext } from './scope'; @@ -3822,34 +3822,51 @@ function saveColumnPatternsStore(items: ColumnPatternSaved[]): void { ipcMain.handle('column-pattern-preview', async (_, spec: ColumnPatternSpec, opts?: { sampleLines?: number }) => { try { - const compiled = compileColumnPattern(spec); - const extractor = makeColumnExtractor(compiled); const N = Math.max(1, Math.min(500, opts?.sampleLines ?? 200)); - const rows: string[][] = []; - let matched = 0; - let scanned = 0; - const pushLine = (text: string) => { - scanned++; - const vals = extractor(text); - if (vals) { matched++; if (rows.length < N) rows.push(vals); } - }; - // Always test the user's own sample line first (they author against it). - if (spec.sample && spec.sample.trim()) pushLine(spec.sample); + // Gather the candidate lines once (the user's sample first, then the file head). + const sampleLines: string[] = []; + if (spec.sample && spec.sample.trim()) sampleLines.push(spec.sample); const handler = getFileHandler(); if (handler) { - const total = handler.getTotalLines(); - const scanTo = Math.min(total, N * 4); - const batch = handler.getLines(0, scanTo); - for (const line of batch) { - if (rows.length >= N) break; - if (!line.text.trim()) continue; - pushLine(line.text); + const scanTo = Math.min(handler.getTotalLines(), N * 4); + for (const line of handler.getLines(0, scanTo)) { + if (line.text.trim()) sampleLines.push(line.text); } } + const collect = (compiled: CompiledColumnPattern) => { + const extractor = makeColumnExtractor(compiled); + const rows: string[][] = []; + let matched = 0; + let scanned = 0; + for (const text of sampleLines) { + scanned++; + const vals = extractor(text); + if (vals) { matched++; if (rows.length < N) rows.push(vals); } + } + return { rows, matched, scanned }; + }; + + let compiled = compileColumnPattern(spec); + let { rows, matched, scanned } = collect(compiled); + + // ✨ Refine from data (paint mode): use the extracted column values to peel each column's + // constant WRAPPER (/…/, xx…xx, …blah…) and re-compile, then re-extract with the smarter + // pattern. No-op when no wrapper is shared across values. + let refined = false; + if (spec.mode === 'paint' && rows.length >= 2 && compiled.fields.length > 0) { + const fieldSamples = compiled.fields.map((_f, ci) => rows.map(r => r[ci]).filter(Boolean)); + const refinedCompiled = refinePaintPattern(spec, fieldSamples); + if (refinedCompiled.regex !== compiled.regex) { + compiled = refinedCompiled; + ({ rows, matched, scanned } = collect(compiled)); + refined = true; + } + } + return { success: true, regex: compiled.regex, flags: compiled.flags, fields: compiled.fields, named: compiled.named, - rows, matched, scanned, + rows, matched, scanned, refined, }; } catch (e) { return { success: false, error: String(e instanceof Error ? e.message : e) }; diff --git a/src/renderer/renderer.ts b/src/renderer/renderer.ts index a196782..c1909b4 100644 --- a/src/renderer/renderer.ts +++ b/src/renderer/renderer.ts @@ -15818,7 +15818,7 @@ async function patcolPreview(): Promise { patcolCompiled = { regex: res.regex, flags: res.flags || '', fields: res.fields || [], named: !!res.named }; if (out) out.textContent = res.regex || '—'; const rate = res.scanned ? Math.round((res.matched / res.scanned) * 100) : 0; - if (status) status.textContent = `${res.fields.length} cols · matched ${(res.matched || 0).toLocaleString()}/${(res.scanned || 0).toLocaleString()} (${rate}%)`; + if (status) status.textContent = `${res.fields.length} cols · matched ${(res.matched || 0).toLocaleString()}/${(res.scanned || 0).toLocaleString()} (${rate}%)${res.refined ? ' · ✨ refined from data' : ''}`; patcolRenderPreview(res); } catch (e) { if (status) status.textContent = 'Failed'; diff --git a/src/tests/columnPattern.test.ts b/src/tests/columnPattern.test.ts index 88ea980..46d42e0 100644 --- a/src/tests/columnPattern.test.ts +++ b/src/tests/columnPattern.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { compileColumnPattern, makeColumnExtractor, ColumnPatternSpec } from '../main/columnPattern'; +import { compileColumnPattern, makeColumnExtractor, refinePaintPattern, ColumnPatternSpec } from '../main/columnPattern'; function extract(spec: ColumnPatternSpec, line: string): string[] | null { return makeColumnExtractor(compileColumnPattern(spec))(line); @@ -109,3 +109,36 @@ describe('compileColumnPattern — raw regex mode', () => { expect(() => compileColumnPattern({ mode: 'regex', pattern: '(' })).toThrow(); }); }); + +describe('refinePaintPattern — refine from data (peel constant wrappers)', () => { + it('peels a /…/ wrapper detected across the column values', () => { + const sample = 'GET /abc/ 200'; + const spans = [{ start: 4, end: 9, name: 'id' }]; // "/abc/" + const c = refinePaintPattern({ mode: 'paint', sample, spans }, [['/abc/', '/xyz/', '/12/']]); + expect(c.regex).toContain('/'); // slash kept literal + expect(makeColumnExtractor(c)('GET /q9/ 200')).toEqual(['q9']); // value inside generalises + }); + + it('peels an xx…xx wrapper (arbitrary constant affix, not a bracket)', () => { + const sample = 'u=xxfooxx end'; + const spans = [{ start: 2, end: 9, name: 'v' }]; // "xxfooxx" + const c = refinePaintPattern({ mode: 'paint', sample, spans }, [['xxfooxx', 'xxbarxx', 'xxbazxx']]); + expect(c.regex).toContain('xx'); + expect(makeColumnExtractor(c)('u=xxwowxx end')).toEqual(['wow']); + }); + + it('adds no wrapper when the sample values do not share one', () => { + const sample = 'a foo b'; + const spans = [{ start: 2, end: 5, name: 'w' }]; // "foo" + const c = refinePaintPattern({ mode: 'paint', sample, spans }, [['foo', 'bar', 'baz']]); + expect(makeColumnExtractor(c)('a zzz b')).toEqual(['zzz']); + }); + + it('peels a multi-char trailing border like " ->" (whitespace-flexible)', () => { + const sample = 'adb -> ok'; + const spans = [{ start: 0, end: 7, name: 'cmd' }]; // "adb ->" + const c = refinePaintPattern({ mode: 'paint', sample, spans }, [['adb ->', 'xyz ->', 'foo ->']]); + expect(makeColumnExtractor(c)('xyz -> ok')).toEqual(['xyz']); // value = just "xyz" + expect(makeColumnExtractor(c)('foo -> ok')).toEqual(['foo']); // 1 space also matches (\\s+) + }); +});