Skip to content

feat(cli): add JSON event output #1110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
KnifeLemon merged 1 commit into from
Nov 4, 2026
Merged

feat(cli): add JSON event output #1110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 132 additions & 31 deletions src/Filee.Cli/Cli.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// The "filee" commands: convert, watch, formats, presets, engines. Writes results for people (one line per file) or, with
// --json, one JSON document for other programs. Exit codes: 0 nothing failed (skipped files included), 1 some
// failed or were cancelled, 2 wrong arguments or a missing input, 3 nothing to convert.
// --json, one JSON document for other programs; while it runs, convert --json writes events to stderr and watch --json
// to stdout, one JSON object per line. Exit codes: 0 nothing failed (skipped files included), 1 some failed or were
// cancelled, 2 wrong arguments or a missing input, 3 nothing to convert.

using System.Text.Json;
using Filee.Core.Conversion;
Expand Down Expand Up @@ -75,7 +76,8 @@ private static async Task<int> ConvertAsync(CliOptions options, CliHost host, Te
? FileSorter.Sort(files.Select(FileStamp.Read).ToList(), file => file, sort, foldersTogether: true).Select(file => file.FilePath).ToList()
: files;

var job = await RunJobAsync(ordered, preset, host, options.Json || options.Quiet ? null : output, cancellationToken);
var events = options.Json && !options.Quiet ? new JobEvents(errors, host.Texts, host.Catalog, progress: true) : null;
var job = await RunJobAsync(ordered, preset, host, options.Json || options.Quiet ? null : output, events, cancellationToken);
var done = job.Files.Count(f => f.State == FileState.Done);
var failed = job.Files.Count(f => f.State is FileState.Failed or FileState.Cancelled);
if (options.Json)
Expand Down Expand Up @@ -105,15 +107,18 @@ private static async Task<int> WatchAsync(CliOptions options, CliHost host, Text
preset.Output.KeepFoldersFrom = rule.Folder; // as the app's watch folders: subfolders stay subfolders

var settle = options.SettleSeconds is { } seconds ? TimeSpan.FromSeconds(seconds) : (TimeSpan?)null;
var events = options.Json ? new JobEvents(output, host.Texts, host.Catalog, progress: false) : null;
await using var watcher = new FolderWatcher(rule,
async (files, token) => await RunJobAsync(files, preset, host, options.Quiet ? null : output, token),
async (files, token) => await RunJobAsync(files, preset, host, options.Quiet || options.Json ? null : output, events, token),
settleTime: settle);
if (!watcher.Start())
{
errors.WriteLine($"filee: {watcher.Error}");
return UsageError;
}
if (!options.Quiet)
if (events is not null)
events.Started(rule.Folder, host.Texts.DisplayName(preset), preset.TargetFormat, rule.ResolvedOutputFolder);
else if (!options.Quiet)
output.WriteLine($"Watching {rule.Folder} → {host.Texts.DisplayName(preset)}, saving to {rule.ResolvedOutputFolder}. Press Ctrl+C to stop.");
try
{
Expand All @@ -122,18 +127,26 @@ private static async Task<int> WatchAsync(CliOptions options, CliHost host, Text
catch (OperationCanceledException)
{
}
if (events is not null)
{
await watcher.DisposeAsync(); // a running batch ends first, so its "file" events come before "stopped"
events.Stopped();
}
return Success;
}

/// <summary>Runs one job and prints each file's result as soon as it is final (when <paramref name="output"/> is given).</summary>
/// <summary>
/// Runs one job and prints each file's result as soon as it is final: a line to <paramref name="output"/>, a "file"
/// event to <paramref name="events"/> (each when given), with "progress" events while files run.
/// </summary>
private static async Task<ConversionJob> RunJobAsync(IReadOnlyList<string> files, Preset preset, CliHost host, TextWriter? output,
CancellationToken cancellationToken)
JobEvents? events, CancellationToken cancellationToken)
{
var job = new ConversionJob(files, preset, host.Texts.DisplayName(preset));
var printed = new HashSet<FileResult>();
void Print(object? sender, ConversionJob updated)
{
if (output is null || updated != job)
if ((output is null && events is null) || updated != job)
return;
lock (printed)
{
Expand All @@ -142,7 +155,12 @@ void Print(object? sender, ConversionJob updated)
// always raises the event as itself; only the last call below, after the job, has no sender.
foreach (var file in job.Files.Where(f => f.State is not (FileState.Pending or FileState.Running)
&& (sender is null || f.State != FileState.Done || f.Outputs.Count > 0) && printed.Add(f)))
output.WriteLine(Describe(file, host.Texts));
{
output?.WriteLine(Describe(file, host.Texts));
events?.File(file);
}
if (sender is not null)
events?.Progress(job);
}
}

Expand Down Expand Up @@ -205,28 +223,7 @@ private static void WriteJson(ConversionJob job, ILocalizer texts, ConverterCata
foreach (var file in job.Files)
{
json.WriteStartObject();
json.WriteString("source", file.SourcePath);
json.WriteString("state", file.State.ToString().ToLowerInvariant());
json.WriteStartArray("outputs");
foreach (var path in file.Outputs)
json.WriteStringValue(path);
json.WriteEndArray();
if (file.State is FileState.Failed or FileState.Cancelled)
json.WriteString("error", Error(file, texts));
else if (file.HasNotice)
json.WriteString("notice", Notice(file, texts));
if (file.MissingFonts.Count > 0)
{
json.WriteStartArray("missingFonts");
foreach (var font in file.MissingFonts)
json.WriteStringValue(font);
json.WriteEndArray();
}
if (file.State is FileState.Failed or FileState.Cancelled && file.ErrorKey is not null)
json.WriteString("errorKey", file.ErrorKey);
if (file.State == FileState.Failed && file.ErrorKey == "error.no_route"
&& file.FailedRoute is { } request && optional.MissingPackages(request) is { Count: > 0 } packages)
WriteStrings(json, "needsEngines", packages);
WriteFileFields(json, file, texts, optional);
json.WriteEndObject();
}
json.WriteEndArray();
Expand All @@ -235,6 +232,33 @@ private static void WriteJson(ConversionJob job, ILocalizer texts, ConverterCata
output.WriteLine(System.Text.Encoding.UTF8.GetString(stream.ToArray()));
}

/// <summary>A file's fields in the final document of 'convert --json', also those of its "file" event.</summary>
private static void WriteFileFields(Utf8JsonWriter json, FileResult file, ILocalizer texts, OptionalEngineRoutes optional)
{
json.WriteString("source", file.SourcePath);
json.WriteString("state", file.State.ToString().ToLowerInvariant());
json.WriteStartArray("outputs");
foreach (var path in file.Outputs)
json.WriteStringValue(path);
json.WriteEndArray();
if (file.State is FileState.Failed or FileState.Cancelled)
json.WriteString("error", Error(file, texts));
else if (file.HasNotice)
json.WriteString("notice", Notice(file, texts));
if (file.MissingFonts.Count > 0)
{
json.WriteStartArray("missingFonts");
foreach (var font in file.MissingFonts)
json.WriteStringValue(font);
json.WriteEndArray();
}
if (file.State is FileState.Failed or FileState.Cancelled && file.ErrorKey is not null)
json.WriteString("errorKey", file.ErrorKey);
if (file.State == FileState.Failed && file.ErrorKey == "error.no_route"
&& file.FailedRoute is { } request && optional.MissingPackages(request) is { Count: > 0 } packages)
WriteStrings(json, "needsEngines", packages);
}

/// <summary>The preset from --to or --preset, with --quality and --overwrite/--skip applied (a copy).</summary>
private static Preset? ResolvePreset(CliOptions options, CliHost host, TextWriter errors)
{
Expand Down Expand Up @@ -461,6 +485,83 @@ private static int ListPresets(CliOptions options, CliHost host, TextWriter outp
return Success;
}

/// <summary>
/// The events of 'convert --json' (to stderr) and 'watch --json' (to stdout), one JSON object per line. A "file" event
/// has the fields of the file's entry in the final document. "progress" (convert only) comes when a running file's
/// fraction moves past 0 or changes, at most about every 150 ms per file, like the progress of 'engines install'.
/// </summary>
private sealed class JobEvents(TextWriter writer, ILocalizer texts, ConverterCatalog catalog, bool progress)
{
private const long ProgressIntervalMs = 150;
private readonly Lock _gate = new();
private readonly OptionalEngineRoutes _optional = new(catalog);
private readonly Dictionary<FileResult, (double Fraction, long At)> _reported = [];

public void Started(string folder, string preset, string target, string outputFolder)
{
lock (_gate)
WriteLine("started", json =>
{
json.WriteString("folder", folder);
json.WriteString("preset", preset);
json.WriteString("target", target);
json.WriteString("outputFolder", outputFolder);
});
}

public void File(FileResult file)
{
lock (_gate)
{
_reported.Remove(file);
WriteLine("file", json => WriteFileFields(json, file, texts, _optional));
}
}

public void Progress(ConversionJob job)
{
if (!progress)
return;
var now = Environment.TickCount64;
lock (_gate)
{
foreach (var file in job.Files.Where(f => f.State == FileState.Running))
{
var fraction = Math.Round(Math.Clamp(file.Progress, 0, 1), 4);
var (lastFraction, lastAt) = _reported.GetValueOrDefault(file, (0.0, now - ProgressIntervalMs));
if (fraction == lastFraction || now - lastAt < ProgressIntervalMs)
continue;
_reported[file] = (fraction, now);
WriteLine("progress", json =>
{
json.WriteString("source", file.SourcePath);
json.WriteNumber("fraction", fraction);
});
}
}
}

public void Stopped()
{
lock (_gate)
WriteLine("stopped", _ => { });
}

private void WriteLine(string name, Action<Utf8JsonWriter> write)
{
using var stream = new MemoryStream();
using (var json = new Utf8JsonWriter(stream))
{
json.WriteStartObject();
json.WriteString("event", name);
write(json);
json.WriteEndObject();
}
writer.WriteLine(System.Text.Encoding.UTF8.GetString(stream.ToArray()));
writer.Flush();
}
}

private sealed class OptionalEngineRoutes
{
private readonly ConverterCatalog _catalog;
Expand Down
15 changes: 12 additions & 3 deletions src/Filee.Cli/Help.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,12 +87,16 @@ no creation date (some Linux file systems), created uses its change or modified
--skip Leave existing output files alone
--keep-dates Give converted files the created and modified dates of the originals
--zip Save the result of each file as one ZIP (a Shapefile's files together)
--json Print the result as JSON (for scripts and other programs)
--json Print the result as JSON (for scripts and other programs); events go to stderr
--quiet Print nothing but errors

With --json, stdout gets one JSON document when the job is done. While it runs, stderr gets one JSON object per
line: {"event":"progress","source":…,"fraction":…} (0 to 1, about every 150 ms while it changes) and
{"event":"file",…} as each file is done, with the fields of its entry in "files".

""";

private static string Watch => FormattableString.Invariant($"""
private static string Watch => FormattableString.Invariant($$"""
Usage: filee watch <folder> (--to <format> | --preset <preset>) [options]

Converts files that land in the folder until Ctrl+C. Files are converted once they are complete (no
Expand All @@ -105,10 +109,15 @@ no creation date (some Linux file systems), created uses its change or modified
-r, --recursive Also watch sub folders (their files keep the sub folder in the output)
--move-originals Move converted sources to <folder>\originals; files already waiting are
converted at start
--settle <seconds> How long a file must stay unchanged (default {FolderWatcher.DefaultSettleTime.TotalSeconds})
--settle <seconds> How long a file must stay unchanged (default {{FolderWatcher.DefaultSettleTime.TotalSeconds}})
--keep-dates Give converted files the created and modified dates of the originals
--zip Save the result of each file as one ZIP (a Shapefile's files together)
--json Print events as JSON instead, one object per line (for scripts and other programs)
--quiet Print nothing but errors

With --json, stdout gets nothing but events: {"event":"started",…} with the folder, preset, target and output
folder, {"event":"file",…} as each file is done (the fields of 'filee convert --json'), and
{"event":"stopped"} on Ctrl+C.

""");
}
Loading
Loading