diff --git a/src/Filee.Cli/Cli.cs b/src/Filee.Cli/Cli.cs index fd79ac15a..7984da40a 100644 --- a/src/Filee.Cli/Cli.cs +++ b/src/Filee.Cli/Cli.cs @@ -1,6 +1,7 @@ // The "filee" commands: convert, watch, formats, presets, engines. Writes results for people (one line per file) or, with -// --json, one JSON document for other programs. Exit codes: 0 nothing failed (skipped files included), 1 some -// failed or were cancelled, 2 wrong arguments or a missing input, 3 nothing to convert. +// --json, one JSON document for other programs; while it runs, convert --json writes events to stderr and watch --json +// to stdout, one JSON object per line. Exit codes: 0 nothing failed (skipped files included), 1 some failed or were +// cancelled, 2 wrong arguments or a missing input, 3 nothing to convert. using System.Text.Json; using Filee.Core.Conversion; @@ -75,7 +76,8 @@ private static async Task ConvertAsync(CliOptions options, CliHost host, Te ? FileSorter.Sort(files.Select(FileStamp.Read).ToList(), file => file, sort, foldersTogether: true).Select(file => file.FilePath).ToList() : files; - var job = await RunJobAsync(ordered, preset, host, options.Json || options.Quiet ? null : output, cancellationToken); + var events = options.Json && !options.Quiet ? new JobEvents(errors, host.Texts, host.Catalog, progress: true) : null; + var job = await RunJobAsync(ordered, preset, host, options.Json || options.Quiet ? null : output, events, cancellationToken); var done = job.Files.Count(f => f.State == FileState.Done); var failed = job.Files.Count(f => f.State is FileState.Failed or FileState.Cancelled); if (options.Json) @@ -105,15 +107,18 @@ private static async Task WatchAsync(CliOptions options, CliHost host, Text preset.Output.KeepFoldersFrom = rule.Folder; // as the app's watch folders: subfolders stay subfolders var settle = options.SettleSeconds is { } seconds ? TimeSpan.FromSeconds(seconds) : (TimeSpan?)null; + var events = options.Json ? new JobEvents(output, host.Texts, host.Catalog, progress: false) : null; await using var watcher = new FolderWatcher(rule, - async (files, token) => await RunJobAsync(files, preset, host, options.Quiet ? null : output, token), + async (files, token) => await RunJobAsync(files, preset, host, options.Quiet || options.Json ? null : output, events, token), settleTime: settle); if (!watcher.Start()) { errors.WriteLine($"filee: {watcher.Error}"); return UsageError; } - if (!options.Quiet) + if (events is not null) + events.Started(rule.Folder, host.Texts.DisplayName(preset), preset.TargetFormat, rule.ResolvedOutputFolder); + else if (!options.Quiet) output.WriteLine($"Watching {rule.Folder} → {host.Texts.DisplayName(preset)}, saving to {rule.ResolvedOutputFolder}. Press Ctrl+C to stop."); try { @@ -122,18 +127,26 @@ private static async Task WatchAsync(CliOptions options, CliHost host, Text catch (OperationCanceledException) { } + if (events is not null) + { + await watcher.DisposeAsync(); // a running batch ends first, so its "file" events come before "stopped" + events.Stopped(); + } return Success; } - /// Runs one job and prints each file's result as soon as it is final (when is given). + /// + /// Runs one job and prints each file's result as soon as it is final: a line to , a "file" + /// event to (each when given), with "progress" events while files run. + /// private static async Task RunJobAsync(IReadOnlyList files, Preset preset, CliHost host, TextWriter? output, - CancellationToken cancellationToken) + JobEvents? events, CancellationToken cancellationToken) { var job = new ConversionJob(files, preset, host.Texts.DisplayName(preset)); var printed = new HashSet(); void Print(object? sender, ConversionJob updated) { - if (output is null || updated != job) + if ((output is null && events is null) || updated != job) return; lock (printed) { @@ -142,7 +155,12 @@ void Print(object? sender, ConversionJob updated) // always raises the event as itself; only the last call below, after the job, has no sender. foreach (var file in job.Files.Where(f => f.State is not (FileState.Pending or FileState.Running) && (sender is null || f.State != FileState.Done || f.Outputs.Count > 0) && printed.Add(f))) - output.WriteLine(Describe(file, host.Texts)); + { + output?.WriteLine(Describe(file, host.Texts)); + events?.File(file); + } + if (sender is not null) + events?.Progress(job); } } @@ -205,28 +223,7 @@ private static void WriteJson(ConversionJob job, ILocalizer texts, ConverterCata foreach (var file in job.Files) { json.WriteStartObject(); - json.WriteString("source", file.SourcePath); - json.WriteString("state", file.State.ToString().ToLowerInvariant()); - json.WriteStartArray("outputs"); - foreach (var path in file.Outputs) - json.WriteStringValue(path); - json.WriteEndArray(); - if (file.State is FileState.Failed or FileState.Cancelled) - json.WriteString("error", Error(file, texts)); - else if (file.HasNotice) - json.WriteString("notice", Notice(file, texts)); - if (file.MissingFonts.Count > 0) - { - json.WriteStartArray("missingFonts"); - foreach (var font in file.MissingFonts) - json.WriteStringValue(font); - json.WriteEndArray(); - } - if (file.State is FileState.Failed or FileState.Cancelled && file.ErrorKey is not null) - json.WriteString("errorKey", file.ErrorKey); - if (file.State == FileState.Failed && file.ErrorKey == "error.no_route" - && file.FailedRoute is { } request && optional.MissingPackages(request) is { Count: > 0 } packages) - WriteStrings(json, "needsEngines", packages); + WriteFileFields(json, file, texts, optional); json.WriteEndObject(); } json.WriteEndArray(); @@ -235,6 +232,33 @@ private static void WriteJson(ConversionJob job, ILocalizer texts, ConverterCata output.WriteLine(System.Text.Encoding.UTF8.GetString(stream.ToArray())); } + /// A file's fields in the final document of 'convert --json', also those of its "file" event. + private static void WriteFileFields(Utf8JsonWriter json, FileResult file, ILocalizer texts, OptionalEngineRoutes optional) + { + json.WriteString("source", file.SourcePath); + json.WriteString("state", file.State.ToString().ToLowerInvariant()); + json.WriteStartArray("outputs"); + foreach (var path in file.Outputs) + json.WriteStringValue(path); + json.WriteEndArray(); + if (file.State is FileState.Failed or FileState.Cancelled) + json.WriteString("error", Error(file, texts)); + else if (file.HasNotice) + json.WriteString("notice", Notice(file, texts)); + if (file.MissingFonts.Count > 0) + { + json.WriteStartArray("missingFonts"); + foreach (var font in file.MissingFonts) + json.WriteStringValue(font); + json.WriteEndArray(); + } + if (file.State is FileState.Failed or FileState.Cancelled && file.ErrorKey is not null) + json.WriteString("errorKey", file.ErrorKey); + if (file.State == FileState.Failed && file.ErrorKey == "error.no_route" + && file.FailedRoute is { } request && optional.MissingPackages(request) is { Count: > 0 } packages) + WriteStrings(json, "needsEngines", packages); + } + /// The preset from --to or --preset, with --quality and --overwrite/--skip applied (a copy). private static Preset? ResolvePreset(CliOptions options, CliHost host, TextWriter errors) { @@ -461,6 +485,83 @@ private static int ListPresets(CliOptions options, CliHost host, TextWriter outp return Success; } + /// + /// The events of 'convert --json' (to stderr) and 'watch --json' (to stdout), one JSON object per line. A "file" event + /// has the fields of the file's entry in the final document. "progress" (convert only) comes when a running file's + /// fraction moves past 0 or changes, at most about every 150 ms per file, like the progress of 'engines install'. + /// + private sealed class JobEvents(TextWriter writer, ILocalizer texts, ConverterCatalog catalog, bool progress) + { + private const long ProgressIntervalMs = 150; + private readonly Lock _gate = new(); + private readonly OptionalEngineRoutes _optional = new(catalog); + private readonly Dictionary _reported = []; + + public void Started(string folder, string preset, string target, string outputFolder) + { + lock (_gate) + WriteLine("started", json => + { + json.WriteString("folder", folder); + json.WriteString("preset", preset); + json.WriteString("target", target); + json.WriteString("outputFolder", outputFolder); + }); + } + + public void File(FileResult file) + { + lock (_gate) + { + _reported.Remove(file); + WriteLine("file", json => WriteFileFields(json, file, texts, _optional)); + } + } + + public void Progress(ConversionJob job) + { + if (!progress) + return; + var now = Environment.TickCount64; + lock (_gate) + { + foreach (var file in job.Files.Where(f => f.State == FileState.Running)) + { + var fraction = Math.Round(Math.Clamp(file.Progress, 0, 1), 4); + var (lastFraction, lastAt) = _reported.GetValueOrDefault(file, (0.0, now - ProgressIntervalMs)); + if (fraction == lastFraction || now - lastAt < ProgressIntervalMs) + continue; + _reported[file] = (fraction, now); + WriteLine("progress", json => + { + json.WriteString("source", file.SourcePath); + json.WriteNumber("fraction", fraction); + }); + } + } + } + + public void Stopped() + { + lock (_gate) + WriteLine("stopped", _ => { }); + } + + private void WriteLine(string name, Action write) + { + using var stream = new MemoryStream(); + using (var json = new Utf8JsonWriter(stream)) + { + json.WriteStartObject(); + json.WriteString("event", name); + write(json); + json.WriteEndObject(); + } + writer.WriteLine(System.Text.Encoding.UTF8.GetString(stream.ToArray())); + writer.Flush(); + } + } + private sealed class OptionalEngineRoutes { private readonly ConverterCatalog _catalog; diff --git a/src/Filee.Cli/Help.cs b/src/Filee.Cli/Help.cs index 1615a61ec..7224be63d 100644 --- a/src/Filee.Cli/Help.cs +++ b/src/Filee.Cli/Help.cs @@ -87,12 +87,16 @@ no creation date (some Linux file systems), created uses its change or modified --skip Leave existing output files alone --keep-dates Give converted files the created and modified dates of the originals --zip Save the result of each file as one ZIP (a Shapefile's files together) - --json Print the result as JSON (for scripts and other programs) + --json Print the result as JSON (for scripts and other programs); events go to stderr --quiet Print nothing but errors + With --json, stdout gets one JSON document when the job is done. While it runs, stderr gets one JSON object per + line: {"event":"progress","source":…,"fraction":…} (0 to 1, about every 150 ms while it changes) and + {"event":"file",…} as each file is done, with the fields of its entry in "files". + """; - private static string Watch => FormattableString.Invariant($""" + private static string Watch => FormattableString.Invariant($$""" Usage: filee watch (--to | --preset ) [options] Converts files that land in the folder until Ctrl+C. Files are converted once they are complete (no @@ -105,10 +109,15 @@ no creation date (some Linux file systems), created uses its change or modified -r, --recursive Also watch sub folders (their files keep the sub folder in the output) --move-originals Move converted sources to \originals; files already waiting are converted at start - --settle How long a file must stay unchanged (default {FolderWatcher.DefaultSettleTime.TotalSeconds}) + --settle How long a file must stay unchanged (default {{FolderWatcher.DefaultSettleTime.TotalSeconds}}) --keep-dates Give converted files the created and modified dates of the originals --zip Save the result of each file as one ZIP (a Shapefile's files together) + --json Print events as JSON instead, one object per line (for scripts and other programs) --quiet Print nothing but errors + With --json, stdout gets nothing but events: {"event":"started",…} with the folder, preset, target and output + folder, {"event":"file",…} as each file is done (the fields of 'filee convert --json'), and + {"event":"stopped"} on Ctrl+C. + """); } diff --git a/tests/Filee.Cli.Tests/CliTests.cs b/tests/Filee.Cli.Tests/CliTests.cs index f778b2376..3157777dc 100644 --- a/tests/Filee.Cli.Tests/CliTests.cs +++ b/tests/Filee.Cli.Tests/CliTests.cs @@ -624,7 +624,7 @@ public async Task Case_distinct_inputs_are_both_converted_and_exact_duplicates_a var (code, output, errors) = await Run("convert", lower, upper, lower, "--to", "jpg", "-o", outputFolder, "--json"); Assert.Equal(Cli.Success, code); - Assert.Equal("", errors); + OnlyEvents(errors); using var json = JsonDocument.Parse(output); Assert.Equal(2, json.RootElement.GetProperty("converted").GetInt32()); Assert.Equal(2, json.RootElement.GetProperty("files").GetArrayLength()); @@ -644,7 +644,7 @@ public async Task Unicode_spaces_and_quotes_stay_in_a_single_input_argument() var (code, output, errors) = await Run("convert", source, "--to", "jpg", "-o", outputFolder, "--json"); Assert.Equal(Cli.Success, code); - Assert.Equal("", errors); + OnlyEvents(errors); using var json = JsonDocument.Parse(output); Assert.Equal(1, json.RootElement.GetProperty("converted").GetInt32()); var result = json.RootElement.GetProperty("files")[0].GetProperty("outputs")[0].GetString(); @@ -700,7 +700,7 @@ public async Task Fon_single_resource_omits_size_suffix_in_all_five_formats(stri var (code, output, errors) = await Run("convert", source, "--to", format, "-o", json, "--json"); Assert.Equal(Cli.Success, code); - Assert.Equal("", errors); + OnlyEvents(errors); Assert.Equal([Path.Combine(json, "Name." + format)], Outputs(output)); Assert.Equal(["Name." + format], Directory.EnumerateFiles(json).Select(Path.GetFileName)); @@ -833,6 +833,116 @@ private static JsonElement Result(string output) private static string[] Strings(JsonElement value) => value.EnumerateArray().Select(e => e.GetString()!).ToArray(); + private static List Events(string lines) => + lines.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Select(Result).ToList(); + + private static void OnlyEvents(string errors) => + Assert.All(Events(errors), e => Assert.Contains(e.GetProperty("event").GetString(), new[] { "progress", "file" })); + + private static (string Name, string Value)[] Fields(IEnumerable properties) => + properties.Select(p => (p.Name, JsonSerializer.Serialize(p.Value))).ToArray(); + + [Fact] + public async Task Convert_json_writes_progress_and_file_events_to_stderr() + { + var errors = new LineWriter(); + var converter = new ControlledConverter("built-in", true, [new("md", "html")]) + { + BeforeConvert = async (progress, token) => + { + progress?.Report(0.5); + for (var i = 0; i < 200 && !errors.Contains("\"event\":\"progress\""); i++) + await Task.Delay(25, token); + }, + }; + var source = Input("md"); + var output = new StringWriter(); + + var code = await Cli.RunAsync(["convert", source, "--to", "html", "--json"], output, errors, + TestContext.Current.CancellationToken, () => ControlledHost(new([converter]))); + + Assert.Equal(Cli.Success, code); + var file = FileResultOf(output.ToString()); + var events = Events(errors.ToString()); + Assert.Contains(events, e => e.GetProperty("event").GetString() == "progress" + && e.GetProperty("source").GetString() == source + && e.GetProperty("fraction").GetDouble() is > 0 and < 1); + var last = events[^1]; + Assert.Equal("file", last.GetProperty("event").GetString()); + Assert.Equal(Fields(file.EnumerateObject()), Fields(last.EnumerateObject().Skip(1))); + Assert.Single(events, e => e.GetProperty("event").GetString() == "file"); + } + + [Fact] + public async Task Convert_json_with_quiet_writes_no_events() + { + var (code, output, errors) = await RunWith(new([new ControlledConverter("built-in", true, [new("md", "html")])]), + "convert", Input("md"), "--to", "html", "--json", "--quiet"); + + Assert.Equal(Cli.Success, code); + Assert.Equal("", errors); + Assert.Equal(1, Result(output).GetProperty("converted").GetInt32()); + } + + [Fact] + public async Task Watch_json_prints_nothing_but_events() + { + using var stop = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var output = new LineWriter(); + var watching = Cli.RunAsync(["watch", _folder, "--to", "jpg", "--settle", "0.2", "--json"], output, TextWriter.Null, + stop.Token, () => CliHost.Create(_settings)); + + await Task.Delay(500, TestContext.Current.CancellationToken); + Png("arrived.png"); + for (var i = 0; i < 100 && !output.Contains("\"event\":\"file\""); i++) + await Task.Delay(100, TestContext.Current.CancellationToken); + stop.Cancel(); + + Assert.Equal(Cli.Success, await watching); + var events = Events(output.ToString()); + Assert.Equal("started", events[0].GetProperty("event").GetString()); + Assert.Equal(Path.GetFullPath(_folder), events[0].GetProperty("folder").GetString()); + Assert.Equal("jpg", events[0].GetProperty("target").GetString()); + Assert.Equal(Path.Combine(Path.GetFullPath(_folder), "converted"), events[0].GetProperty("outputFolder").GetString()); + var file = Assert.Single(events, e => e.GetProperty("event").GetString() == "file"); + Assert.Equal("done", file.GetProperty("state").GetString()); + Assert.EndsWith("arrived.png", file.GetProperty("source").GetString()); + Assert.EndsWith("arrived.jpg", file.GetProperty("outputs")[0].GetString()); + Assert.Equal("stopped", events[^1].GetProperty("event").GetString()); + Assert.Equal(3, events.Count); + } + + private sealed class LineWriter : StringWriter + { + private readonly Lock _gate = new(); + + public override void Write(char value) + { + lock (_gate) + base.Write(value); + } + + public override void Write(string? value) + { + lock (_gate) + base.Write(value); + } + + public override void WriteLine(string? value) + { + lock (_gate) + base.WriteLine(value); + } + + public bool Contains(string text) => ToString().Contains(text, StringComparison.Ordinal); + + public override string ToString() + { + lock (_gate) + return base.ToString(); + } + } + private static void Hints(JsonElement file, params string[] packages) { Assert.Equal("failed", file.GetProperty("state").GetString()); @@ -899,7 +1009,7 @@ public async Task Convert_json_keeps_existing_fields() var catalog = new ConverterCatalog([converter]); var (code, output, errors) = await RunWith(catalog, "convert", source, "--to", "html", "--json"); Assert.Equal(0, code); - Assert.Equal("", errors); + OnlyEvents(errors); var root = Result(output); Assert.Equal(["preset", "target", "converted", "skipped", "failed", "files"], root.EnumerateObject().Select(p => p.Name)); Assert.Equal("HTML", root.GetProperty("preset").GetString()); @@ -959,7 +1069,7 @@ public async Task No_route_distinguishes_missing_packages_from_impossible_target var catalog = new ConverterCatalog([Package("pandoc", false, new ConversionEdge("md", "pptx"))]); var (code, output, errors) = await RunWith(catalog, "convert", Input("md"), "--to", "pptx", "--json"); Assert.Equal(1, code); - Assert.Equal("", errors); + OnlyEvents(errors); Hints(FileResultOf(output), "pandoc"); (code, output, _) = await RunWith(catalog, "convert", Input("txt"), "--to", "mp3", "--json"); Assert.Equal(1, code); @@ -1122,7 +1232,7 @@ public async Task Cancelled_files_do_not_invent_failure_metadata() var root = Result(output.ToString()); Assert.Equal(1, root.GetProperty("failed").GetInt32()); Assert.Equal(0, root.GetProperty("converted").GetInt32()); - Assert.Equal("", errors.ToString()); + OnlyEvents(errors.ToString()); var file = root.GetProperty("files")[0]; Assert.Equal(["source", "state", "outputs", "error"], file.EnumerateObject().Select(p => p.Name)); Assert.Equal("cancelled", file.GetProperty("state").GetString()); @@ -1242,9 +1352,14 @@ private sealed class ControlledConverter(string id, bool available, IReadOnlyLis public bool NeedsShaping { get; init; } public bool ReadsImages { get; set; } public Func>>? Convert { get; set; } + public Func?, CancellationToken, Task>? BeforeConvert { get; init; } public EngineStatus GetStatus() => Available ? EngineStatus.Available() : EngineStatus.Unavailable(Reason); - public Task> ConvertAsync(ConversionStep step, IProgress? progress, CancellationToken cancellationToken) => - Convert?.Invoke(step, cancellationToken) ?? Write(step); + public async Task> ConvertAsync(ConversionStep step, IProgress? progress, CancellationToken cancellationToken) + { + if (BeforeConvert is not null) + await BeforeConvert(progress, cancellationToken); + return await (Convert?.Invoke(step, cancellationToken) ?? Write(step)); + } public static Task> Write(ConversionStep step) { if (step.Output.Allocate(step.To) is not { } path)