diff --git a/.github/workflows/remove-unicode-characters.yml b/.github/workflows/remove-unicode-characters.yml
index b42f32ac..4435f847 100644
--- a/.github/workflows/remove-unicode-characters.yml
+++ b/.github/workflows/remove-unicode-characters.yml
@@ -5,11 +5,13 @@ on:
branches: [main]
paths:
- 'se5/RemoveUnicodeCharacters/**'
+ - 'se5/RemoveUnicodeCharacters.Tests/**'
- 'se5/Plugin-Shared/**'
- '.github/workflows/remove-unicode-characters.yml'
pull_request:
paths:
- 'se5/RemoveUnicodeCharacters/**'
+ - 'se5/RemoveUnicodeCharacters.Tests/**'
- 'se5/Plugin-Shared/**'
- '.github/workflows/remove-unicode-characters.yml'
workflow_dispatch:
@@ -90,8 +92,22 @@ jobs:
echo "ref=$ref" >> "$GITHUB_OUTPUT"
echo "release=$release_flag" >> "$GITHUB_OUTPUT"
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Setup .NET 8
+ uses: actions/setup-dotnet@v4
+ with:
+ dotnet-version: '8.0.x'
+
+ - name: Test
+ run: dotnet test se5/RemoveUnicodeCharacters.Tests/RemoveUnicodeCharacters.Tests.csproj -c Release
+
build:
- needs: prepare
+ needs: [prepare, test]
runs-on: ubuntu-latest
strategy:
fail-fast: false
diff --git a/se5/RemoveUnicodeCharacters.Tests/BatchModeTests.cs b/se5/RemoveUnicodeCharacters.Tests/BatchModeTests.cs
new file mode 100644
index 00000000..e4e9de48
--- /dev/null
+++ b/se5/RemoveUnicodeCharacters.Tests/BatchModeTests.cs
@@ -0,0 +1,121 @@
+using SubtitleEdit.Plugins.RemoveUnicodeCharacters;
+using SubtitleEdit.Plugins.Shared;
+using System.Text.Json;
+
+namespace RemoveUnicodeCharacters.Tests;
+
+/// Subtitle Edit Batch convert: "batch" applies the batch list headless; Configure edits that list.
+public class BatchModeTests
+{
+ private static JsonElement Json(string json) => JsonDocument.Parse(json).RootElement.Clone();
+
+ private static PluginRequest Request(string? settingsJson, params string[] texts) => new()
+ {
+ RequestType = PluginRequestType.Batch,
+ Subtitle = new PluginSubtitle
+ {
+ Format = "Advanced Sub Station Alpha",
+ Paragraphs = texts.Select((t, i) => new PluginParagraph { StartMs = i * 1000, EndMs = i * 1000 + 900, Text = t, Style = "Top" }).ToList(),
+ },
+ Settings = settingsJson == null ? null : Json(settingsJson),
+ };
+
+ [Fact]
+ public void Batch_NoSettings_ReplacesOnlyBuiltInDefaults()
+ {
+ var response = BatchMode.BuildBatchResponse(Request(null, "♪ Привет ♫", "don’t", "Plain"));
+
+ Assert.Equal(PluginStatus.Ok, response.Status);
+ Assert.Equal(new[] { "# Привет #", "don’t", "Plain" }, response.Subtitle!.Paragraphs!.Select(p => p.Text));
+ Assert.Equal("Advanced Sub Station Alpha", response.Subtitle.Format);
+ Assert.All(response.Subtitle.Paragraphs!, p => Assert.Equal("Top", p.Style));
+ Assert.Null(response.Settings);
+ Assert.Null(response.SettingsVersion);
+ }
+
+ [Fact]
+ public void Batch_NothingToChange_ReturnsOkWithoutSubtitle()
+ {
+ var response = BatchMode.BuildBatchResponse(Request(null, "Привет", "café"));
+
+ Assert.Equal(PluginStatus.Ok, response.Status);
+ Assert.Null(response.Subtitle);
+ }
+
+ [Fact]
+ public void Batch_NotConfigured_UsesNonBlankWindowReplacementsOnly()
+ {
+ // Blank window entries are also saved for unchecked rows - they must not remove anything in batch.
+ var settings = "{\"replacements\":{\"2019\":\"'\",\"0410\":\"\",\"266A\":\"\"}}";
+
+ var response = BatchMode.BuildBatchResponse(Request(settings, "don’t А ♪ ♫"));
+
+ Assert.Equal("don't А ♪ #", response.Subtitle!.Paragraphs![0].Text);
+ }
+
+ [Fact]
+ public void Batch_ConfiguredList_ReplacesListedAndKeepsOthers()
+ {
+ var settings = "{\"replacements\":{\"2019\":\"'\"},\"batchReplacements\":{\"200B\":\"\",\"2014\":\"-\"},\"batchRemoveOthers\":false}";
+
+ var response = BatchMode.BuildBatchResponse(Request(settings, "ab — don’t ♪"));
+
+ Assert.Equal("ab - don’t ♪", response.Subtitle!.Paragraphs![0].Text);
+ Assert.Contains("2 ", response.Message);
+ }
+
+ [Fact]
+ public void Batch_RemoveOthers_RemovesUnlistedNonAnsi()
+ {
+ var settings = "{\"batchReplacements\":{\"266A\":\"#\"},\"batchRemoveOthers\":true}";
+
+ var response = BatchMode.BuildBatchResponse(Request(settings, "♪ Привет café ♪"));
+
+ Assert.Equal("# café #", response.Subtitle!.Paragraphs![0].Text);
+ }
+
+ [Fact]
+ public void Batch_LegacyRequestWithoutParagraphs_ReturnsSubRip()
+ {
+ var request = new PluginRequest
+ {
+ RequestType = PluginRequestType.Batch,
+ Subtitle = new PluginSubtitle { Format = "SubRip", SubRip = "1\n00:00:01,000 --> 00:00:02,000\n♪ Hi\n\n" },
+ };
+
+ var response = BatchMode.BuildBatchResponse(request);
+
+ Assert.Equal("SubRip", response.Subtitle!.Format);
+ Assert.Contains("# Hi", response.Subtitle.Native);
+ }
+
+ [Fact]
+ public void ConfigureSettings_KeepWindowReplacements()
+ {
+ var list = new SortedDictionary { [''] = "" };
+ var settings = BatchMode.BuildSettings(new Dictionary { ["2019"] = "'" }, null, list, removeOthers: true);
+
+ Assert.Equal("'", settings.GetProperty("replacements").GetProperty("2019").GetString());
+ Assert.Equal("", settings.GetProperty("batchReplacements").GetProperty("200B").GetString());
+ Assert.True(settings.GetProperty("batchRemoveOthers").GetBoolean());
+ Assert.Equal(new[] { '' }, BatchMode.LoadBatchList(settings).Keys);
+ }
+
+ [Fact]
+ public void RunSettings_KeepBatchConfiguration()
+ {
+ var previous = Json("{\"replacements\":{},\"batchReplacements\":{\"2014\":\"-\"},\"batchRemoveOthers\":true}");
+
+ var settings = BatchMode.BuildSettings(new Dictionary { ["266A"] = "*" }, previous);
+
+ Assert.Equal("*", settings.GetProperty("replacements").GetProperty("266A").GetString());
+ Assert.Equal("-", settings.GetProperty("batchReplacements").GetProperty("2014").GetString());
+ Assert.True(BatchMode.LoadRemoveOthers(settings));
+ }
+
+ [Fact]
+ public void ParseCharacters_AcceptsGlyphsAndCodes()
+ {
+ Assert.Equal(new[] { '—', '', '♪' }, BatchMode.ParseCharacters("—abc U+200B u+266a ♪ U+0041"));
+ }
+}
diff --git a/se5/RemoveUnicodeCharacters.Tests/RemoveUnicodeCharacters.Tests.csproj b/se5/RemoveUnicodeCharacters.Tests/RemoveUnicodeCharacters.Tests.csproj
new file mode 100644
index 00000000..2b0ff222
--- /dev/null
+++ b/se5/RemoveUnicodeCharacters.Tests/RemoveUnicodeCharacters.Tests.csproj
@@ -0,0 +1,27 @@
+
+
+
+ net8.0
+ enable
+ enable
+ false
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/se5/RemoveUnicodeCharacters/BatchMode.cs b/se5/RemoveUnicodeCharacters/BatchMode.cs
new file mode 100644
index 00000000..392c1949
--- /dev/null
+++ b/se5/RemoveUnicodeCharacters/BatchMode.cs
@@ -0,0 +1,311 @@
+using SubtitleEdit.Plugins.Shared;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace SubtitleEdit.Plugins.RemoveUnicodeCharacters;
+
+///
+/// Subtitle Edit Batch convert support. The window asks per character, so a batch run only
+/// touches the characters in the batch list (edited via Configure...). Until the list has
+/// been configured it holds the built-in defaults plus every non-blank replacement typed in the
+/// window. Other non-ANSI characters are kept unless "remove all others" is switched on.
+///
+public static class BatchMode
+{
+ public const int SettingsVersion = 1;
+
+ /// Window replacements (char hex → replacement), saved by every normal run.
+ public const string ReplacementsKey = "replacements";
+
+ /// Batch-only list (char hex → replacement); written by Configure, kept but ignored by a normal run.
+ public const string BatchReplacementsKey = "batchReplacements";
+
+ /// Batch-only: also remove non-ANSI characters that are not in the batch list.
+ public const string BatchRemoveOthersKey = "batchRemoveOthers";
+
+ /// Built-in default replacements (used when the user has no persisted setting for a character).
+ public static readonly IReadOnlyDictionary DefaultReplacements = new Dictionary
+ {
+ ['♪'] = "#",
+ ['♫'] = "#",
+ };
+
+ public static bool IsNonAnsi(char c) => c > 255;
+
+ public static string ToHex(char c) => ((int)c).ToString("X4");
+
+ public static Dictionary LoadReplacements(JsonElement? settings, string key = ReplacementsKey)
+ {
+ var result = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ if (settings is null || settings.Value.ValueKind != JsonValueKind.Object)
+ {
+ return result;
+ }
+
+ if (!settings.Value.TryGetProperty(key, out var replacements)
+ || replacements.ValueKind != JsonValueKind.Object)
+ {
+ return result;
+ }
+
+ foreach (var property in replacements.EnumerateObject())
+ {
+ if (property.Value.ValueKind == JsonValueKind.String)
+ {
+ result[property.Name] = property.Value.GetString() ?? string.Empty;
+ }
+ }
+
+ return result;
+ }
+
+ public static bool LoadRemoveOthers(JsonElement? settings) =>
+ settings is { ValueKind: JsonValueKind.Object } s
+ && s.TryGetProperty(BatchRemoveOthersKey, out var value)
+ && value.ValueKind == JsonValueKind.True;
+
+ ///
+ /// The characters a batch run replaces. Configured list if there is one, else the built-in
+ /// defaults plus the window's non-blank replacements (blank ones are also saved for rows the
+ /// user left unchecked, so they are no proof the user wants that character removed).
+ ///
+ public static SortedDictionary LoadBatchList(JsonElement? settings)
+ {
+ var result = new SortedDictionary();
+ if (settings is { ValueKind: JsonValueKind.Object } s && s.TryGetProperty(BatchReplacementsKey, out var configured)
+ && configured.ValueKind == JsonValueKind.Object)
+ {
+ foreach (var (hex, replacement) in LoadReplacements(settings, BatchReplacementsKey))
+ {
+ if (TryParseHex(hex, out var c))
+ {
+ result[c] = replacement;
+ }
+ }
+
+ return result;
+ }
+
+ foreach (var (c, replacement) in DefaultReplacements)
+ {
+ result[c] = replacement;
+ }
+
+ foreach (var (hex, replacement) in LoadReplacements(settings))
+ {
+ if (TryParseHex(hex, out var c))
+ {
+ if (replacement.Length > 0)
+ {
+ result[c] = replacement;
+ }
+ else
+ {
+ result.Remove(c); // the user cleared a built-in default
+ }
+ }
+ }
+
+ return result;
+ }
+
+ private static bool TryParseHex(string hex, out char c)
+ {
+ c = default;
+ if (int.TryParse(hex, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var code) && code is > 255 and <= 0xFFFF)
+ {
+ c = (char)code;
+ return true;
+ }
+
+ return false;
+ }
+
+ ///
+ /// Characters to add to the batch list from what the user typed: each non-ANSI character as
+ /// is, and "U+XXXX" codes for characters that are hard to type (e.g. U+200B zero-width space).
+ ///
+ public static List ParseCharacters(string? input)
+ {
+ var result = new List();
+ if (string.IsNullOrEmpty(input))
+ {
+ return result;
+ }
+
+ for (var i = 0; i < input.Length; i++)
+ {
+ if ((input[i] == 'U' || input[i] == 'u') && i + 1 < input.Length && input[i + 1] == '+')
+ {
+ var end = i + 2;
+ while (end < input.Length && end - (i + 2) < 4 && Uri.IsHexDigit(input[end]))
+ {
+ end++;
+ }
+
+ if (end - (i + 2) == 4 && TryParseHex(input.Substring(i + 2, 4), out var coded))
+ {
+ if (!result.Contains(coded))
+ {
+ result.Add(coded);
+ }
+
+ i = end - 1;
+ continue;
+ }
+ }
+
+ if (IsNonAnsi(input[i]) && !result.Contains(input[i]))
+ {
+ result.Add(input[i]);
+ }
+ }
+
+ return result;
+ }
+
+ /// Replaces the listed characters (and removes the rest if ).
+ public static string Apply(string text, IReadOnlyDictionary list, bool removeOthers, ref int occurrences)
+ {
+ StringBuilder? sb = null;
+ for (var i = 0; i < text.Length; i++)
+ {
+ var c = text[i];
+ string? replacement = null;
+ if (IsNonAnsi(c))
+ {
+ if (list.TryGetValue(c, out var listed))
+ {
+ replacement = listed;
+ }
+ else if (removeOthers)
+ {
+ replacement = string.Empty;
+ }
+ }
+
+ if (replacement == null)
+ {
+ sb?.Append(c);
+ continue;
+ }
+
+ sb ??= new StringBuilder(text.Length).Append(text, 0, i);
+ sb.Append(replacement);
+ occurrences++;
+ }
+
+ return sb?.ToString() ?? text;
+ }
+
+ ///
+ /// A "batch" request (Batch convert): apply the batch list to all lines, without UI.
+ /// Nothing to change is "ok" without a subtitle.
+ ///
+ public static PluginResponse BuildBatchResponse(PluginRequest request)
+ {
+ var list = LoadBatchList(request.Settings);
+ var removeOthers = LoadRemoveOthers(request.Settings);
+ var occurrences = 0;
+ var changedLines = 0;
+
+ PluginSubtitle subtitle;
+ if (request.Subtitle.Paragraphs is { } paragraphs)
+ {
+ var result = new List(paragraphs.Count);
+ foreach (var paragraph in paragraphs)
+ {
+ var copy = paragraph.Clone();
+ var text = paragraph.Text ?? string.Empty;
+ copy.Text = Apply(text, list, removeOthers, ref occurrences);
+ if (!ReferenceEquals(copy.Text, text))
+ {
+ changedLines++;
+ }
+
+ result.Add(copy);
+ }
+
+ subtitle = new PluginSubtitle { Format = request.Subtitle.Format, Paragraphs = result };
+ }
+ else
+ {
+ var blocks = SubRipParser.Parse(request.Subtitle.SubRip);
+ foreach (var block in blocks)
+ {
+ var text = block.Text;
+ block.Text = Apply(text, list, removeOthers, ref occurrences);
+ if (!ReferenceEquals(block.Text, text))
+ {
+ changedLines++;
+ }
+ }
+
+ subtitle = new PluginSubtitle { Format = "SubRip", Native = SubRipParser.Serialize(blocks) };
+ }
+
+ if (occurrences == 0)
+ {
+ return new PluginResponse { Status = PluginStatus.Ok, Message = "Remove Unicode characters: nothing to change." };
+ }
+
+ return new PluginResponse
+ {
+ Status = PluginStatus.Ok,
+ Message = $"Replaced {occurrences} Unicode character occurrence(s) across {changedLines} line(s).",
+ UndoDescription = "Remove Unicode characters",
+ Subtitle = subtitle,
+ };
+ }
+
+ ///
+ /// The settings JSON. null keeps the batch keys from
+ /// (a normal run must not touch the batch configuration).
+ ///
+ public static JsonElement BuildSettings(
+ IDictionary replacements,
+ JsonElement? previous,
+ IReadOnlyDictionary? batchList = null,
+ bool removeOthers = false)
+ {
+ var root = new JsonObject
+ {
+ [ReplacementsKey] = ToJson(replacements),
+ };
+
+ if (batchList != null)
+ {
+ root[BatchReplacementsKey] = ToJson(batchList.ToDictionary(kvp => ToHex(kvp.Key), kvp => kvp.Value));
+ root[BatchRemoveOthersKey] = removeOthers;
+ }
+ else if (previous is { ValueKind: JsonValueKind.Object } p)
+ {
+ foreach (var key in new[] { BatchReplacementsKey, BatchRemoveOthersKey })
+ {
+ if (p.TryGetProperty(key, out var value))
+ {
+ root[key] = JsonNode.Parse(value.GetRawText());
+ }
+ }
+ }
+
+ using var doc = JsonDocument.Parse(root.ToJsonString());
+ return doc.RootElement.Clone();
+ }
+
+ private static JsonObject ToJson(IEnumerable> map)
+ {
+ var obj = new JsonObject();
+ foreach (var (key, value) in map.OrderBy(kvp => kvp.Key, StringComparer.OrdinalIgnoreCase))
+ {
+ obj[key] = value;
+ }
+
+ return obj;
+ }
+}
diff --git a/se5/RemoveUnicodeCharacters/MainWindow.axaml b/se5/RemoveUnicodeCharacters/MainWindow.axaml
index 08bea557..6e114a8b 100644
--- a/se5/RemoveUnicodeCharacters/MainWindow.axaml
+++ b/se5/RemoveUnicodeCharacters/MainWindow.axaml
@@ -55,6 +55,31 @@
Opacity="0.7"
TextWrapping="Wrap" />
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
@@ -104,6 +129,7 @@
VerticalAlignment="Center" />
Built-in default replacements (used when the user has no persisted setting for a character).
- private static readonly Dictionary DefaultReplacements = new()
- {
- ['♪'] = "#",
- ['♫'] = "#",
- };
-
private readonly PluginRequest _request;
+ private readonly bool _isConfigure;
private readonly List _blocks;
private readonly Dictionary _persistedReplacements;
private readonly ObservableCollection _rows = new();
@@ -36,21 +30,32 @@ public partial class MainWindow : Window
private ListBox _charsList = null!;
private Button _applyButton = null!;
private Button _googleButton = null!;
+ private Border _configurePanel = null!;
+ private TextBox _addCharsBox = null!;
+ private CheckBox _removeOthersCheckBox = null!;
public MainWindow() : this(new PluginRequest()) { }
public MainWindow(PluginRequest request)
{
_request = request;
+ _isConfigure = request.RequestType == PluginRequestType.Configure;
InitializeComponent();
_blocks = SubRipParser.Parse(request.Subtitle.SubRip);
- _persistedReplacements = LoadReplacementsFromSettings(request.Settings);
+ _persistedReplacements = BatchMode.LoadReplacements(request.Settings);
- BuildRows();
_charsList.ItemsSource = _rows;
_charsList.SelectionChanged += (_, _) => UpdateGoogleButton();
+ if (_isConfigure)
+ {
+ InitConfigure();
+ return;
+ }
+
+ BuildRows();
+
var scope = request.SelectedIndices.Count > 0
? $"the {request.SelectedIndices.Count} selected line(s)"
: "all lines";
@@ -70,6 +75,72 @@ private void InitializeComponent()
_charsList = this.FindControl("CharsList")!;
_applyButton = this.FindControl