From 5eceaf2016f2990c9bcd204b5df2b305934325ad Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Thu, 1 Oct 2026 08:18:35 +1300 Subject: [PATCH] Remove deprecated parallel corpus analysis --- src/Serval/src/Serval.Client/Client.g.cs | 25 ---------- .../IServalConfiguratorExtensions.cs | 6 +++ .../Dtos/ParallelCorpusAnalysisDto.cs | 11 ----- .../Dtos/TranslationBuildDto.cs | 3 -- .../src/Serval.Translation/Models/Build.cs | 1 - .../Models/ParallelCorpusAnalysis.cs | 7 --- .../Serval.Translation/Services/DtoMapper.cs | 10 ---- .../Services/PlatformService.cs | 13 +----- .../TranslationEngineTests.cs | 46 ------------------- .../Usings.cs | 1 - .../Services/PlatformServiceTests.cs | 19 ++------ 11 files changed, 10 insertions(+), 132 deletions(-) delete mode 100644 src/Serval/src/Serval.Translation/Dtos/ParallelCorpusAnalysisDto.cs delete mode 100644 src/Serval/src/Serval.Translation/Models/ParallelCorpusAnalysis.cs diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 432dd5c3..55cd748f 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -11522,10 +11522,6 @@ public partial class TranslationBuild [Newtonsoft.Json.JsonProperty("phases", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public System.Collections.Generic.IList? Phases { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("analysis", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - [System.Obsolete] - public System.Collections.Generic.IList? Analysis { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetQuoteConvention", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? TargetQuoteConvention { get; set; } = default!; @@ -11749,27 +11745,6 @@ public enum PhaseStage } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class ParallelCorpusAnalysis - { - - [Newtonsoft.Json.JsonProperty("parallelCorpusRef", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string ParallelCorpusRef { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("sourceQuoteConvention", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - [System.Obsolete] - public string? SourceQuoteConvention { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("targetQuoteConvention", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string TargetQuoteConvention { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("canDenormalizeQuotes", Required = Newtonsoft.Json.Required.Always)] - public bool CanDenormalizeQuotes { get; set; } = default!; - - } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TranslationEngine { diff --git a/src/Serval/src/Serval.Translation/Configuration/IServalConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IServalConfiguratorExtensions.cs index 7c4680d2..63dd3ed6 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IServalConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IServalConfiguratorExtensions.cs @@ -88,6 +88,12 @@ public static IServalConfigurator AddTranslationDataAccess(this IServalConfigura c.Indexes.CreateOrUpdateAsync( new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.DateFinished)) ), + // Remove the old parallel corpus analysis + c => + c.UpdateManyAsync( + Builders.Filter.Exists("analysis"), + Builders.Update.Unset("analysis") + ), ] ); configurator.DataAccess.AddRepository( diff --git a/src/Serval/src/Serval.Translation/Dtos/ParallelCorpusAnalysisDto.cs b/src/Serval/src/Serval.Translation/Dtos/ParallelCorpusAnalysisDto.cs deleted file mode 100644 index 8faf0e94..00000000 --- a/src/Serval/src/Serval.Translation/Dtos/ParallelCorpusAnalysisDto.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.Translation.Dtos; - -public record ParallelCorpusAnalysisDto -{ - public required string ParallelCorpusRef { get; init; } - - [Obsolete] - public string? SourceQuoteConvention { get; init; } - public required string TargetQuoteConvention { get; init; } - public bool CanDenormalizeQuotes { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Dtos/TranslationBuildDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationBuildDto.cs index 60976aab..cbf96e38 100644 --- a/src/Serval/src/Serval.Translation/Dtos/TranslationBuildDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationBuildDto.cs @@ -37,9 +37,6 @@ public record TranslationBuildDto public string? DeploymentVersion { get; init; } public required ExecutionDataDto ExecutionData { get; init; } public IReadOnlyList? Phases { get; init; } - - [Obsolete] - public IReadOnlyList? Analysis { get; init; } public string? TargetQuoteConvention { get; init; } public bool CanDenormalizeQuotes { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Models/Build.cs b/src/Serval/src/Serval.Translation/Models/Build.cs index 1433d3b3..c59dd55b 100644 --- a/src/Serval/src/Serval.Translation/Models/Build.cs +++ b/src/Serval/src/Serval.Translation/Models/Build.cs @@ -23,6 +23,5 @@ public record Build : IOwnedEntity public DateTime? DateStarted { get; set; } public DateTime? DateCompleted { get; set; } public IReadOnlyList? Phases { get; init; } - public IReadOnlyList? Analysis { get; init; } public string? TargetQuoteConvention { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Models/ParallelCorpusAnalysis.cs b/src/Serval/src/Serval.Translation/Models/ParallelCorpusAnalysis.cs deleted file mode 100644 index 3c233dcb..00000000 --- a/src/Serval/src/Serval.Translation/Models/ParallelCorpusAnalysis.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Translation.Models; - -public record ParallelCorpusAnalysis -{ - public required string ParallelCorpusRef { get; init; } - public required string TargetQuoteConvention { get; init; } -} diff --git a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs index 3755704a..6d3aea86 100644 --- a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -57,7 +57,6 @@ public TranslationBuildDto Map(Build source) DeploymentVersion = source.DeploymentVersion, ExecutionData = Map(source.ExecutionData), Phases = source.Phases?.Select(Map).ToList(), - Analysis = source.Analysis?.Select(a => Map(a, targetQuoteConvention)).ToList(), TargetQuoteConvention = targetQuoteConvention, CanDenormalizeQuotes = targetQuoteConvention != "", }; @@ -218,15 +217,6 @@ private static PhaseDto Map(Phase source) => Started = source.Started, }; - private static ParallelCorpusAnalysisDto Map(ParallelCorpusAnalysis source, string targetQuoteConvention) => - new() - { - ParallelCorpusRef = source.ParallelCorpusRef, - TargetQuoteConvention = targetQuoteConvention, - SourceQuoteConvention = "ignore", - CanDenormalizeQuotes = targetQuoteConvention != "", - }; - private static ExecutionDataDto Map(ExecutionData source) => new() { diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 8df5d1cc..fcd8c450 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -348,20 +348,9 @@ public async Task UpdateTargetQuoteConventionAsync( Engine? engine = await _engines.GetAsync(engineId, cancellationToken); if (engine is null) return; - var analysis = engine - .ParallelCorpora.Select(pc => new ParallelCorpusAnalysis - { - ParallelCorpusRef = pc.Id, - TargetQuoteConvention = quoteConvention, - }) - .ToList(); await _builds.UpdateAsync( b => b.Id == buildId && b.EngineRef == engineId, - u => - { - u.Set(b => b.TargetQuoteConvention, quoteConvention); - u.Set(b => b.Analysis, analysis); - }, + u => u.Set(b => b.TargetQuoteConvention, quoteConvention), cancellationToken: cancellationToken ); } diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index 420db37b..1b13ee51 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -2488,52 +2488,6 @@ await corporaClient.UpdateAsync( Assert.That(newEngine2.ParallelCorpora[0].TargetCorpora[0].Files.Count, Is.EqualTo(1)); } - [Test] - public async Task MongoMigration_TargetQuoteConvention() - { - await _env.Builds.InsertAsync( - new Build() - { - Id = "111111111111111111111111", - EngineRef = NMT_ENGINE1_ID, - Owner = "client1", - Analysis = - [ - new Translation.Models.ParallelCorpusAnalysis() - { - ParallelCorpusRef = "111111111111111111111112", - TargetQuoteConvention = "", - }, - new Translation.Models.ParallelCorpusAnalysis() - { - ParallelCorpusRef = "111111111111111111111113", - TargetQuoteConvention = "standard_english", - }, - ], - } - ); - - Build? unmigratedBuild = await _env.Builds.GetAsync(b => b.Id == "111111111111111111111111"); - Assert.That(unmigratedBuild, Is.Not.Null); - Assert.Multiple(() => - { - Assert.That(unmigratedBuild.Analysis, Has.Count.EqualTo(2)); - Assert.That(unmigratedBuild.TargetQuoteConvention, Is.Null); - }); - - await MongoMigrations.MigrateTargetQuoteConvention( - _env.MongoClient.GetDatabase("serval_test").GetCollection("translation.builds") - ); - - Build? migratedBuild = await _env.Builds.GetAsync(b => b.Id == "111111111111111111111111"); - Assert.That(migratedBuild, Is.Not.Null); - Assert.Multiple(() => - { - Assert.That(migratedBuild.Analysis, Has.Count.EqualTo(2)); - Assert.That(migratedBuild.TargetQuoteConvention, Is.EqualTo("standard_english")); - }); - } - [Test] public async Task GetAllPretranslationConfidencesAsync_Exists() { diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs index 7007ba61..eda68682 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/Usings.cs @@ -16,6 +16,5 @@ global using Serval.Shared.Contracts; global using Serval.Shared.Controllers; global using Serval.Shared.Services; -global using Serval.Translation.Configuration; global using SIL.DataAccess; global using SIL.ObjectModel; diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index ad10f21f..71ccb597 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -331,12 +331,7 @@ public async Task UpdateTargetQuoteConventionAsync() build = await env.Builds.GetAsync(c => c.Id == build.Id); - Assert.Multiple(() => - { - Assert.That(build?.TargetQuoteConvention, Is.EqualTo(expected)); - Assert.That(build?.Analysis, Has.Count.EqualTo(1)); - }); - Assert.That(build?.Analysis?[0].TargetQuoteConvention, Is.EqualTo(expected)); + Assert.That(build?.TargetQuoteConvention, Is.EqualTo(expected)); } [Test] @@ -356,11 +351,7 @@ public async Task UpdateTargetQuoteConventionAsync_NoEngine() build = await env.Builds.GetAsync(c => c.Id == build.Id); - Assert.Multiple(() => - { - Assert.That(build?.TargetQuoteConvention, Is.Null); - Assert.That(build?.Analysis, Is.Null); - }); + Assert.That(build?.TargetQuoteConvention, Is.Null); } [Test] @@ -391,11 +382,7 @@ public async Task UpdateTargetQuoteConventionAsync_NoParallelCorpora() build = await env.Builds.GetAsync(c => c.Id == build.Id); - Assert.Multiple(() => - { - Assert.That(build?.TargetQuoteConvention, Is.EqualTo("")); - Assert.That(build?.Analysis, Has.Count.EqualTo(0)); - }); + Assert.That(build?.TargetQuoteConvention, Is.Empty); } [Test]