From fb3b4ad37fea12d00042bc1c806ea38efe5cc66c Mon Sep 17 00:00:00 2001 From: thevoid0013 Date: Sat, 19 Sep 2026 21:11:12 +0545 Subject: [PATCH 1/2] Add .idea to gitignore and update CountryController --- .gitignore | 1 + .idea/.idea.DataImport/.idea/.gitignore | 15 ----------- .idea/.idea.DataImport/.idea/encodings.xml | 4 --- .idea/.idea.DataImport/.idea/indexLayout.xml | 10 -------- .idea/.idea.DataImport/.idea/vcs.xml | 7 ------ .../Controllers/CountryController.cs | 25 ++++++++++--------- 6 files changed, 14 insertions(+), 48 deletions(-) delete mode 100644 .idea/.idea.DataImport/.idea/.gitignore delete mode 100644 .idea/.idea.DataImport/.idea/encodings.xml delete mode 100644 .idea/.idea.DataImport/.idea/indexLayout.xml delete mode 100644 .idea/.idea.DataImport/.idea/vcs.xml diff --git a/.gitignore b/.gitignore index d745ade..bb552ef 100644 --- a/.gitignore +++ b/.gitignore @@ -369,3 +369,4 @@ FodyWeavers.xsd .idea/ Imports/ +.idea/ diff --git a/.idea/.idea.DataImport/.idea/.gitignore b/.idea/.idea.DataImport/.idea/.gitignore deleted file mode 100644 index 3697fcf..0000000 --- a/.idea/.idea.DataImport/.idea/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Rider ignored files -/modules.xml -/contentModel.xml -/projectSettingsUpdater.xml -/.idea.DataImport.iml -# Ignored default folder with query files -/queries/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/.idea.DataImport/.idea/encodings.xml b/.idea/.idea.DataImport/.idea/encodings.xml deleted file mode 100644 index df87cf9..0000000 --- a/.idea/.idea.DataImport/.idea/encodings.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/.idea.DataImport/.idea/indexLayout.xml b/.idea/.idea.DataImport/.idea/indexLayout.xml deleted file mode 100644 index 7c38c58..0000000 --- a/.idea/.idea.DataImport/.idea/indexLayout.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - ../../Backend - - - - - \ No newline at end of file diff --git a/.idea/.idea.DataImport/.idea/vcs.xml b/.idea/.idea.DataImport/.idea/vcs.xml deleted file mode 100644 index 254e5f3..0000000 --- a/.idea/.idea.DataImport/.idea/vcs.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/DataImport.API/Controllers/CountryController.cs b/DataImport.API/Controllers/CountryController.cs index 035444a..20a2a65 100644 --- a/DataImport.API/Controllers/CountryController.cs +++ b/DataImport.API/Controllers/CountryController.cs @@ -1,5 +1,6 @@ using DataImport.API.Controllers.BaseController; using DataImport.Commands.Queries; +using DataImport.Data.Enums; // <-- add this namespace DataImport.API.Controllers; @@ -7,6 +8,7 @@ namespace DataImport.API.Controllers; public class CountryController : ApiControllerBasev1 { private readonly IMediator _mediator; + public CountryController(IMediator mediator) { _mediator = mediator; @@ -15,30 +17,29 @@ public CountryController(IMediator mediator) [HttpPost] [Route("get-count-by-country")] public async Task GetCountryCount( - [FromQuery] GetCountryCountQuery request, + [FromQuery] Country country, CancellationToken ct) { - if (string.IsNullOrWhiteSpace(request?.country.ToString())) - return BadRequest("Country name is required"); - - var result = await _mediator.Send( - new GetCountryCountQuery(request.country), - ct); + if (!Enum.IsDefined(typeof(Country), country)) + return BadRequest("Valid country is required"); + var result = await _mediator.Send(new GetCountryCountQuery(country), ct); return Ok(result); } [HttpPost] [Route("get-sanctions-by-country")] public async Task GetCountrySanctionsPaged( - [FromQuery] GetCountrySanctionsPagedQuery request, - CancellationToken ct) + [FromQuery] Country country, + [FromQuery] int pageSize = 20, + [FromQuery] int page = 1, + CancellationToken ct = default) { - if (string.IsNullOrWhiteSpace(request?.country.ToString())) - return BadRequest("Country name is required"); + if (!Enum.IsDefined(typeof(Country), country)) + return BadRequest("Valid country is required"); var result = await _mediator.Send( - new GetCountrySanctionsPagedQuery(request.country), + new GetCountrySanctionsPagedQuery(country, pageSize, page), ct); return Ok(result); From 9cc1f9d10f1fb390bec134b256ba91829e5e1b04 Mon Sep 17 00:00:00 2001 From: thevoid0013 Date: Sun, 20 Sep 2026 08:13:16 +0545 Subject: [PATCH 2/2] Delisted features added to the code --- ...014844_AddedFieldsForDelisting.Designer.cs | 135 +++++++++++++++ .../20260920014844_AddedFieldsForDelisting.cs | 82 +++++++++ .../SanctionsDbContextModelSnapshot.cs | 18 ++ DataImport.Data/Models/DataImportLog.cs | 6 +- DataImport.Data/Models/SanctionDetail.cs | 9 + .../Commands/ParseSdnXmlCommandHandler.cs | 42 +++-- .../SaveSanctionDetailsCommandHandler.cs | 161 ++++++++++++++---- 7 files changed, 403 insertions(+), 50 deletions(-) create mode 100644 DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.Designer.cs create mode 100644 DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.cs diff --git a/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.Designer.cs b/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.Designer.cs new file mode 100644 index 0000000..751f541 --- /dev/null +++ b/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.Designer.cs @@ -0,0 +1,135 @@ +// +using System; +using DataImport.Data.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DataImport.Migrations +{ + [DbContext(typeof(SanctionsDbContext))] + [Migration("20260920014844_AddedFieldsForDelisting")] + partial class AddedFieldsForDelisting + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.10") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("DataImport.Data.Models.DataImportLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ErrorMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("Inserted") + .HasColumnType("int"); + + b.Property("Parsed") + .HasColumnType("int"); + + b.Property("RanAtUtc") + .HasColumnType("datetime2"); + + b.Property("Removed") + .HasColumnType("int"); + + b.Property("Succeeded") + .HasColumnType("bit"); + + b.Property("Unchanged") + .HasColumnType("int"); + + b.Property("Updated") + .HasColumnType("int"); + + b.Property("WasDownloaded") + .HasColumnType("bit"); + + b.HasKey("Id"); + + b.HasIndex("RanAtUtc"); + + b.HasIndex("Succeeded", "RanAtUtc"); + + b.ToTable("DataImportLogs"); + }); + + modelBuilder.Entity("DataImport.Data.Models.SanctionDetail", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentHash") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Country") + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("FirstName") + .HasMaxLength(350) + .HasColumnType("nvarchar(350)"); + + b.Property("ImportedAtUtc") + .HasColumnType("datetime2"); + + b.Property("IsActive") + .HasColumnType("bit"); + + b.Property("LastName") + .IsRequired() + .HasMaxLength(350) + .HasColumnType("nvarchar(350)"); + + b.Property("LastSeenAtUtc") + .HasColumnType("datetime2"); + + b.Property("RecordUniqueId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("RemovedAtUtc") + .HasColumnType("datetime2"); + + b.Property("SdnType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("XmlRecord") + .IsRequired() + .HasColumnType("xml"); + + b.HasKey("Id"); + + b.HasIndex("RecordUniqueId") + .IsUnique(); + + b.HasIndex("SdnType", "LastName"); + + b.HasIndex("IsActive", "SdnType", "LastName"); + + b.ToTable("SanctionDetails"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.cs b/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.cs new file mode 100644 index 0000000..0f155b5 --- /dev/null +++ b/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.cs @@ -0,0 +1,82 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DataImport.Migrations +{ + /// + public partial class AddedFieldsForDelisting : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "ContentHash", + table: "SanctionDetails", + type: "nvarchar(64)", + maxLength: 64, + nullable: true); + + migrationBuilder.AddColumn( + name: "IsActive", + table: "SanctionDetails", + type: "bit", + nullable: false, + defaultValue: false); + + migrationBuilder.AddColumn( + name: "LastSeenAtUtc", + table: "SanctionDetails", + type: "datetime2", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "RemovedAtUtc", + table: "SanctionDetails", + type: "datetime2", + nullable: true); + + migrationBuilder.AddColumn( + name: "Removed", + table: "DataImportLogs", + type: "int", + nullable: false, + defaultValue: 0); + + migrationBuilder.CreateIndex( + name: "IX_SanctionDetails_IsActive_SdnType_LastName", + table: "SanctionDetails", + columns: new[] { "IsActive", "SdnType", "LastName" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_SanctionDetails_IsActive_SdnType_LastName", + table: "SanctionDetails"); + + migrationBuilder.DropColumn( + name: "ContentHash", + table: "SanctionDetails"); + + migrationBuilder.DropColumn( + name: "IsActive", + table: "SanctionDetails"); + + migrationBuilder.DropColumn( + name: "LastSeenAtUtc", + table: "SanctionDetails"); + + migrationBuilder.DropColumn( + name: "RemovedAtUtc", + table: "SanctionDetails"); + + migrationBuilder.DropColumn( + name: "Removed", + table: "DataImportLogs"); + } + } +} diff --git a/DataImport.Data/Migrations/SanctionsDbContextModelSnapshot.cs b/DataImport.Data/Migrations/SanctionsDbContextModelSnapshot.cs index 41cf0e5..329e389 100644 --- a/DataImport.Data/Migrations/SanctionsDbContextModelSnapshot.cs +++ b/DataImport.Data/Migrations/SanctionsDbContextModelSnapshot.cs @@ -42,6 +42,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("RanAtUtc") .HasColumnType("datetime2"); + b.Property("Removed") + .HasColumnType("int"); + b.Property("Succeeded") .HasColumnType("bit"); @@ -69,6 +72,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("uniqueidentifier"); + b.Property("ContentHash") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + b.Property("Country") .HasMaxLength(100) .HasColumnType("nvarchar(100)"); @@ -80,16 +87,25 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("ImportedAtUtc") .HasColumnType("datetime2"); + b.Property("IsActive") + .HasColumnType("bit"); + b.Property("LastName") .IsRequired() .HasMaxLength(350) .HasColumnType("nvarchar(350)"); + b.Property("LastSeenAtUtc") + .HasColumnType("datetime2"); + b.Property("RecordUniqueId") .IsRequired() .HasMaxLength(100) .HasColumnType("nvarchar(100)"); + b.Property("RemovedAtUtc") + .HasColumnType("datetime2"); + b.Property("SdnType") .IsRequired() .HasMaxLength(20) @@ -106,6 +122,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("SdnType", "LastName"); + b.HasIndex("IsActive", "SdnType", "LastName"); + b.ToTable("SanctionDetails"); }); #pragma warning restore 612, 618 diff --git a/DataImport.Data/Models/DataImportLog.cs b/DataImport.Data/Models/DataImportLog.cs index 6219c6a..91ec9b3 100644 --- a/DataImport.Data/Models/DataImportLog.cs +++ b/DataImport.Data/Models/DataImportLog.cs @@ -17,9 +17,9 @@ public class DataImportLog public int Inserted { get; set; } public int Updated { get; set; } public int Unchanged { get; set; } - public bool WasDownloaded { get; set; } + public int Removed { get; set; } + public bool WasDownloaded { get; set; } public bool Succeeded { get; set; } - public string? ErrorMessage { get; set; } - + public string? ErrorMessage { get; set; } } } diff --git a/DataImport.Data/Models/SanctionDetail.cs b/DataImport.Data/Models/SanctionDetail.cs index e327f0e..52b9a6a 100644 --- a/DataImport.Data/Models/SanctionDetail.cs +++ b/DataImport.Data/Models/SanctionDetail.cs @@ -9,6 +9,7 @@ namespace DataImport.Data.Models { [Index(nameof(RecordUniqueId), IsUnique = true)] [Index(nameof(SdnType), nameof(LastName))] + [Index(nameof(IsActive), nameof(SdnType), nameof(LastName))] [Table("SanctionDetails")] public class SanctionDetail { @@ -36,6 +37,14 @@ public class SanctionDetail [Required] [StringLength(20)] public string SdnType { get; set; } = null!; + public bool IsActive { get; set; } = true; + + public DateTime? RemovedAtUtc { get; set; } + + public DateTime LastSeenAtUtc { get; set; } = DateTime.UtcNow; + + [StringLength(64)] + public string? ContentHash { get; set; } } } diff --git a/DataImport/Commands/ParseSdnXmlCommandHandler.cs b/DataImport/Commands/ParseSdnXmlCommandHandler.cs index 9385324..55e08b0 100644 --- a/DataImport/Commands/ParseSdnXmlCommandHandler.cs +++ b/DataImport/Commands/ParseSdnXmlCommandHandler.cs @@ -8,6 +8,8 @@ namespace DataImport.Commands { /// /// Parses raw SDN.XML into individual SanctionDetail records — one per <sdnEntry>. + /// Fails loudly on malformed input: a silently skipped entry would later look + /// like a delisting to the removal sweep. /// public record ParseSdnXmlCommand(Stream RawXml) : IRequest>; @@ -35,45 +37,59 @@ public Task> Handle(ParseSdnXmlCommand request, Cancellatio var doc = XDocument.Load(request.RawXml); var results = new List(); + var seenUids = new HashSet(StringComparer.Ordinal); foreach (var entry in doc.Descendants(Ns + "sdnEntry")) { + cancellationToken.ThrowIfCancellationRequested(); + var uid = entry.Element(Ns + "uid")?.Value; if (string.IsNullOrWhiteSpace(uid)) { - // Shouldn't happen in practice, but skip rather than blow up the whole import. - continue; + throw new InvalidOperationException( + "Encountered an sdnEntry with no uid; refusing to import a partial list."); } - var country = entry.Element(Ns + "addressList")? - .Element(Ns + "address")? - .Element(Ns + "country")? - .Value; + if (!seenUids.Add(uid)) + { + throw new InvalidOperationException( + $"Duplicate sdnEntry uid {uid} in file; refusing to import."); + } var sdnType = entry.Element(Ns + "sdnType")?.Value; var lastName = entry.Element(Ns + "lastName")?.Value; - var firstName = entry.Element(Ns + "firstName")?.Value; + var firstName = entry.Element(Ns + "firstName")?.Value; if (string.IsNullOrWhiteSpace(lastName) || string.IsNullOrWhiteSpace(sdnType)) { - // Both are required by OFAC's schema. If either is missing, this entry - // is malformed — skip it rather than let a NOT NULL violation blow up - // the whole batch save later. - _logger.LogWarning("Skipping sdnEntry {Uid} — missing required lastName or sdnType.", uid); - continue; + // Both are required by OFAC's schema. Skipping would make this entry + // look delisted to the removal sweep, so fail the whole import instead. + throw new InvalidOperationException( + $"sdnEntry {uid} is missing required lastName or sdnType; refusing to import a partial list."); } + var country = entry.Element(Ns + "addressList")? + .Element(Ns + "address")? + .Element(Ns + "country")? + .Value; + results.Add(new SanctionDetail { RecordUniqueId = uid, SdnType = sdnType, LastName = lastName, - FirstName = firstName, + FirstName = firstName, Country = country, XmlRecord = entry.ToString(SaveOptions.DisableFormatting) }); } + if (results.Count == 0) + { + throw new InvalidOperationException( + "Parsed 0 sdnEntry records. Check the XML namespace (Ns) against the root xmlns of a fresh download."); + } + _logger.LogInformation("Parsed {Count} SDN entries.", results.Count); return Task.FromResult(results); diff --git a/DataImport/Commands/SaveSanctionDetailsCommandHandler.cs b/DataImport/Commands/SaveSanctionDetailsCommandHandler.cs index a72ee68..2d8bf50 100644 --- a/DataImport/Commands/SaveSanctionDetailsCommandHandler.cs +++ b/DataImport/Commands/SaveSanctionDetailsCommandHandler.cs @@ -3,36 +3,51 @@ using MediatR; using MethodTimer; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; namespace DataImport.Commands { /// /// Upserts parsed SanctionDetail records: inserts new ones, updates changed - /// ones, leaves unchanged ones alone. Processes in batches so a run of - /// ~18k records doesn't sit in one giant transaction/timeout. + /// ones, leaves unchanged ones alone, reactivates relisted ones, and + /// soft-deletes (IsActive = false) records that are no longer in the file. + /// Processes in batches so a run of ~18k records doesn't sit in one giant + /// transaction/timeout. /// public record SaveSanctionDetailsCommand(List Records) : IRequest; - public record SaveSanctionDetailsResult(int Inserted, int Updated, int Unchanged); + public record SaveSanctionDetailsResult(int Inserted, int Updated, int Unchanged, int Removed); public class SaveSanctionDetailsCommandHandler : IRequestHandler { private const int BatchSize = 500; + private const int DefaultMaxRemovalsPerRun = 500; + private const int MaxUidsToLog = 50; private readonly SanctionsDbContext _db; private readonly ILogger _logger; + private readonly int _maxRemovalsPerRun; - public SaveSanctionDetailsCommandHandler(SanctionsDbContext db, ILogger logger) + public SaveSanctionDetailsCommandHandler( + SanctionsDbContext db, + ILogger logger, + IConfiguration configuration) { _db = db; _logger = logger; + _maxRemovalsPerRun = configuration.GetValue("Import:MaxRemovalsPerRun", DefaultMaxRemovalsPerRun); } [Time] public async Task Handle(SaveSanctionDetailsCommand request, CancellationToken cancellationToken) { + if (request.Records.Count == 0) + { + throw new InvalidOperationException("SaveSanctionDetailsCommand received 0 records; refusing to proceed."); + } + _logger.LogInformation("Loading existing SDN records for comparison..."); var existingByUid = await _db.SanctionDetails @@ -41,55 +56,133 @@ public async Task Handle(SaveSanctionDetailsCommand r _logger.LogInformation("Loaded {Count} existing records.", existingByUid.Count); + // ---- Work out removals BEFORE writing anything ---- + var fileUids = request.Records + .Select(r => r.RecordUniqueId) + .ToHashSet(StringComparer.Ordinal); + + var toRemove = existingByUid.Values + .Where(d => d.IsActive && !fileUids.Contains(d.RecordUniqueId)) + .Select(d => d.RecordUniqueId) + .ToList(); + + if (toRemove.Count > _maxRemovalsPerRun) + { + // A truncated/bad file looks identical to mass delisting. Legitimate mass + // removals (e.g. a program terminated) do happen: verify against OFAC's + // Recent Actions, then raise Import:MaxRemovalsPerRun and rerun. + throw new InvalidOperationException( + $"{toRemove.Count} records would be delisted, exceeding the limit of {_maxRemovalsPerRun}. " + + "Aborting before any changes were made. Verify against OFAC Recent Actions."); + } + + // ---- Upsert ---- _db.ChangeTracker.AutoDetectChangesEnabled = false; int inserted = 0, updated = 0, unchanged = 0; var processed = 0; + var relisted = new List(); - foreach (var batch in request.Records.Chunk(BatchSize)) + try { - foreach (var record in batch) + foreach (var batch in request.Records.Chunk(BatchSize)) { - if (existingByUid.TryGetValue(record.RecordUniqueId, out var existingRecord)) + foreach (var record in batch) { - if (existingRecord.XmlRecord != record.XmlRecord - || existingRecord.Country != record.Country - || existingRecord.LastName != record.LastName - || existingRecord.FirstName != record.FirstName - || existingRecord.SdnType != record.SdnType) + if (existingByUid.TryGetValue(record.RecordUniqueId, out var existingRecord)) { - existingRecord.XmlRecord = record.XmlRecord; - existingRecord.Country = record.Country; - existingRecord.ImportedAtUtc = DateTime.UtcNow; - existingRecord.LastName = record.LastName; - existingRecord.FirstName = record.FirstName; - existingRecord.SdnType = record.SdnType; - - _db.Attach(existingRecord); - _db.Entry(existingRecord).State = EntityState.Modified; - - updated++; + var isRelisted = !existingRecord.IsActive; + + if (isRelisted + || existingRecord.XmlRecord != record.XmlRecord + || existingRecord.Country != record.Country + || existingRecord.LastName != record.LastName + || existingRecord.FirstName != record.FirstName + || existingRecord.SdnType != record.SdnType) + { + // Attach BEFORE mutating: EF snapshots the current (old) values on + // attach, so DetectChanges below can flag only the columns that + // really changed. A pure relist then updates just IsActive / + // RemovedAtUtc / ImportedAtUtc and never rewrites XmlRecord or the + // full-text indexed name columns. + _db.Attach(existingRecord); + + if (isRelisted) + { + relisted.Add(record.RecordUniqueId); + } + + existingRecord.XmlRecord = record.XmlRecord; + existingRecord.Country = record.Country; + existingRecord.ImportedAtUtc = DateTime.UtcNow; + existingRecord.LastName = record.LastName; + existingRecord.FirstName = record.FirstName; + existingRecord.SdnType = record.SdnType; + existingRecord.IsActive = true; + existingRecord.RemovedAtUtc = null; + + updated++; + } + else + { + unchanged++; + } } else { - unchanged++; + _db.SanctionDetails.Add(record); + inserted++; } } - else - { - _db.SanctionDetails.Add(record); - inserted++; - } + + // AutoDetectChanges is off, so run it once per batch. Without this call EF + // would not see the property changes above and nothing would be saved. + _db.ChangeTracker.DetectChanges(); + await _db.SaveChangesAsync(cancellationToken); + _db.ChangeTracker.Clear(); + + processed += batch.Length; + _logger.LogInformation("Saved {Processed}/{Total} records...", processed, request.Records.Count); } + } + finally + { + // The context is shared with the orchestrator (DataImportLog save), so restore this. + _db.ChangeTracker.AutoDetectChangesEnabled = true; + } + + if (relisted.Count > 0) + { + _logger.LogWarning( + "{Count} previously delisted SDN records were relisted (showing up to {Max}): {Uids}", + relisted.Count, + MaxUidsToLog, + string.Join(", ", relisted.Take(MaxUidsToLog))); + } + + // ---- Sweep: only reached if the whole upsert succeeded ---- + if (toRemove.Count > 0) + { + var removedAtUtc = DateTime.UtcNow; - await _db.SaveChangesAsync(cancellationToken); - _db.ChangeTracker.Clear(); + foreach (var chunk in toRemove.Chunk(BatchSize)) + { + await _db.SanctionDetails + .Where(d => d.IsActive && chunk.Contains(d.RecordUniqueId)) + .ExecuteUpdateAsync(s => s + .SetProperty(d => d.IsActive, false) + .SetProperty(d => d.RemovedAtUtc, removedAtUtc), + cancellationToken); + } - processed += batch.Length; - _logger.LogInformation("Saved {Processed}/{Total} records...", processed, request.Records.Count); + // Bounded by the threshold above, so safe to log in full for audit. + _logger.LogWarning( + "Delisted {Count} SDN records no longer present in file: {Uids}", + toRemove.Count, + string.Join(", ", toRemove)); } - return new SaveSanctionDetailsResult(inserted, updated, unchanged); + return new SaveSanctionDetailsResult(inserted, updated, unchanged, toRemove.Count); } } } \ No newline at end of file