diff --git a/.gitignore b/.gitignore
index d745ade..bb552ef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -369,3 +369,4 @@ FodyWeavers.xsd
.idea/
Imports/
+.idea/
diff --git a/.idea/.idea.DataImport/.idea/.gitignore b/.idea/.idea.DataImport/.idea/.gitignore
deleted file mode 100644
index 3697fcf..0000000
--- a/.idea/.idea.DataImport/.idea/.gitignore
+++ /dev/null
@@ -1,15 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Rider ignored files
-/modules.xml
-/contentModel.xml
-/projectSettingsUpdater.xml
-/.idea.DataImport.iml
-# Ignored default folder with query files
-/queries/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml
diff --git a/.idea/.idea.DataImport/.idea/encodings.xml b/.idea/.idea.DataImport/.idea/encodings.xml
deleted file mode 100644
index df87cf9..0000000
--- a/.idea/.idea.DataImport/.idea/encodings.xml
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-
-
\ No newline at end of file
diff --git a/.idea/.idea.DataImport/.idea/indexLayout.xml b/.idea/.idea.DataImport/.idea/indexLayout.xml
deleted file mode 100644
index 7c38c58..0000000
--- a/.idea/.idea.DataImport/.idea/indexLayout.xml
+++ /dev/null
@@ -1,10 +0,0 @@
-
-
-
-
- ../../Backend
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/.idea.DataImport/.idea/vcs.xml b/.idea/.idea.DataImport/.idea/vcs.xml
deleted file mode 100644
index 254e5f3..0000000
--- a/.idea/.idea.DataImport/.idea/vcs.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/DataImport.API/Controllers/CountryController.cs b/DataImport.API/Controllers/CountryController.cs
index 035444a..20a2a65 100644
--- a/DataImport.API/Controllers/CountryController.cs
+++ b/DataImport.API/Controllers/CountryController.cs
@@ -1,5 +1,6 @@
using DataImport.API.Controllers.BaseController;
using DataImport.Commands.Queries;
+using DataImport.Data.Enums; // <-- add this
namespace DataImport.API.Controllers;
@@ -7,6 +8,7 @@ namespace DataImport.API.Controllers;
public class CountryController : ApiControllerBasev1
{
private readonly IMediator _mediator;
+
public CountryController(IMediator mediator)
{
_mediator = mediator;
@@ -15,30 +17,29 @@ public CountryController(IMediator mediator)
[HttpPost]
[Route("get-count-by-country")]
public async Task GetCountryCount(
- [FromQuery] GetCountryCountQuery request,
+ [FromQuery] Country country,
CancellationToken ct)
{
- if (string.IsNullOrWhiteSpace(request?.country.ToString()))
- return BadRequest("Country name is required");
-
- var result = await _mediator.Send(
- new GetCountryCountQuery(request.country),
- ct);
+ if (!Enum.IsDefined(typeof(Country), country))
+ return BadRequest("Valid country is required");
+ var result = await _mediator.Send(new GetCountryCountQuery(country), ct);
return Ok(result);
}
[HttpPost]
[Route("get-sanctions-by-country")]
public async Task GetCountrySanctionsPaged(
- [FromQuery] GetCountrySanctionsPagedQuery request,
- CancellationToken ct)
+ [FromQuery] Country country,
+ [FromQuery] int pageSize = 20,
+ [FromQuery] int page = 1,
+ CancellationToken ct = default)
{
- if (string.IsNullOrWhiteSpace(request?.country.ToString()))
- return BadRequest("Country name is required");
+ if (!Enum.IsDefined(typeof(Country), country))
+ return BadRequest("Valid country is required");
var result = await _mediator.Send(
- new GetCountrySanctionsPagedQuery(request.country),
+ new GetCountrySanctionsPagedQuery(country, pageSize, page),
ct);
return Ok(result);
diff --git a/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.Designer.cs b/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.Designer.cs
new file mode 100644
index 0000000..751f541
--- /dev/null
+++ b/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.Designer.cs
@@ -0,0 +1,135 @@
+//
+using System;
+using DataImport.Data.Data;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+
+#nullable disable
+
+namespace DataImport.Migrations
+{
+ [DbContext(typeof(SanctionsDbContext))]
+ [Migration("20260920014844_AddedFieldsForDelisting")]
+ partial class AddedFieldsForDelisting
+ {
+ ///
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("ProductVersion", "10.0.10")
+ .HasAnnotation("Relational:MaxIdentifierLength", 128);
+
+ SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
+
+ modelBuilder.Entity("DataImport.Data.Models.DataImportLog", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("int");
+
+ SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"));
+
+ b.Property("ErrorMessage")
+ .HasColumnType("nvarchar(max)");
+
+ b.Property("Inserted")
+ .HasColumnType("int");
+
+ b.Property("Parsed")
+ .HasColumnType("int");
+
+ b.Property("RanAtUtc")
+ .HasColumnType("datetime2");
+
+ b.Property("Removed")
+ .HasColumnType("int");
+
+ b.Property("Succeeded")
+ .HasColumnType("bit");
+
+ b.Property("Unchanged")
+ .HasColumnType("int");
+
+ b.Property("Updated")
+ .HasColumnType("int");
+
+ b.Property("WasDownloaded")
+ .HasColumnType("bit");
+
+ b.HasKey("Id");
+
+ b.HasIndex("RanAtUtc");
+
+ b.HasIndex("Succeeded", "RanAtUtc");
+
+ b.ToTable("DataImportLogs");
+ });
+
+ modelBuilder.Entity("DataImport.Data.Models.SanctionDetail", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uniqueidentifier");
+
+ b.Property("ContentHash")
+ .HasMaxLength(64)
+ .HasColumnType("nvarchar(64)");
+
+ b.Property("Country")
+ .HasMaxLength(100)
+ .HasColumnType("nvarchar(100)");
+
+ b.Property("FirstName")
+ .HasMaxLength(350)
+ .HasColumnType("nvarchar(350)");
+
+ b.Property("ImportedAtUtc")
+ .HasColumnType("datetime2");
+
+ b.Property("IsActive")
+ .HasColumnType("bit");
+
+ b.Property("LastName")
+ .IsRequired()
+ .HasMaxLength(350)
+ .HasColumnType("nvarchar(350)");
+
+ b.Property("LastSeenAtUtc")
+ .HasColumnType("datetime2");
+
+ b.Property("RecordUniqueId")
+ .IsRequired()
+ .HasMaxLength(100)
+ .HasColumnType("nvarchar(100)");
+
+ b.Property("RemovedAtUtc")
+ .HasColumnType("datetime2");
+
+ b.Property("SdnType")
+ .IsRequired()
+ .HasMaxLength(20)
+ .HasColumnType("nvarchar(20)");
+
+ b.Property("XmlRecord")
+ .IsRequired()
+ .HasColumnType("xml");
+
+ b.HasKey("Id");
+
+ b.HasIndex("RecordUniqueId")
+ .IsUnique();
+
+ b.HasIndex("SdnType", "LastName");
+
+ b.HasIndex("IsActive", "SdnType", "LastName");
+
+ b.ToTable("SanctionDetails");
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
diff --git a/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.cs b/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.cs
new file mode 100644
index 0000000..0f155b5
--- /dev/null
+++ b/DataImport.Data/Migrations/20260920014844_AddedFieldsForDelisting.cs
@@ -0,0 +1,82 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace DataImport.Migrations
+{
+ ///
+ public partial class AddedFieldsForDelisting : Migration
+ {
+ ///
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.AddColumn(
+ name: "ContentHash",
+ table: "SanctionDetails",
+ type: "nvarchar(64)",
+ maxLength: 64,
+ nullable: true);
+
+ migrationBuilder.AddColumn(
+ name: "IsActive",
+ table: "SanctionDetails",
+ type: "bit",
+ nullable: false,
+ defaultValue: false);
+
+ migrationBuilder.AddColumn(
+ name: "LastSeenAtUtc",
+ table: "SanctionDetails",
+ type: "datetime2",
+ nullable: false,
+ defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
+
+ migrationBuilder.AddColumn(
+ name: "RemovedAtUtc",
+ table: "SanctionDetails",
+ type: "datetime2",
+ nullable: true);
+
+ migrationBuilder.AddColumn(
+ name: "Removed",
+ table: "DataImportLogs",
+ type: "int",
+ nullable: false,
+ defaultValue: 0);
+
+ migrationBuilder.CreateIndex(
+ name: "IX_SanctionDetails_IsActive_SdnType_LastName",
+ table: "SanctionDetails",
+ columns: new[] { "IsActive", "SdnType", "LastName" });
+ }
+
+ ///
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropIndex(
+ name: "IX_SanctionDetails_IsActive_SdnType_LastName",
+ table: "SanctionDetails");
+
+ migrationBuilder.DropColumn(
+ name: "ContentHash",
+ table: "SanctionDetails");
+
+ migrationBuilder.DropColumn(
+ name: "IsActive",
+ table: "SanctionDetails");
+
+ migrationBuilder.DropColumn(
+ name: "LastSeenAtUtc",
+ table: "SanctionDetails");
+
+ migrationBuilder.DropColumn(
+ name: "RemovedAtUtc",
+ table: "SanctionDetails");
+
+ migrationBuilder.DropColumn(
+ name: "Removed",
+ table: "DataImportLogs");
+ }
+ }
+}
diff --git a/DataImport.Data/Migrations/SanctionsDbContextModelSnapshot.cs b/DataImport.Data/Migrations/SanctionsDbContextModelSnapshot.cs
index 41cf0e5..329e389 100644
--- a/DataImport.Data/Migrations/SanctionsDbContextModelSnapshot.cs
+++ b/DataImport.Data/Migrations/SanctionsDbContextModelSnapshot.cs
@@ -42,6 +42,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property("RanAtUtc")
.HasColumnType("datetime2");
+ b.Property("Removed")
+ .HasColumnType("int");
+
b.Property("Succeeded")
.HasColumnType("bit");
@@ -69,6 +72,10 @@ protected override void BuildModel(ModelBuilder modelBuilder)
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
+ b.Property("ContentHash")
+ .HasMaxLength(64)
+ .HasColumnType("nvarchar(64)");
+
b.Property("Country")
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
@@ -80,16 +87,25 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property("ImportedAtUtc")
.HasColumnType("datetime2");
+ b.Property("IsActive")
+ .HasColumnType("bit");
+
b.Property("LastName")
.IsRequired()
.HasMaxLength(350)
.HasColumnType("nvarchar(350)");
+ b.Property("LastSeenAtUtc")
+ .HasColumnType("datetime2");
+
b.Property("RecordUniqueId")
.IsRequired()
.HasMaxLength(100)
.HasColumnType("nvarchar(100)");
+ b.Property("RemovedAtUtc")
+ .HasColumnType("datetime2");
+
b.Property("SdnType")
.IsRequired()
.HasMaxLength(20)
@@ -106,6 +122,8 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.HasIndex("SdnType", "LastName");
+ b.HasIndex("IsActive", "SdnType", "LastName");
+
b.ToTable("SanctionDetails");
});
#pragma warning restore 612, 618
diff --git a/DataImport.Data/Models/DataImportLog.cs b/DataImport.Data/Models/DataImportLog.cs
index 6219c6a..91ec9b3 100644
--- a/DataImport.Data/Models/DataImportLog.cs
+++ b/DataImport.Data/Models/DataImportLog.cs
@@ -17,9 +17,9 @@ public class DataImportLog
public int Inserted { get; set; }
public int Updated { get; set; }
public int Unchanged { get; set; }
- public bool WasDownloaded { get; set; }
+ public int Removed { get; set; }
+ public bool WasDownloaded { get; set; }
public bool Succeeded { get; set; }
- public string? ErrorMessage { get; set; }
-
+ public string? ErrorMessage { get; set; }
}
}
diff --git a/DataImport.Data/Models/SanctionDetail.cs b/DataImport.Data/Models/SanctionDetail.cs
index e327f0e..52b9a6a 100644
--- a/DataImport.Data/Models/SanctionDetail.cs
+++ b/DataImport.Data/Models/SanctionDetail.cs
@@ -9,6 +9,7 @@ namespace DataImport.Data.Models
{
[Index(nameof(RecordUniqueId), IsUnique = true)]
[Index(nameof(SdnType), nameof(LastName))]
+ [Index(nameof(IsActive), nameof(SdnType), nameof(LastName))]
[Table("SanctionDetails")]
public class SanctionDetail
{
@@ -36,6 +37,14 @@ public class SanctionDetail
[Required]
[StringLength(20)]
public string SdnType { get; set; } = null!;
+ public bool IsActive { get; set; } = true;
+
+ public DateTime? RemovedAtUtc { get; set; }
+
+ public DateTime LastSeenAtUtc { get; set; } = DateTime.UtcNow;
+
+ [StringLength(64)]
+ public string? ContentHash { get; set; }
}
}
diff --git a/DataImport/Commands/ParseSdnXmlCommandHandler.cs b/DataImport/Commands/ParseSdnXmlCommandHandler.cs
index 9385324..55e08b0 100644
--- a/DataImport/Commands/ParseSdnXmlCommandHandler.cs
+++ b/DataImport/Commands/ParseSdnXmlCommandHandler.cs
@@ -8,6 +8,8 @@ namespace DataImport.Commands
{
///
/// Parses raw SDN.XML into individual SanctionDetail records — one per <sdnEntry>.
+ /// Fails loudly on malformed input: a silently skipped entry would later look
+ /// like a delisting to the removal sweep.
///
public record ParseSdnXmlCommand(Stream RawXml) : IRequest>;
@@ -35,45 +37,59 @@ public Task> Handle(ParseSdnXmlCommand request, Cancellatio
var doc = XDocument.Load(request.RawXml);
var results = new List();
+ var seenUids = new HashSet(StringComparer.Ordinal);
foreach (var entry in doc.Descendants(Ns + "sdnEntry"))
{
+ cancellationToken.ThrowIfCancellationRequested();
+
var uid = entry.Element(Ns + "uid")?.Value;
if (string.IsNullOrWhiteSpace(uid))
{
- // Shouldn't happen in practice, but skip rather than blow up the whole import.
- continue;
+ throw new InvalidOperationException(
+ "Encountered an sdnEntry with no uid; refusing to import a partial list.");
}
- var country = entry.Element(Ns + "addressList")?
- .Element(Ns + "address")?
- .Element(Ns + "country")?
- .Value;
+ if (!seenUids.Add(uid))
+ {
+ throw new InvalidOperationException(
+ $"Duplicate sdnEntry uid {uid} in file; refusing to import.");
+ }
var sdnType = entry.Element(Ns + "sdnType")?.Value;
var lastName = entry.Element(Ns + "lastName")?.Value;
- var firstName = entry.Element(Ns + "firstName")?.Value;
+ var firstName = entry.Element(Ns + "firstName")?.Value;
if (string.IsNullOrWhiteSpace(lastName) || string.IsNullOrWhiteSpace(sdnType))
{
- // Both are required by OFAC's schema. If either is missing, this entry
- // is malformed — skip it rather than let a NOT NULL violation blow up
- // the whole batch save later.
- _logger.LogWarning("Skipping sdnEntry {Uid} — missing required lastName or sdnType.", uid);
- continue;
+ // Both are required by OFAC's schema. Skipping would make this entry
+ // look delisted to the removal sweep, so fail the whole import instead.
+ throw new InvalidOperationException(
+ $"sdnEntry {uid} is missing required lastName or sdnType; refusing to import a partial list.");
}
+ var country = entry.Element(Ns + "addressList")?
+ .Element(Ns + "address")?
+ .Element(Ns + "country")?
+ .Value;
+
results.Add(new SanctionDetail
{
RecordUniqueId = uid,
SdnType = sdnType,
LastName = lastName,
- FirstName = firstName,
+ FirstName = firstName,
Country = country,
XmlRecord = entry.ToString(SaveOptions.DisableFormatting)
});
}
+ if (results.Count == 0)
+ {
+ throw new InvalidOperationException(
+ "Parsed 0 sdnEntry records. Check the XML namespace (Ns) against the root xmlns of a fresh download.");
+ }
+
_logger.LogInformation("Parsed {Count} SDN entries.", results.Count);
return Task.FromResult(results);
diff --git a/DataImport/Commands/SaveSanctionDetailsCommandHandler.cs b/DataImport/Commands/SaveSanctionDetailsCommandHandler.cs
index a72ee68..2d8bf50 100644
--- a/DataImport/Commands/SaveSanctionDetailsCommandHandler.cs
+++ b/DataImport/Commands/SaveSanctionDetailsCommandHandler.cs
@@ -3,36 +3,51 @@
using MediatR;
using MethodTimer;
using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace DataImport.Commands
{
///
/// Upserts parsed SanctionDetail records: inserts new ones, updates changed
- /// ones, leaves unchanged ones alone. Processes in batches so a run of
- /// ~18k records doesn't sit in one giant transaction/timeout.
+ /// ones, leaves unchanged ones alone, reactivates relisted ones, and
+ /// soft-deletes (IsActive = false) records that are no longer in the file.
+ /// Processes in batches so a run of ~18k records doesn't sit in one giant
+ /// transaction/timeout.
///
public record SaveSanctionDetailsCommand(List Records) : IRequest;
- public record SaveSanctionDetailsResult(int Inserted, int Updated, int Unchanged);
+ public record SaveSanctionDetailsResult(int Inserted, int Updated, int Unchanged, int Removed);
public class SaveSanctionDetailsCommandHandler : IRequestHandler
{
private const int BatchSize = 500;
+ private const int DefaultMaxRemovalsPerRun = 500;
+ private const int MaxUidsToLog = 50;
private readonly SanctionsDbContext _db;
private readonly ILogger _logger;
+ private readonly int _maxRemovalsPerRun;
- public SaveSanctionDetailsCommandHandler(SanctionsDbContext db, ILogger logger)
+ public SaveSanctionDetailsCommandHandler(
+ SanctionsDbContext db,
+ ILogger logger,
+ IConfiguration configuration)
{
_db = db;
_logger = logger;
+ _maxRemovalsPerRun = configuration.GetValue("Import:MaxRemovalsPerRun", DefaultMaxRemovalsPerRun);
}
[Time]
public async Task Handle(SaveSanctionDetailsCommand request, CancellationToken cancellationToken)
{
+ if (request.Records.Count == 0)
+ {
+ throw new InvalidOperationException("SaveSanctionDetailsCommand received 0 records; refusing to proceed.");
+ }
+
_logger.LogInformation("Loading existing SDN records for comparison...");
var existingByUid = await _db.SanctionDetails
@@ -41,55 +56,133 @@ public async Task Handle(SaveSanctionDetailsCommand r
_logger.LogInformation("Loaded {Count} existing records.", existingByUid.Count);
+ // ---- Work out removals BEFORE writing anything ----
+ var fileUids = request.Records
+ .Select(r => r.RecordUniqueId)
+ .ToHashSet(StringComparer.Ordinal);
+
+ var toRemove = existingByUid.Values
+ .Where(d => d.IsActive && !fileUids.Contains(d.RecordUniqueId))
+ .Select(d => d.RecordUniqueId)
+ .ToList();
+
+ if (toRemove.Count > _maxRemovalsPerRun)
+ {
+ // A truncated/bad file looks identical to mass delisting. Legitimate mass
+ // removals (e.g. a program terminated) do happen: verify against OFAC's
+ // Recent Actions, then raise Import:MaxRemovalsPerRun and rerun.
+ throw new InvalidOperationException(
+ $"{toRemove.Count} records would be delisted, exceeding the limit of {_maxRemovalsPerRun}. " +
+ "Aborting before any changes were made. Verify against OFAC Recent Actions.");
+ }
+
+ // ---- Upsert ----
_db.ChangeTracker.AutoDetectChangesEnabled = false;
int inserted = 0, updated = 0, unchanged = 0;
var processed = 0;
+ var relisted = new List();
- foreach (var batch in request.Records.Chunk(BatchSize))
+ try
{
- foreach (var record in batch)
+ foreach (var batch in request.Records.Chunk(BatchSize))
{
- if (existingByUid.TryGetValue(record.RecordUniqueId, out var existingRecord))
+ foreach (var record in batch)
{
- if (existingRecord.XmlRecord != record.XmlRecord
- || existingRecord.Country != record.Country
- || existingRecord.LastName != record.LastName
- || existingRecord.FirstName != record.FirstName
- || existingRecord.SdnType != record.SdnType)
+ if (existingByUid.TryGetValue(record.RecordUniqueId, out var existingRecord))
{
- existingRecord.XmlRecord = record.XmlRecord;
- existingRecord.Country = record.Country;
- existingRecord.ImportedAtUtc = DateTime.UtcNow;
- existingRecord.LastName = record.LastName;
- existingRecord.FirstName = record.FirstName;
- existingRecord.SdnType = record.SdnType;
-
- _db.Attach(existingRecord);
- _db.Entry(existingRecord).State = EntityState.Modified;
-
- updated++;
+ var isRelisted = !existingRecord.IsActive;
+
+ if (isRelisted
+ || existingRecord.XmlRecord != record.XmlRecord
+ || existingRecord.Country != record.Country
+ || existingRecord.LastName != record.LastName
+ || existingRecord.FirstName != record.FirstName
+ || existingRecord.SdnType != record.SdnType)
+ {
+ // Attach BEFORE mutating: EF snapshots the current (old) values on
+ // attach, so DetectChanges below can flag only the columns that
+ // really changed. A pure relist then updates just IsActive /
+ // RemovedAtUtc / ImportedAtUtc and never rewrites XmlRecord or the
+ // full-text indexed name columns.
+ _db.Attach(existingRecord);
+
+ if (isRelisted)
+ {
+ relisted.Add(record.RecordUniqueId);
+ }
+
+ existingRecord.XmlRecord = record.XmlRecord;
+ existingRecord.Country = record.Country;
+ existingRecord.ImportedAtUtc = DateTime.UtcNow;
+ existingRecord.LastName = record.LastName;
+ existingRecord.FirstName = record.FirstName;
+ existingRecord.SdnType = record.SdnType;
+ existingRecord.IsActive = true;
+ existingRecord.RemovedAtUtc = null;
+
+ updated++;
+ }
+ else
+ {
+ unchanged++;
+ }
}
else
{
- unchanged++;
+ _db.SanctionDetails.Add(record);
+ inserted++;
}
}
- else
- {
- _db.SanctionDetails.Add(record);
- inserted++;
- }
+
+ // AutoDetectChanges is off, so run it once per batch. Without this call EF
+ // would not see the property changes above and nothing would be saved.
+ _db.ChangeTracker.DetectChanges();
+ await _db.SaveChangesAsync(cancellationToken);
+ _db.ChangeTracker.Clear();
+
+ processed += batch.Length;
+ _logger.LogInformation("Saved {Processed}/{Total} records...", processed, request.Records.Count);
}
+ }
+ finally
+ {
+ // The context is shared with the orchestrator (DataImportLog save), so restore this.
+ _db.ChangeTracker.AutoDetectChangesEnabled = true;
+ }
+
+ if (relisted.Count > 0)
+ {
+ _logger.LogWarning(
+ "{Count} previously delisted SDN records were relisted (showing up to {Max}): {Uids}",
+ relisted.Count,
+ MaxUidsToLog,
+ string.Join(", ", relisted.Take(MaxUidsToLog)));
+ }
+
+ // ---- Sweep: only reached if the whole upsert succeeded ----
+ if (toRemove.Count > 0)
+ {
+ var removedAtUtc = DateTime.UtcNow;
- await _db.SaveChangesAsync(cancellationToken);
- _db.ChangeTracker.Clear();
+ foreach (var chunk in toRemove.Chunk(BatchSize))
+ {
+ await _db.SanctionDetails
+ .Where(d => d.IsActive && chunk.Contains(d.RecordUniqueId))
+ .ExecuteUpdateAsync(s => s
+ .SetProperty(d => d.IsActive, false)
+ .SetProperty(d => d.RemovedAtUtc, removedAtUtc),
+ cancellationToken);
+ }
- processed += batch.Length;
- _logger.LogInformation("Saved {Processed}/{Total} records...", processed, request.Records.Count);
+ // Bounded by the threshold above, so safe to log in full for audit.
+ _logger.LogWarning(
+ "Delisted {Count} SDN records no longer present in file: {Uids}",
+ toRemove.Count,
+ string.Join(", ", toRemove));
}
- return new SaveSanctionDetailsResult(inserted, updated, unchanged);
+ return new SaveSanctionDetailsResult(inserted, updated, unchanged, toRemove.Count);
}
}
}
\ No newline at end of file