Bugfixes export

Directory.Build.props

@@ -3,7 +3,7 @@
 		<PackageProjectUrl>https://www.github.com/JJConsulting/JJMasterData</PackageProjectUrl>
 		<RepositoryUrl>https://www.github.com/JJConsulting/JJMasterData</RepositoryUrl>
 		<PackageIcon>JJMasterData.png</PackageIcon>
-		<VersionPrefix>5.0.4</VersionPrefix>
+		<VersionPrefix>5.0.5</VersionPrefix>
 		<Version>$(VersionPrefix)</Version>
 		<Version Condition="'$(VersionSuffix)' != ''">$(VersionPrefix)-$(VersionSuffix)</Version>
 		<PackageVersion>$(Version)</PackageVersion>

src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs

@@ -31,7 +31,7 @@ internal sealed class ExportJobHandler(
     IStringLocalizer<MasterDataResources> localizer,
     IMasterDataUser masterDataUser) : BackgroundJobHandler<ExportRequest>
 {
-    private const int RecordsPerPage = 100_000;
+    private const int RecordsPerPage = 10_000;
 
     public override async Task<object?> ExecuteAsync(
         ExportRequest request,
@@ -43,15 +43,36 @@ internal sealed class ExportJobHandler(
         var formElement = request.FormElement;
         var format = formats.GetRequired(request.FormatId);
         var columns = GetColumns(request.FormElement, request.ExportAllFields);
-        var source = await CreateSourceAsync(formElement, request, cancellationToken);
+        List<Dictionary<string, object?>> firstPage;
+        long totalRecords;
+        EntityParameters? parameters = null;
+        if (request.Rows is not null)
+        {
+            firstPage = request.Rows;
+            totalRecords = request.Rows.Count;
+        }
+        else
+        {
+            parameters = new EntityParameters
+            {
+                Filters = new Dictionary<string, object?>(request.Filters),
+                RecordsPerPage = RecordsPerPage,
+                OrderBy = OrderByData.FromString(request.OrderBy),
+                CurrentPage = 1
+            };
+            var result = await entityRepository.GetDictionaryListResultAsync(formElement, parameters);
+            cancellationToken.ThrowIfCancellationRequested();
+            firstPage = result.Data;
+            totalRecords = result.TotalOfRecords;
+        }
 
         var context = new ExportContext
         {
             FormElement = formElement,
             Columns = columns,
-            Rows = GetRowsAsync(formElement, source, cancellationToken),
+            Rows = GetRowsAsync(formElement, firstPage, totalRecords, parameters, cancellationToken),
             UserValues = new Dictionary<string, object?>(request.UserValues),
-            TotalRecords = source.Total,
+            TotalRecords = totalRecords,
             Progress = new Progress<ExportProgress>(current => progress.Report(
                 new BackgroundJobProgress(current.Percentage, current.Message, current)))
         };
@@ -99,60 +120,39 @@ private List<FormElementField> GetColumns(FormElement formElement, bool exportAl
             .ToList();
     }
 
-    private async Task<ExportSource> CreateSourceAsync(
-        FormElement formElement,
-        ExportRequest request,
-        CancellationToken cancellationToken)
-    {
-        if (request.Rows is not null)
-            return new ExportSource(request.Rows, request.Rows.Count, null);
-
-        var parameters = new EntityParameters
-        {
-            Filters = new Dictionary<string, object?>(request.Filters),
-            RecordsPerPage = RecordsPerPage,
-            OrderBy = OrderByData.FromString(request.OrderBy),
-            CurrentPage = 1
-        };
-        var firstPage = await entityRepository.GetDictionaryListResultAsync(formElement, parameters);
-        cancellationToken.ThrowIfCancellationRequested();
-        return new ExportSource(
-            firstPage.Data,
-            firstPage.TotalOfRecords,
-            parameters);
-    }
-
     private async IAsyncEnumerable<Dictionary<string, object?>> GetRowsAsync(
         FormElement formElement,
-        ExportSource source,
+        List<Dictionary<string, object?>> firstPage,
+        long totalOfRecords,
+        EntityParameters? parameters,
         [EnumeratorCancellation] CancellationToken cancellationToken)
     {
-        var totalPages = source.Parameters is null ? 1 :
-            (int)Math.Ceiling((source.Total ?? 0) / (double)RecordsPerPage);
+        var totalPages = parameters is null ? 1 :
+            (int)Math.Ceiling(totalOfRecords / (double)RecordsPerPage);
 
         for (var page = 1; page <= Math.Max(1, totalPages); page++)
         {
             List<Dictionary<string, object?>> rows;
             if (page == 1)
-                rows = source.FirstPage;
+                rows = firstPage;
             else
             {
                 var pageParameters = new EntityParameters
                 {
-                    Filters = source.Parameters!.Filters,
-                    RecordsPerPage = source.Parameters.RecordsPerPage,
-                    OrderBy = source.Parameters.OrderBy,
+                    Filters = parameters!.Filters,
+                    RecordsPerPage = parameters.RecordsPerPage,
+                    OrderBy = parameters.OrderBy,
                     CurrentPage = page
                 };
-                var result = await entityRepository.GetDictionaryListResultAsync(formElement, pageParameters);
+                var result = await entityRepository.GetDictionaryListResultAsync(
+                    formElement, pageParameters, recoverTotalOfRecords: false);
                 rows = result.Data;
             }
 
             foreach (var sourceRow in rows)
             {
                 cancellationToken.ThrowIfCancellationRequested();
-                var row = new Dictionary<string, object?>(sourceRow, StringComparer.OrdinalIgnoreCase);
-                yield return row;
+                yield return sourceRow;
             }
         }
     }
@@ -170,14 +170,4 @@ private string GetFileName(FormElement formElement, string extension)
         name = HttpUtility.UrlEncode(name, Encoding.UTF8);
         return $"{name}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.{extension.TrimStart('.').ToLowerInvariant()}";
     }
-
-    private sealed class ExportSource(
-        List<Dictionary<string, object?>> firstPage,
-        long? total,
-        EntityParameters? parameters)
-    {
-        public List<Dictionary<string, object?>> FirstPage { get; init; } = firstPage;
-        public long? Total { get; init; } = total;
-        public EntityParameters? Parameters { get; init; } = parameters;
-    }
 }

src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs

@@ -0,0 +1,19 @@
+namespace JJMasterData.Core.DataManager.Exportation;
+
+internal sealed class ExportProgressReporter(ExportContext context)
+{
+    private const long ReportInterval = 50;
+    private long _lastReported;
+
+    public void Report(long processed, bool completed = false)
+    {
+        if (!completed && processed - _lastReported < ReportInterval)
+            return;
+
+        if (processed == _lastReported)
+            return;
+
+        _lastReported = processed;
+        context.Progress.Report(new ExportProgress(processed, context.TotalRecords, $"Exporting {processed:N0} records..."));
+    }
+}

src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs

@@ -1,16 +1,13 @@
 using System.IO;
-using JJMasterData.Core.DataManager.Services;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
 
-internal sealed class CsvExportFormat(FieldFormattingService fieldFormattingService)
-    : DelimitedTextExportFormat<CsvExportOptions>(fieldFormattingService)
+internal sealed class CsvExportFormat : DelimitedTextExportFormat<CsvExportOptions>
 {
     public override string Id => "csv";
     public override string DisplayName => "CSV";
     public override string FileExtension => "csv";
-    public override string ContentType => "text/csv";
-
+
     protected override string GetDelimiter(CsvExportOptions options) => options.Delimiter switch
     {
         CsvExportDelimiter.Semicolon => ";",

src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs

@@ -1,24 +1,22 @@
+using System.Globalization;
 using System.IO;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using CsvHelper;
 using CsvHelper.Configuration;
-using JJMasterData.Core.DataDictionary.Models;
 using JJMasterData.Core.DataManager.Exportation.Abstractions;
-using JJMasterData.Core.DataManager.Models;
 using JJMasterData.Core.DataManager.Services;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
 
-internal abstract class DelimitedTextExportFormat<TOptions>(FieldFormattingService fieldFormattingService) : IExportFormat<TOptions> where TOptions : ExportFormatOptions, new()
+internal abstract class DelimitedTextExportFormat<TOptions> : IExportFormat<TOptions> where TOptions : ExportFormatOptions, new()
 {
     protected abstract string GetDelimiter(TOptions options);
 
     public abstract string Id { get; }
     public abstract string DisplayName { get; }
     public abstract string FileExtension { get; }
-    public abstract string ContentType { get; }
 
     public async Task WriteAsync(
         ExportContext context,
@@ -28,7 +26,7 @@ public async Task WriteAsync(
     {
         await using var textWriter = new StreamWriter(output, new UTF8Encoding(true), leaveOpen: true);
         await using var csv = new CsvWriter(textWriter,
-            new CsvConfiguration(System.Globalization.CultureInfo.CurrentCulture)
+            new CsvConfiguration(CultureInfo.CurrentCulture)
             {
                 Delimiter = GetDelimiter(options),
                 HasHeaderRecord = false
@@ -42,21 +40,21 @@ public async Task WriteAsync(
         }
 
         long processed = 0;
+        var progress = new ExportProgressReporter(context);
         await foreach (var row in context.Rows.WithCancellation(cancellationToken))
         {
-            var formState = new FormStateData(row, context.UserValues, PageState.List);
             foreach (var field in context.Columns)
             {
-                var value = await fieldFormattingService.FormatGridValueAsync(
-                    new FormElementFieldSelector(context.FormElement, field.Name), formState);
+                row.TryGetValue(field.Name, out var rawValue);
+                var value = FieldFormattingService.FormatValue(field, rawValue);
                 csv.WriteField(value);
             }
             await csv.NextRecordAsync();
             processed++;
-            context.Progress.Report(new ExportProgress(
-                processed, context.TotalRecords, $"Exporting {processed:N0} records..."));
+            progress.Report(processed);
         }
 
+        progress.Report(processed, completed: true);
         await textWriter.FlushAsync(cancellationToken);
     }
-}
+}

src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs

@@ -3,14 +3,12 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
-using JJMasterData.Core.DataDictionary.Models;
 using JJMasterData.Core.DataManager.Exportation.Abstractions;
-using JJMasterData.Core.DataManager.Models;
 using JJMasterData.Core.DataManager.Services;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
 
-public sealed class ExcelXlsExportFormat(FieldFormattingService fieldFormattingService) : IExportFormat<ExcelXlsExportOptions>
+public sealed class ExcelXlsExportFormat : IExportFormat<ExcelXlsExportOptions>
 {
     public string Id => "excel";
     public string DisplayName => "Excel (.xls)";
@@ -30,20 +28,21 @@ public async Task WriteAsync(ExportContext context, ExcelXlsExportOptions option
         }
         await writer.WriteAsync("<tbody>");
         long processed = 0;
+        var progress = new ExportProgressReporter(context);
         await foreach (var row in context.Rows.WithCancellation(cancellationToken))
         {
             await writer.WriteAsync("<tr>");
-            var formState = new FormStateData(row, context.UserValues, PageState.List);
             foreach (var column in context.Columns)
             {
-                var value = await fieldFormattingService.FormatGridValueAsync(
-                    new FormElementFieldSelector(context.FormElement, column.Name), formState);
+                row.TryGetValue(column.Name, out var rawValue);
+                var value = FieldFormattingService.FormatValue(column, rawValue);
                 await writer.WriteAsync($"<td>{HttpUtility.HtmlEncode(value)}</td>");
             }
             await writer.WriteAsync("</tr>");
             processed++;
-            context.Progress.Report(new ExportProgress(processed, context.TotalRecords, $"Exporting {processed:N0} records..."));
+            progress.Report(processed);
         }
+        progress.Report(processed, completed: true);
         await writer.WriteAsync("</tbody></table></body></html>");
         await writer.FlushAsync(cancellationToken);
     }

src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs

@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
-using JJMasterData.Core.DataManager.Exportation;
 using MiniExcelLibs;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
@@ -14,6 +13,7 @@ internal sealed class ExcelXlsxDataReader(
     private IAsyncEnumerator<Dictionary<string, object?>>? _enumerator;
     private Dictionary<string, object?>? _current;
     private long _processed;
+    private readonly ExportProgressReporter _progress = new(context);
     private bool _disposed;
 
     public override int FieldCount => context.Columns.Count;
@@ -54,15 +54,13 @@ public override async Task<bool> ReadAsync(CancellationToken token = default)
         if (!await _enumerator.MoveNextAsync())
         {
             _current = null;
+            _progress.Report(_processed, completed: true);
             return false;
         }
 
         _current = _enumerator.Current;
         _processed++;
-        context.Progress.Report(new ExportProgress(
-            _processed,
-            context.TotalRecords,
-            $"Exporting {_processed:N0} records..."));
+        _progress.Report(_processed);
         return true;
     }
 
@@ -78,8 +76,8 @@ public override async ValueTask DisposeAsync()
 
         _disposed = true;
         _current = null;
+
         if (_enumerator is not null)
             await _enumerator.DisposeAsync();
-        GC.SuppressFinalize(this);
     }
 }

src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs

@@ -1,7 +1,6 @@
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
-using JJMasterData.Core.DataManager.Exportation;
 using JJMasterData.Core.DataManager.Exportation.Abstractions;
 using MiniExcelLibs;
 using MiniExcelLibs.OpenXml;

src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs

@@ -1,16 +1,13 @@
 using System.IO;
-using JJMasterData.Core.DataManager.Services;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
 
-internal sealed class TextExportFormat(FieldFormattingService fieldFormattingService)
-    : DelimitedTextExportFormat<TextExportOptions>(fieldFormattingService)
+internal sealed class TextExportFormat : DelimitedTextExportFormat<TextExportOptions>
 {
     public override string Id => "txt";
     public override string DisplayName => "Text";
     public override string FileExtension => "txt";
-    public override string ContentType => "text/plain";
-
+
     protected override string GetDelimiter(TextExportOptions options) => options.Delimiter switch
     {
         TextExportDelimiter.Tab => "\t",

src/MasterData.Core/DataManager/Expressions/Providers/DefaultExpressionProvider.cs

@@ -8,7 +8,6 @@
 using Microsoft.Extensions.Options;
 using NCalc;
 using NCalc.Factories;
-using NCalc.Handlers;
 
 namespace JJMasterData.Core.DataManager.Expressions.Providers;
 
@@ -63,7 +62,7 @@ private static string PrepareExpressionWithParameters(
             if (expression.Contains(quotedToken, StringComparison.InvariantCultureIgnoreCase))
             {
                 expression = expression.Replace(quotedToken, kvp.Key);
-                parameters[kvp.Key] = value?.ToString();
+                parameters[kvp.Key] = value?.ToString() ?? string.Empty;
             }
             else
             {