diff --git a/Directory.Build.props b/Directory.Build.props index e337e46e6..a86731c04 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -3,7 +3,7 @@ https://www.github.com/JJConsulting/JJMasterData https://www.github.com/JJConsulting/JJMasterData JJMasterData.png - 5.0.4 + 5.0.5 $(VersionPrefix) $(VersionPrefix)-$(VersionSuffix) $(Version) diff --git a/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs b/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs index be8a22bcb..2032fc51b 100644 --- a/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs +++ b/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs @@ -31,7 +31,7 @@ internal sealed class ExportJobHandler( IStringLocalizer localizer, IMasterDataUser masterDataUser) : BackgroundJobHandler { - private const int RecordsPerPage = 100_000; + private const int RecordsPerPage = 10_000; public override async Task ExecuteAsync( ExportRequest request, @@ -43,15 +43,36 @@ internal sealed class ExportJobHandler( var formElement = request.FormElement; var format = formats.GetRequired(request.FormatId); var columns = GetColumns(request.FormElement, request.ExportAllFields); - var source = await CreateSourceAsync(formElement, request, cancellationToken); + List> firstPage; + long totalRecords; + EntityParameters? parameters = null; + if (request.Rows is not null) + { + firstPage = request.Rows; + totalRecords = request.Rows.Count; + } + else + { + parameters = new EntityParameters + { + Filters = new Dictionary(request.Filters), + RecordsPerPage = RecordsPerPage, + OrderBy = OrderByData.FromString(request.OrderBy), + CurrentPage = 1 + }; + var result = await entityRepository.GetDictionaryListResultAsync(formElement, parameters); + cancellationToken.ThrowIfCancellationRequested(); + firstPage = result.Data; + totalRecords = result.TotalOfRecords; + } var context = new ExportContext { FormElement = formElement, Columns = columns, - Rows = GetRowsAsync(formElement, source, cancellationToken), + Rows = GetRowsAsync(formElement, firstPage, totalRecords, parameters, cancellationToken), UserValues = new Dictionary(request.UserValues), - TotalRecords = source.Total, + TotalRecords = totalRecords, Progress = new Progress(current => progress.Report( new BackgroundJobProgress(current.Percentage, current.Message, current))) }; @@ -99,60 +120,39 @@ private List GetColumns(FormElement formElement, bool exportAl .ToList(); } - private async Task CreateSourceAsync( - FormElement formElement, - ExportRequest request, - CancellationToken cancellationToken) - { - if (request.Rows is not null) - return new ExportSource(request.Rows, request.Rows.Count, null); - - var parameters = new EntityParameters - { - Filters = new Dictionary(request.Filters), - RecordsPerPage = RecordsPerPage, - OrderBy = OrderByData.FromString(request.OrderBy), - CurrentPage = 1 - }; - var firstPage = await entityRepository.GetDictionaryListResultAsync(formElement, parameters); - cancellationToken.ThrowIfCancellationRequested(); - return new ExportSource( - firstPage.Data, - firstPage.TotalOfRecords, - parameters); - } - private async IAsyncEnumerable> GetRowsAsync( FormElement formElement, - ExportSource source, + List> firstPage, + long totalOfRecords, + EntityParameters? parameters, [EnumeratorCancellation] CancellationToken cancellationToken) { - var totalPages = source.Parameters is null ? 1 : - (int)Math.Ceiling((source.Total ?? 0) / (double)RecordsPerPage); + var totalPages = parameters is null ? 1 : + (int)Math.Ceiling(totalOfRecords / (double)RecordsPerPage); for (var page = 1; page <= Math.Max(1, totalPages); page++) { List> rows; if (page == 1) - rows = source.FirstPage; + rows = firstPage; else { var pageParameters = new EntityParameters { - Filters = source.Parameters!.Filters, - RecordsPerPage = source.Parameters.RecordsPerPage, - OrderBy = source.Parameters.OrderBy, + Filters = parameters!.Filters, + RecordsPerPage = parameters.RecordsPerPage, + OrderBy = parameters.OrderBy, CurrentPage = page }; - var result = await entityRepository.GetDictionaryListResultAsync(formElement, pageParameters); + var result = await entityRepository.GetDictionaryListResultAsync( + formElement, pageParameters, recoverTotalOfRecords: false); rows = result.Data; } foreach (var sourceRow in rows) { cancellationToken.ThrowIfCancellationRequested(); - var row = new Dictionary(sourceRow, StringComparer.OrdinalIgnoreCase); - yield return row; + yield return sourceRow; } } } @@ -170,14 +170,4 @@ private string GetFileName(FormElement formElement, string extension) name = HttpUtility.UrlEncode(name, Encoding.UTF8); return $"{name}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.{extension.TrimStart('.').ToLowerInvariant()}"; } - - private sealed class ExportSource( - List> firstPage, - long? total, - EntityParameters? parameters) - { - public List> FirstPage { get; init; } = firstPage; - public long? Total { get; init; } = total; - public EntityParameters? Parameters { get; init; } = parameters; - } } diff --git a/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs b/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs new file mode 100644 index 000000000..82ac99249 --- /dev/null +++ b/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs @@ -0,0 +1,19 @@ +namespace JJMasterData.Core.DataManager.Exportation; + +internal sealed class ExportProgressReporter(ExportContext context) +{ + private const long ReportInterval = 50; + private long _lastReported; + + public void Report(long processed, bool completed = false) + { + if (!completed && processed - _lastReported < ReportInterval) + return; + + if (processed == _lastReported) + return; + + _lastReported = processed; + context.Progress.Report(new ExportProgress(processed, context.TotalRecords, $"Exporting {processed:N0} records...")); + } +} diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs index f3ab43f98..9a083cfc2 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs @@ -1,16 +1,13 @@ using System.IO; -using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -internal sealed class CsvExportFormat(FieldFormattingService fieldFormattingService) - : DelimitedTextExportFormat(fieldFormattingService) +internal sealed class CsvExportFormat : DelimitedTextExportFormat { public override string Id => "csv"; public override string DisplayName => "CSV"; public override string FileExtension => "csv"; - public override string ContentType => "text/csv"; - + protected override string GetDelimiter(CsvExportOptions options) => options.Delimiter switch { CsvExportDelimiter.Semicolon => ";", diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs index 36294131f..5066a3ab0 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs @@ -1,24 +1,22 @@ +using System.Globalization; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; using CsvHelper; using CsvHelper.Configuration; -using JJMasterData.Core.DataDictionary.Models; using JJMasterData.Core.DataManager.Exportation.Abstractions; -using JJMasterData.Core.DataManager.Models; using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -internal abstract class DelimitedTextExportFormat(FieldFormattingService fieldFormattingService) : IExportFormat where TOptions : ExportFormatOptions, new() +internal abstract class DelimitedTextExportFormat : IExportFormat where TOptions : ExportFormatOptions, new() { protected abstract string GetDelimiter(TOptions options); public abstract string Id { get; } public abstract string DisplayName { get; } public abstract string FileExtension { get; } - public abstract string ContentType { get; } public async Task WriteAsync( ExportContext context, @@ -28,7 +26,7 @@ public async Task WriteAsync( { await using var textWriter = new StreamWriter(output, new UTF8Encoding(true), leaveOpen: true); await using var csv = new CsvWriter(textWriter, - new CsvConfiguration(System.Globalization.CultureInfo.CurrentCulture) + new CsvConfiguration(CultureInfo.CurrentCulture) { Delimiter = GetDelimiter(options), HasHeaderRecord = false @@ -42,21 +40,21 @@ public async Task WriteAsync( } long processed = 0; + var progress = new ExportProgressReporter(context); await foreach (var row in context.Rows.WithCancellation(cancellationToken)) { - var formState = new FormStateData(row, context.UserValues, PageState.List); foreach (var field in context.Columns) { - var value = await fieldFormattingService.FormatGridValueAsync( - new FormElementFieldSelector(context.FormElement, field.Name), formState); + row.TryGetValue(field.Name, out var rawValue); + var value = FieldFormattingService.FormatValue(field, rawValue); csv.WriteField(value); } await csv.NextRecordAsync(); processed++; - context.Progress.Report(new ExportProgress( - processed, context.TotalRecords, $"Exporting {processed:N0} records...")); + progress.Report(processed); } + progress.Report(processed, completed: true); await textWriter.FlushAsync(cancellationToken); } -} \ No newline at end of file +} diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs index 87dd45b66..209491842 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs @@ -3,14 +3,12 @@ using System.Threading; using System.Threading.Tasks; using System.Web; -using JJMasterData.Core.DataDictionary.Models; using JJMasterData.Core.DataManager.Exportation.Abstractions; -using JJMasterData.Core.DataManager.Models; using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -public sealed class ExcelXlsExportFormat(FieldFormattingService fieldFormattingService) : IExportFormat +public sealed class ExcelXlsExportFormat : IExportFormat { public string Id => "excel"; public string DisplayName => "Excel (.xls)"; @@ -30,20 +28,21 @@ public async Task WriteAsync(ExportContext context, ExcelXlsExportOptions option } await writer.WriteAsync(""); long processed = 0; + var progress = new ExportProgressReporter(context); await foreach (var row in context.Rows.WithCancellation(cancellationToken)) { await writer.WriteAsync(""); - var formState = new FormStateData(row, context.UserValues, PageState.List); foreach (var column in context.Columns) { - var value = await fieldFormattingService.FormatGridValueAsync( - new FormElementFieldSelector(context.FormElement, column.Name), formState); + row.TryGetValue(column.Name, out var rawValue); + var value = FieldFormattingService.FormatValue(column, rawValue); await writer.WriteAsync($"{HttpUtility.HtmlEncode(value)}"); } await writer.WriteAsync(""); processed++; - context.Progress.Report(new ExportProgress(processed, context.TotalRecords, $"Exporting {processed:N0} records...")); + progress.Report(processed); } + progress.Report(processed, completed: true); await writer.WriteAsync(""); await writer.FlushAsync(cancellationToken); } diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs index 963f7d847..8d908f559 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using JJMasterData.Core.DataManager.Exportation; using MiniExcelLibs; namespace JJMasterData.Core.DataManager.Exportation.Formats; @@ -14,6 +13,7 @@ internal sealed class ExcelXlsxDataReader( private IAsyncEnumerator>? _enumerator; private Dictionary? _current; private long _processed; + private readonly ExportProgressReporter _progress = new(context); private bool _disposed; public override int FieldCount => context.Columns.Count; @@ -54,15 +54,13 @@ public override async Task ReadAsync(CancellationToken token = default) if (!await _enumerator.MoveNextAsync()) { _current = null; + _progress.Report(_processed, completed: true); return false; } _current = _enumerator.Current; _processed++; - context.Progress.Report(new ExportProgress( - _processed, - context.TotalRecords, - $"Exporting {_processed:N0} records...")); + _progress.Report(_processed); return true; } @@ -78,8 +76,8 @@ public override async ValueTask DisposeAsync() _disposed = true; _current = null; + if (_enumerator is not null) await _enumerator.DisposeAsync(); - GC.SuppressFinalize(this); } } diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs index 39d22296d..c41c23426 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs @@ -1,7 +1,6 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using JJMasterData.Core.DataManager.Exportation; using JJMasterData.Core.DataManager.Exportation.Abstractions; using MiniExcelLibs; using MiniExcelLibs.OpenXml; diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs index 18d046c11..cd961fad7 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs @@ -1,16 +1,13 @@ using System.IO; -using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -internal sealed class TextExportFormat(FieldFormattingService fieldFormattingService) - : DelimitedTextExportFormat(fieldFormattingService) +internal sealed class TextExportFormat : DelimitedTextExportFormat { public override string Id => "txt"; public override string DisplayName => "Text"; public override string FileExtension => "txt"; - public override string ContentType => "text/plain"; - + protected override string GetDelimiter(TextExportOptions options) => options.Delimiter switch { TextExportDelimiter.Tab => "\t", diff --git a/src/MasterData.Core/DataManager/Expressions/Providers/DefaultExpressionProvider.cs b/src/MasterData.Core/DataManager/Expressions/Providers/DefaultExpressionProvider.cs index 0cc0308e0..03f26a04f 100644 --- a/src/MasterData.Core/DataManager/Expressions/Providers/DefaultExpressionProvider.cs +++ b/src/MasterData.Core/DataManager/Expressions/Providers/DefaultExpressionProvider.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.Options; using NCalc; using NCalc.Factories; -using NCalc.Handlers; namespace JJMasterData.Core.DataManager.Expressions.Providers; @@ -63,7 +62,7 @@ private static string PrepareExpressionWithParameters( if (expression.Contains(quotedToken, StringComparison.InvariantCultureIgnoreCase)) { expression = expression.Replace(quotedToken, kvp.Key); - parameters[kvp.Key] = value?.ToString(); + parameters[kvp.Key] = value?.ToString() ?? string.Empty; } else { diff --git a/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs b/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs index 9933803e4..68f6088dd 100644 --- a/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs +++ b/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs @@ -39,7 +39,7 @@ public async ValueTask FormatGridValueAsync( break; case FormComponent.Number: case FormComponent.Slider: - stringValue = GetNumericValueAsString(field, value,$"N{field.NumberOfDecimalPlaces}"); + stringValue = GetNumericValueAsString(field, value, $"N{field.NumberOfDecimalPlaces}"); break; case FormComponent.Currency: stringValue = GetNumericValueAsString(field, value,$"C{field.NumberOfDecimalPlaces}"); @@ -63,7 +63,7 @@ public async ValueTask FormatGridValueAsync( if (field.EncodeHtml) stringValue = HttpUtility.HtmlEncode(stringValue); - + return stringValue.Trim(); } @@ -166,4 +166,4 @@ private static string GetNumericValueAsString(FormElementField field, object? va return stringValue ?? string.Empty; } -} \ No newline at end of file +} diff --git a/test/MasterData.Core.Test/DataManager/Expressions/DefaultExpressionProviderTests.cs b/test/MasterData.Core.Test/DataManager/Expressions/DefaultExpressionProviderTests.cs new file mode 100644 index 000000000..940612f45 --- /dev/null +++ b/test/MasterData.Core.Test/DataManager/Expressions/DefaultExpressionProviderTests.cs @@ -0,0 +1,54 @@ +using JJMasterData.Core.Configuration.Options; +using JJMasterData.Core.DataManager.Expressions.Providers; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NCalc.DependencyInjection; +using NCalc.Factories; + +namespace JJMasterData.Core.Test.DataManager.Expressions; + +public class DefaultExpressionProviderTests +{ + [Theory] + [InlineData(null, true)] + [InlineData("", true)] + [InlineData("filled", false)] + public void QuotedParameterEqualsEmptyStringOnlyWhenEmpty(string? value, bool expected) + { + var provider = CreateProvider(); + + var result = provider.Evaluate("'{VALOR}' = ''", new Dictionary + { + ["VALOR"] = value + }); + + Assert.Equal(expected, result); + } + + [Fact] + public void UnquotedNullParameterRemainsNull() + { + var provider = CreateProvider(); + + var result = provider.Evaluate("{VALOR} = null", new Dictionary + { + ["VALOR"] = null + }); + + Assert.Equal(true, result); + } + + private static DefaultExpressionProvider CreateProvider() + { + var services = new ServiceCollection(); + services.AddNCalc(); + var serviceProvider = services.BuildServiceProvider(); + + return new DefaultExpressionProvider( + serviceProvider.GetRequiredService(), + serviceProvider, + Options.Create(new MasterDataCoreOptions()), + NullLogger.Instance); + } +}