diff --git a/.gitignore b/.gitignore index 967063c4..c6b86061 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,6 @@ yarn-integrity application-dev.properties agents + +# Local CSV uploaded source files +/data-agent-backend/.local/ diff --git a/data-agent-backend/pom.xml b/data-agent-backend/pom.xml index 51802601..9b570ea0 100644 --- a/data-agent-backend/pom.xml +++ b/data-agent-backend/pom.xml @@ -93,6 +93,11 @@ + + org.apache.commons + commons-csv + 1.14.1 + org.springframework.boot spring-boot-starter-web diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/AgentService.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/AgentService.java index ab3a7bb0..558701cb 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/AgentService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/AgentService.java @@ -52,11 +52,11 @@ import io.github.malonetalk.service.DatasourceService; import io.github.malonetalk.service.PermissionService; import io.github.malonetalk.service.SessionDatasourceService; +import io.github.malonetalk.utils.DateTimeUtils; import io.opentelemetry.context.Context; import io.opentelemetry.instrumentation.reactor.v3_1.ContextPropagationOperator; import jakarta.annotation.PostConstruct; import java.time.LocalDate; -import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -74,8 +74,6 @@ @RequiredArgsConstructor public class AgentService { - private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai"); - private final ModelFactory modelFactory; private final List allToolBeans; private final ModelProperties modelProperties; @@ -296,7 +294,7 @@ private ReActAgent createAgent(ToolCallContext toolCallContext) { } private String systemPrompt() { - LocalDate today = LocalDate.now(DEFAULT_ZONE); + LocalDate today = LocalDate.now(DateTimeUtils.DEFAULT_ZONE); return """ 你是一个数据助手,可以帮助用户查询数据库中的数据。 当前时区是 Asia/Shanghai。 diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java index 2f3e9fa5..e0382d6d 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java @@ -22,7 +22,9 @@ import io.github.malonetalk.exception.BusinessException; import io.github.malonetalk.exception.ErrorCode; import io.github.malonetalk.model.converter.JdbcColumnMetadataConverter; +import io.github.malonetalk.model.po.CsvImportPo; import io.github.malonetalk.model.po.DatasourcePo; +import io.github.malonetalk.service.csv.CsvAccessService; import io.github.malonetalk.utils.SemanticUtils; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -48,12 +50,14 @@ public class SchemaReader { private final DynamicDataSourceManager dynamicDataSourceManager; private final JdbcColumnMetadataConverter columnMetadataConverter; + private final CsvAccessService csvAccess; public List getTables(DatasourcePo datasource) { javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource); try (Connection conn = ds.getConnection()) { - return getTables(conn); + var ready = csvAccess.readyFilter(datasource); + return getTables(conn).stream().filter(table -> ready.test(table.tableName())).toList(); } catch (SQLException e) { log.error("Failed to read tables: {}", e.getMessage(), e); throw BusinessException.of( @@ -64,6 +68,17 @@ public List getTables(DatasourcePo datasource) { } public List getTableSchema(DatasourcePo datasource, String tableName) { + csvAccess.requireReady(datasource, tableName); + return readTableSchema(datasource, tableName); + } + + public List getImportTableSchema( + DatasourcePo datasource, CsvImportPo task) { + csvAccess.requireImportTable(datasource, task); + return readTableSchema(datasource, task.getPhysicalTableName()); + } + + private List readTableSchema(DatasourcePo datasource, String tableName) { javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource); try (Connection conn = ds.getConnection()) { @@ -86,7 +101,9 @@ public Map> getTableColumnNames( return Map.of(); } Map> columnNamesByTable = new LinkedHashMap<>(); + var ready = csvAccess.readyFilter(datasource); for (String tableName : tableNames) { + if (!ready.test(tableName)) continue; columnNamesByTable.putIfAbsent(tableName, new LinkedHashSet<>()); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/impls/DateInfoTool.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/impls/DateInfoTool.java index 27eee6ef..8d885196 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/impls/DateInfoTool.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/impls/DateInfoTool.java @@ -26,6 +26,7 @@ import io.github.malonetalk.agent.tools.ToolResultBoConverter; import io.github.malonetalk.exception.ExceptionResponseMapper; import io.github.malonetalk.model.bo.ToolResultBo; +import io.github.malonetalk.utils.DateTimeUtils; import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; @@ -52,7 +53,6 @@ @AllArgsConstructor(access = AccessLevel.PRIVATE) public class DateInfoTool implements DataAgentTool { - private static final String DEFAULT_TIMEZONE = "Asia/Shanghai"; private static final String HOLIDAY_API_BASE_URL = "https://timor.tech/api/holiday/info"; private final ObjectMapper objectMapper; @@ -122,10 +122,12 @@ public ToolResultBo getDateInfo( DateInfo resolve(String startDateText, String endDateText, String timezoneText) { ZoneId zoneId = - ZoneId.of(StringUtils.hasText(timezoneText) ? timezoneText : DEFAULT_TIMEZONE); + StringUtils.hasText(timezoneText) + ? ZoneId.of(timezoneText) + : DateTimeUtils.DEFAULT_ZONE; LocalDate startDate = StringUtils.hasText(startDateText) - ? LocalDate.parse(startDateText) + ? DateTimeUtils.parseDate(startDateText) : LocalDate.now(clock.withZone(zoneId)); Optional holidayApiResponse = queryHolidayApi(startDate); HolidayDetail holiday = holidayApiResponse.map(HolidayApiResponse::holiday).orElse(null); @@ -144,7 +146,7 @@ DateInfo resolve(String startDateText, String endDateText, String timezoneText) .holidayDataAvailable(holidayApiResponse.isPresent()) .dateDiff( StringUtils.hasText(endDateText) - ? resolveDateDiff(startDate, LocalDate.parse(endDateText)) + ? resolveDateDiff(startDate, DateTimeUtils.parseDate(endDateText)) : null) .build(); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/common/CsvLimits.java b/data-agent-backend/src/main/java/io/github/malonetalk/common/CsvLimits.java new file mode 100644 index 00000000..cdb90e5e --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/common/CsvLimits.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.common; + +/** Shared CSV limits keep request validation, file parsing and schema conversion consistent. */ +public final class CsvLimits { + public static final int MAX_COLUMNS = 100; + public static final int MAX_NAME_LENGTH = 200; + + private CsvLimits() {} +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/config/CsvImportConfig.java b/data-agent-backend/src/main/java/io/github/malonetalk/config/CsvImportConfig.java new file mode 100644 index 00000000..0eea9c38 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/config/CsvImportConfig.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.config; + +import java.nio.file.Path; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +@Data +@Configuration +@EnableScheduling +@ConfigurationProperties(prefix = "csv-import") +public class CsvImportConfig { + private Integer datasourceId = 0; + private Path directory = Path.of(System.getProperty("java.io.tmpdir"), "data-agent-csv"); + private String writerUrl = ""; + private String writerUsername = ""; + private String writerPassword = ""; + + // Keep Boot's applicationTaskExecutor available for MVC/SSE. + @Bean(value = "csvImportExecutor", defaultCandidate = false) + public ThreadPoolTaskExecutor csvImportExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(2); + executor.setMaxPoolSize(2); + executor.setQueueCapacity(0); + executor.setThreadNamePrefix("csv-import-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(40); + return executor; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/CsvImportController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/CsvImportController.java new file mode 100644 index 00000000..7be3b2e7 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/CsvImportController.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.controller; + +import io.github.malonetalk.annotation.RequirePermission; +import io.github.malonetalk.enums.PermissionEnum; +import io.github.malonetalk.model.dto.CsvImportConfirmDto; +import io.github.malonetalk.model.dto.CsvParseOptions; +import io.github.malonetalk.model.vo.BooleanVo; +import io.github.malonetalk.model.vo.CsvImportPageVo; +import io.github.malonetalk.model.vo.CsvImportVo; +import io.github.malonetalk.service.csv.CsvImportService; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@RequestMapping("/api/csv-imports") +@RequiredArgsConstructor +public class CsvImportController { + private final CsvImportService service; + + @PostMapping("/preview") + @RequirePermission(PermissionEnum.DATA_CSV_IMPORT) + public CsvImportVo preview( + @RequestParam MultipartFile file, + @RequestParam(defaultValue = "UTF-8") String encoding, + @RequestParam(defaultValue = ",") String delimiter, + @RequestParam(defaultValue = "true") boolean header, + @RequestParam(defaultValue = "true") boolean emptyAsNull) + throws Exception { + return service.preview( + file, + CsvParseOptions.builder() + .encoding(encoding) + .delimiter(delimiter) + .header(header) + .emptyAsNull(emptyAsNull) + .build()); + } + + @PutMapping("/{id}/preview") + @RequirePermission(PermissionEnum.DATA_CSV_IMPORT) + public CsvImportVo repreview( + @PathVariable String id, @Valid @RequestBody CsvParseOptions options) throws Exception { + return service.repreview(id, options); + } + + @PostMapping("/{id}/confirm") + @ResponseStatus(HttpStatus.ACCEPTED) + @RequirePermission({PermissionEnum.DATA_CSV_IMPORT, PermissionEnum.SEMANTIC_TABLE_SYNC}) + public CsvImportVo confirm( + @PathVariable String id, @Valid @RequestBody CsvImportConfirmDto request) { + return service.confirm(id, request); + } + + @GetMapping("/{id}") + @RequirePermission( + value = { + PermissionEnum.DATA_CSV_IMPORT, + PermissionEnum.DATA_CSV_DELETE, + PermissionEnum.DATA_DATASOURCE_READ + }, + anyOf = true) + public CsvImportVo get(@PathVariable String id) { + return service.get(id); + } + + @GetMapping + @RequirePermission( + value = { + PermissionEnum.DATA_CSV_IMPORT, + PermissionEnum.DATA_CSV_DELETE, + PermissionEnum.DATA_DATASOURCE_READ + }, + anyOf = true) + public CsvImportPageVo list( + @RequestParam(defaultValue = "1") int page, + @RequestParam(defaultValue = "10") int pageSize) { + return service.list(page, pageSize); + } + + @PostMapping("/{id}/sync") + @ResponseStatus(HttpStatus.ACCEPTED) + @RequirePermission({PermissionEnum.DATA_CSV_IMPORT, PermissionEnum.SEMANTIC_TABLE_SYNC}) + public BooleanVo sync(@PathVariable String id) { + service.retrySync(id); + return BooleanVo.TRUE; + } + + @DeleteMapping("/{id}") + @RequirePermission(PermissionEnum.DATA_CSV_DELETE) + public BooleanVo delete(@PathVariable String id) throws Exception { + service.delete(id); + return BooleanVo.TRUE; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/enums/CsvImportStatus.java b/data-agent-backend/src/main/java/io/github/malonetalk/enums/CsvImportStatus.java new file mode 100644 index 00000000..6c5f2ed1 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/enums/CsvImportStatus.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.enums; + +public enum CsvImportStatus { + PREVIEWED, + QUEUED, + IMPORTING, + DATA_READY, + SYNCING, + READY, + IMPORT_FAILED, + SYNC_FAILED, + CLEANING, + CLEANUP_FAILED, + DELETING, + DELETE_FAILED, + EXPIRED +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/enums/DatasourceSourceType.java b/data-agent-backend/src/main/java/io/github/malonetalk/enums/DatasourceSourceType.java new file mode 100644 index 00000000..7b6cdb7e --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/enums/DatasourceSourceType.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.enums; + +public enum DatasourceSourceType { + DATABASE, + CSV +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/enums/PermissionEnum.java b/data-agent-backend/src/main/java/io/github/malonetalk/enums/PermissionEnum.java index b36ac762..b931b54b 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/enums/PermissionEnum.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/enums/PermissionEnum.java @@ -37,6 +37,8 @@ public enum PermissionEnum { DATA_DATASOURCE_DELETE("data:datasource:delete", "数据源:删除", "数据源"), DATA_DATASOURCE_SET_STATUS("data:datasource:set-status", "数据源:启停", "数据源"), DATA_SCHEMA_READ("data:schema:read", "表结构:查看", "表结构"), + DATA_CSV_IMPORT("data:csv:import", "CSV:导入", "数据源"), + DATA_CSV_DELETE("data:csv:delete", "CSV:删除", "数据源"), SEMANTIC_DOMAIN_READ("semantic:domain:read", "数据领域:查看", "数据领域"), SEMANTIC_DOMAIN_CREATE("semantic:domain:create", "数据领域:创建", "数据领域"), SEMANTIC_DOMAIN_UPDATE("semantic:domain:update", "数据领域:修改", "数据领域"), diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/exception/ErrorCode.java b/data-agent-backend/src/main/java/io/github/malonetalk/exception/ErrorCode.java index a6e79b0b..e8e0da2d 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/exception/ErrorCode.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/exception/ErrorCode.java @@ -106,6 +106,61 @@ public enum ErrorCode { HttpStatus.NOT_ACCEPTABLE, "No acceptable response media type is available."), + /** 上传文件或 multipart 请求超过服务器配置的大小限制。 */ + PAYLOAD_TOO_LARGE( + "PAYLOAD_TOO_LARGE", + HttpStatus.PAYLOAD_TOO_LARGE, + "Uploaded file or request exceeds the size limit."), + + /** CSV 托管数据源或专用写入连接尚未正确配置。 */ + CSV_NOT_CONFIGURED( + "CSV_NOT_CONFIGURED", + HttpStatus.BAD_REQUEST, + "CSV import is not configured correctly."), + + /** CSV 托管数据源已被停用。 */ + CSV_DATASOURCE_DISABLED( + "CSV_DATASOURCE_DISABLED", HttpStatus.BAD_REQUEST, "The CSV datasource is disabled."), + + /** CSV 文件格式、编码、内容或大小不符合导入要求。 */ + CSV_FILE_INVALID( + "CSV_FILE_INVALID", + HttpStatus.BAD_REQUEST, + "The CSV file does not meet import requirements."), + + /** CSV 字段定义或字段值与所选类型不匹配。 */ + CSV_SCHEMA_INVALID( + "CSV_SCHEMA_INVALID", + HttpStatus.BAD_REQUEST, + "The CSV column definition or value is invalid."), + + /** CSV 导入记录不存在或已删除。 */ + CSV_TASK_NOT_FOUND("CSV_TASK_NOT_FOUND", HttpStatus.NOT_FOUND, "CSV import task not found."), + + /** CSV 任务当前状态不允许该操作,或并发请求已改变任务状态。 */ + CSV_TASK_STATE_CONFLICT( + "CSV_TASK_STATE_CONFLICT", + HttpStatus.CONFLICT, + "The CSV task state does not allow this operation."), + + /** CSV 表未发布就绪,或不属于当前数据源。 */ + CSV_TABLE_NOT_READY( + "CSV_TABLE_NOT_READY", + HttpStatus.FORBIDDEN, + "The CSV table is not ready or does not belong to this datasource."), + + /** CSV 表仍被关联关系或指标引用,不能删除。 */ + CSV_TABLE_REFERENCED( + "CSV_TABLE_REFERENCED", + HttpStatus.CONFLICT, + "The CSV table is still referenced and cannot be deleted."), + + /** CSV 物理表、上传文件或语义元数据删除失败。 */ + CSV_DELETE_FAILED( + "CSV_DELETE_FAILED", + HttpStatus.INTERNAL_SERVER_ERROR, + "Failed to delete the CSV import. Please try again later."), + /** 读取数据库 schema 失败,通常是数据源连接或元数据读取异常。 */ SCHEMA_READ_FAILED( "SCHEMA_READ_FAILED", diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/exception/ExceptionResponseMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/exception/ExceptionResponseMapper.java index 010661d0..172e9313 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/exception/ExceptionResponseMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/exception/ExceptionResponseMapper.java @@ -27,6 +27,7 @@ import org.springframework.web.HttpMediaTypeNotAcceptableException; import org.springframework.web.HttpMediaTypeNotSupportedException; import org.springframework.web.HttpRequestMethodNotSupportedException; +import org.springframework.web.multipart.MaxUploadSizeExceededException; import org.springframework.web.servlet.resource.NoResourceFoundException; /** 将已知异常统一映射为 ErrorCode 和对外提示,供 HTTP、SSE、agent tool 共用。 */ @@ -53,6 +54,10 @@ private ErrorResponse map(Throwable current) { if (current instanceof AuthenticationException) { return of(ErrorCode.MODEL_AUTHENTICATION_FAILED); } + // Multipart limits are enforced before controller validation runs. + if (current instanceof MaxUploadSizeExceededException) { + return of(ErrorCode.PAYLOAD_TOO_LARGE); + } if (current instanceof IllegalArgumentException) { // 未迁移的裸参数断言统一视为请求参数错误,避免落成 500; // 原始 message 可能含连接串等内部细节,只进日志不进响应体。 @@ -100,11 +105,18 @@ private ErrorResponse fromBusinessException(BusinessException exception) { /** 统一日志策略:5xx 记完整堆栈,4xx 记一行摘要;HTTP、SSE、tool 三出口共用。 */ public void logMapped(Logger logger, Throwable exception, ErrorResponse errorResponse) { + logMapped(logger, exception, errorResponse, "Mapped"); + } + + /** Carries task or operation context while preserving the shared exception logging policy. */ + public void logMapped( + Logger logger, Throwable exception, ErrorResponse errorResponse, String context) { if (errorResponse.isServerError()) { - logger.error("Mapped server exception", exception); + logger.error("{} server exception: {}", context, errorResponse.errorCode(), exception); } else { logger.warn( - "Mapped client error: {} - {}", + "{} client error: {} - {}", + context, errorResponse.errorCode(), errorResponse.message()); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java index bc7818c9..6d348da5 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java @@ -31,6 +31,8 @@ public interface ColumnSemanticInfoMapper { List selectByDatasourceId(@Param("datasourceId") Integer datasourceId); + List selectByDatasourceIdForUpdate(@Param("datasourceId") Integer datasourceId); + List selectByDatasourceIdAndTableName( @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/CsvImportMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/CsvImportMapper.java new file mode 100644 index 00000000..9ef3b248 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/CsvImportMapper.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.mapper; + +import io.github.malonetalk.model.po.CsvImportPo; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface CsvImportMapper extends AuditableMapper {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/RoleHiddenColumnMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/RoleHiddenColumnMapper.java index bf724487..d66fac61 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/RoleHiddenColumnMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/RoleHiddenColumnMapper.java @@ -19,6 +19,7 @@ import io.github.malonetalk.model.po.RoleHiddenColumn; import java.util.List; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @@ -39,4 +40,10 @@ List selectByRoleAndDatasource( @Param("roleId") Integer roleId, @Param("datasourceId") Integer datasourceId); int deleteByRoleId(@Param("roleId") Integer roleId); + + @Delete( + "DELETE FROM role_hidden_column WHERE datasource_id = #{datasourceId} AND table_name =" + + " #{tableName}") + int deleteImportedTable( + @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/RoleTablePermissionMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/RoleTablePermissionMapper.java index d4e18275..67b9075a 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/RoleTablePermissionMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/RoleTablePermissionMapper.java @@ -19,6 +19,7 @@ import io.github.malonetalk.model.po.RoleTablePermission; import java.util.List; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @@ -39,4 +40,10 @@ int deleteByRoleAndDatasource( @Param("roleId") Integer roleId, @Param("datasourceId") Integer datasourceId); int deleteByRoleId(@Param("roleId") Integer roleId); + + @Delete( + "DELETE FROM role_table_permission WHERE datasource_id = #{datasourceId} AND table_name" + + " = #{tableName}") + int deleteImportedTable( + @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java index 5ca5eecb..ca2797e3 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java @@ -22,6 +22,7 @@ import io.github.malonetalk.entity.TableInfo; import java.time.LocalDateTime; import java.util.List; +import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @@ -45,6 +46,8 @@ int deletePhysicalMissingByIds( List selectByDatasourceId(@Param("datasourceId") Integer datasourceId); + List selectByDatasourceIdForUpdate(@Param("datasourceId") Integer datasourceId); + IPage selectPageByDatasourceId( IPage page, @Param("query") TableSemanticPageQuery query, @@ -65,10 +68,18 @@ int markPhysicalMissingByDatasourceIdAndTableNames( @Param("tableNames") List tableNames, @Param("now") LocalDateTime now); + int markPhysicalMissingByIds(@Param("ids") List ids, @Param("now") LocalDateTime now); + int countByDomain(@Param("domain") String domain); int updateDomain( @Param("oldDomain") String oldDomain, @Param("newDomain") String newDomain, @Param("now") LocalDateTime now); + + @Delete( + "DELETE FROM table_info WHERE datasource_id = #{datasourceId} AND table_name =" + + " #{tableName}") + int deleteImportedTable( + @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/bo/ColumnSyncSourceBo.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/bo/ColumnSyncSourceBo.java new file mode 100644 index 00000000..9a7af6b4 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/bo/ColumnSyncSourceBo.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.bo; + +/** Physical column attributes used when comparing and applying semantic schema snapshots. */ +public record ColumnSyncSourceBo( + String columnName, + String description, + String typeName, + Boolean primaryKey, + String indexInfo) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/bo/CsvPreviewBo.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/bo/CsvPreviewBo.java new file mode 100644 index 00000000..41f8b6c1 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/bo/CsvPreviewBo.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.bo; + +import io.github.malonetalk.model.dto.CsvColumnDto; +import java.util.List; + +public record CsvPreviewBo(List columns, List> rows) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/bo/TableSyncSourceBo.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/bo/TableSyncSourceBo.java new file mode 100644 index 00000000..1bc035a0 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/bo/TableSyncSourceBo.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.bo; + +import java.util.List; + +/** Physical table snapshot shared by schema conversion, CSV publication and semantic sync. */ +public record TableSyncSourceBo( + String tableName, String description, List columns) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/converter/CsvImportConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/converter/CsvImportConverter.java new file mode 100644 index 00000000..22493d82 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/converter/CsvImportConverter.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.converter; + +import io.github.malonetalk.common.CsvLimits; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.model.bo.CsvPreviewBo; +import io.github.malonetalk.model.dto.CsvColumnDto; +import io.github.malonetalk.model.dto.CsvParseOptions; +import io.github.malonetalk.model.po.CsvImportPo; +import io.github.malonetalk.model.vo.CsvImportVo; +import io.github.malonetalk.utils.CsvPhysicalNames; +import io.github.malonetalk.utils.JsonUtil; +import java.util.List; +import org.mapstruct.BeanMapping; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.Named; +import org.mapstruct.ReportingPolicy; +import org.springframework.beans.factory.annotation.Autowired; +import tools.jackson.core.type.TypeReference; + +@Mapper( + componentModel = "spring", + imports = {CsvPhysicalNames.class, CsvLimits.class}, + unmappedTargetPolicy = ReportingPolicy.ERROR) +public abstract class CsvImportConverter { + @Autowired private JsonUtil jsonUtil; + + @Mapping(target = "columns", source = "task.columnsJson") + @Mapping(target = "options", source = "task.parseOptionsJson") + public abstract CsvImportVo toVo(CsvImportPo task, List> preview); + + // Map upload metadata only; the service owns the initial status and retention policy. + @BeanMapping(ignoreByDefault = true) + @Mapping(target = "id", source = "id") + @Mapping(target = "datasourceId", source = "datasourceId") + @Mapping(target = "originalFilename", source = "filename") + @Mapping( + target = "displayName", + expression = + "java(filename.substring(0, Math.min(filename.length() - 4," + + " CsvLimits.MAX_NAME_LENGTH)))") + @Mapping(target = "description", constant = "") + @Mapping(target = "physicalTableName", expression = "java(CsvPhysicalNames.table(id))") + @Mapping(target = "attemptId", constant = "") + @Mapping(target = "fileSha256", source = "hash") + @Mapping(target = "fileSize", source = "size") + @Mapping(target = "parseOptionsJson", source = "options", qualifiedByName = "csvJson") + @Mapping(target = "columnsJson", source = "preview.columns", qualifiedByName = "csvJson") + @Mapping(target = "processedRows", constant = "0L") + @Mapping(target = "rowCount", constant = "0L") + public abstract CsvImportPo previewTask( + String id, + Integer datasourceId, + String filename, + String hash, + long size, + CsvParseOptions options, + CsvPreviewBo preview); + + // Corrupt persisted metadata remains an internal failure, not a user-facing business error. + @Named("csvJson") + public String json(Object value) { + try { + return jsonUtil.toJson(value); + } catch (BusinessException e) { + throw new IllegalStateException("Cannot serialize CSV metadata", e.getCause()); + } + } + + public List columns(String json) { + try { + return jsonUtil.toObj(json, new TypeReference<>() {}); + } catch (BusinessException e) { + throw new IllegalStateException("Cannot read CSV columns", e.getCause()); + } + } + + public CsvParseOptions options(String json) { + try { + return jsonUtil.toObj(json, new TypeReference<>() {}); + } catch (BusinessException e) { + throw new IllegalStateException("Cannot read CSV options", e.getCause()); + } + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/converter/CsvSchemaConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/converter/CsvSchemaConverter.java new file mode 100644 index 00000000..d69333ed --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/converter/CsvSchemaConverter.java @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.converter; + +import static io.github.malonetalk.common.CsvLimits.MAX_COLUMNS; +import static io.github.malonetalk.common.CsvLimits.MAX_NAME_LENGTH; + +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.model.dto.CsvColumnDto; +import io.github.malonetalk.utils.DateTimeUtils; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.springframework.stereotype.Component; + +@Component +public class CsvSchemaConverter { + public void validate(List columns, int expected) { + if (columns.size() != expected || expected < 1 || expected > MAX_COLUMNS) { + throw BusinessException.of( + ErrorCode.CSV_SCHEMA_INVALID, "字段数量与文件不一致,最多允许 " + MAX_COLUMNS + " 列。"); + } + int rowBytes = 0; + for (int i = 0; i < columns.size(); i++) { + CsvColumnDto column = columns.get(i); + if (column.index() != i + || column.name() == null + || column.name().isBlank() + || column.name().length() > MAX_NAME_LENGTH) { + throw BusinessException.of( + ErrorCode.CSV_SCHEMA_INVALID, + "字段必须按原文件顺序提交,并填写不超过 " + MAX_NAME_LENGTH + " 字的名称。"); + } + sqlType(column); + rowBytes += "VARCHAR".equals(column.type()) ? column.length() * 4 + 2 : 32; + } + if (rowBytes > 60000) + throw BusinessException.of( + ErrorCode.CSV_SCHEMA_INVALID, "VARCHAR 字段总长度过大,请将部分字段改为 TEXT。"); + } + + public String sqlType(CsvColumnDto column) { + return switch (column.type()) { + case "BIGINT", "DATE", "DATETIME", "TEXT" -> column.type(); + case "BOOLEAN" -> "TINYINT(1)"; + case "VARCHAR" -> { + if (column.length() == null || column.length() < 1 || column.length() > 4000) + throw BusinessException.of( + ErrorCode.CSV_SCHEMA_INVALID, "VARCHAR 长度必须在 1 到 4000 之间。"); + yield "VARCHAR(" + column.length() + ")"; + } + case "DECIMAL" -> { + if (column.precision() == null + || column.scale() == null + || column.precision() < 1 + || column.precision() > 65 + || column.scale() < 0 + || column.scale() > 30 + || column.scale() > column.precision()) + throw BusinessException.of(ErrorCode.CSV_SCHEMA_INVALID, "DECIMAL 精度或小数位数无效。"); + yield "DECIMAL(" + column.precision() + "," + column.scale() + ")"; + } + default -> throw BusinessException.of(ErrorCode.CSV_SCHEMA_INVALID, "不支持的字段类型。"); + }; + } + + public Optional value(String raw, CsvColumnDto column, boolean emptyAsNull) { + if (raw.isEmpty() && emptyAsNull) return Optional.empty(); + String text = raw.strip(); + Object value = + switch (column.type()) { + case "VARCHAR" -> { + if (raw.codePointCount(0, raw.length()) > column.length()) + throw BusinessException.of( + ErrorCode.CSV_SCHEMA_INVALID, + "文本超过 VARCHAR 长度,请改为 TEXT 或增加长度。"); + yield raw; + } + case "TEXT" -> raw; + case "BIGINT" -> { + if (!text.matches("[+-]?[0-9]+")) + throw BusinessException.of(ErrorCode.CSV_SCHEMA_INVALID, "不是有效整数。"); + yield Long.valueOf(text); + } + case "DECIMAL" -> { + BigDecimal number = + new BigDecimal(text) + .setScale(column.scale(), RoundingMode.UNNECESSARY); + if (number.precision() > column.precision()) + throw BusinessException.of( + ErrorCode.CSV_SCHEMA_INVALID, "数值超过 DECIMAL 精度。"); + yield number; + } + case "BOOLEAN" -> { + if (!Set.of("true", "false").contains(text.toLowerCase(Locale.ROOT))) + throw BusinessException.of( + ErrorCode.CSV_SCHEMA_INVALID, "布尔值必须为 true 或 false。"); + yield Boolean.parseBoolean(text); + } + case "DATE" -> DateTimeUtils.parseMysqlDate(text); + case "DATETIME" -> DateTimeUtils.parseMysqlDateTime(text); + default -> + throw BusinessException.of(ErrorCode.CSV_SCHEMA_INVALID, "不支持的字段类型。"); + }; + return Optional.of(value); + } + + public CsvColumnDto infer(int index, String name, List values, boolean emptyAsNull) { + List present = + values.stream() + .filter(s -> !emptyAsNull || !s.isEmpty()) + .map(String::strip) + .toList(); + String type = "VARCHAR"; + int scale = 0; + if (!present.isEmpty()) { + Set kinds = present.stream().map(this::kind).collect(Collectors.toSet()); + if (kinds.size() == 1) type = kinds.iterator().next(); + else if (Set.of("BIGINT", "DECIMAL").containsAll(kinds)) type = "DECIMAL"; + if ("DECIMAL".equals(type)) { + scale = + present.stream() + .map(BigDecimal::new) + .mapToInt(n -> Math.max(0, n.scale())) + .max() + .orElse(0); + final int decimalScale = scale; + if (scale > 18 + || present.stream() + .map(BigDecimal::new) + .anyMatch(n -> n.precision() - n.scale() > 38 - decimalScale)) + type = "VARCHAR"; + } + } + if ("VARCHAR".equals(type) && values.stream().anyMatch(s -> s.length() > 255)) + type = "TEXT"; + return CsvColumnDto.builder() + .index(index) + .name(name) + .type(type) + .length(255) + .precision(38) + .scale(scale) + .build(); + } + + private String kind(String text) { + if (Set.of("true", "false").contains(text.toLowerCase(Locale.ROOT))) return "BOOLEAN"; + if (text.matches("[+-]?[0-9]+")) { + String digits = text.replaceFirst("^[+-]", ""); + if (digits.length() > 15 || digits.length() > 1 && digits.startsWith("0")) + return "VARCHAR"; + try { + Long.parseLong(text); + return "BIGINT"; + } catch (NumberFormatException ignored) { + return "VARCHAR"; + } + } + if (text.matches("[+-]?[0-9]+[.][0-9]+")) return "DECIMAL"; + try { + if (text.length() == 10) { + DateTimeUtils.parseMysqlDate(text); + return "DATE"; + } + if (text.length() == 19) { + DateTimeUtils.parseMysqlDateTime(text); + return "DATETIME"; + } + } catch (RuntimeException ignored) { + // Ambiguous or invalid dates remain text. + } + return "VARCHAR"; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/converter/SemanticSyncConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/converter/SemanticSyncConverter.java new file mode 100644 index 00000000..acce85b7 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/converter/SemanticSyncConverter.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.converter; + +import io.github.malonetalk.dto.datasource.PhysicalColumnInfo; +import io.github.malonetalk.entity.ColumnInfo; +import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.model.bo.ColumnSyncSourceBo; +import io.github.malonetalk.model.bo.TableSyncSourceBo; +import io.github.malonetalk.model.po.CsvImportPo; +import io.github.malonetalk.utils.SemanticUtils; +import java.util.List; +import java.util.Optional; +import org.mapstruct.BeanMapping; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.ReportingPolicy; + +/** Builds the same physical schema snapshot for JDBC sync and CSV publication. */ +@Mapper(componentModel = "spring", unmappedTargetPolicy = ReportingPolicy.ERROR) +public abstract class SemanticSyncConverter { + // Business defaults and persistence identities are assigned by the applying service. + @BeanMapping(ignoreByDefault = true) + @Mapping(target = "datasourceId", source = "datasourceId") + @Mapping(target = "tableName", source = "table.tableName") + @Mapping(target = "physicalTableDescription", source = "table.description") + @Mapping(target = "tableDescription", source = "table.description") + public abstract TableInfo toTableInfo(Integer datasourceId, TableSyncSourceBo table); + + @BeanMapping(ignoreByDefault = true) + @Mapping(target = "datasourceId", source = "datasourceId") + @Mapping(target = "tableId", source = "tableId") + @Mapping(target = "columnName", source = "column.columnName") + @Mapping(target = "physicalColumnDescription", source = "column.description") + @Mapping(target = "columnDescription", source = "column.description") + @Mapping(target = "typeName", source = "column.typeName") + @Mapping(target = "primaryKey", source = "column.primaryKey") + @Mapping(target = "indexInfo", source = "column.indexInfo") + public abstract ColumnInfo toColumnInfo( + Integer datasourceId, Integer tableId, ColumnSyncSourceBo column); + + public TableSyncSourceBo toTable( + String name, String description, List columns) { + return new TableSyncSourceBo( + name, + SemanticUtils.trimToNull(description), + columns.stream().map(this::toColumn).toList()); + } + + public TableSyncSourceBo toCsvTable(CsvImportPo task, List columns) { + String description = + Optional.ofNullable(task.getDescription()) + .filter(text -> !text.isBlank()) + .map(text -> task.getDisplayName() + ":" + text) + .orElse(task.getDisplayName()); + return toTable( + task.getPhysicalTableName(), + description.substring(0, Math.min(500, description.length())), + columns); + } + + private ColumnSyncSourceBo toColumn(PhysicalColumnInfo column) { + return new ColumnSyncSourceBo( + SemanticUtils.requireTrimmed(column.columnName(), "Missing physical columnName."), + SemanticUtils.trimToNull(column.remarks()), + column.formattedTypeName(), + column.primaryKey(), + column.formattedIndexInfo()); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/dto/CsvColumnDto.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/dto/CsvColumnDto.java new file mode 100644 index 00000000..aff237d0 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/dto/CsvColumnDto.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.dto; + +import static io.github.malonetalk.common.CsvLimits.MAX_NAME_LENGTH; + +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Size; +import lombok.Builder; + +@Builder +public record CsvColumnDto( + @Min(0) @Max(99) int index, + @NotBlank @Size(max = MAX_NAME_LENGTH) String name, + @NotBlank String type, + Integer length, + Integer precision, + Integer scale) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/dto/CsvImportConfirmDto.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/dto/CsvImportConfirmDto.java new file mode 100644 index 00000000..3997fa4d --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/dto/CsvImportConfirmDto.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.dto; + +import static io.github.malonetalk.common.CsvLimits.MAX_COLUMNS; +import static io.github.malonetalk.common.CsvLimits.MAX_NAME_LENGTH; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; +import java.util.List; +import lombok.Builder; + +@Builder +public record CsvImportConfirmDto( + @NotBlank @Size(max = MAX_NAME_LENGTH) String displayName, + @Size(max = 400) String description, + @NotEmpty @Size(max = MAX_COLUMNS) List<@NotNull @Valid CsvColumnDto> columns) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/dto/CsvParseOptions.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/dto/CsvParseOptions.java new file mode 100644 index 00000000..b61f4581 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/dto/CsvParseOptions.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.dto; + +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; +import lombok.Builder; + +@Builder +public record CsvParseOptions( + @NotBlank String encoding, + @NotNull @Pattern(regexp = "[,;\\t]") String delimiter, + boolean header, + boolean emptyAsNull) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/po/CsvImportPo.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/po/CsvImportPo.java new file mode 100644 index 00000000..15e271e8 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/po/CsvImportPo.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.po; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import io.github.malonetalk.enums.CsvImportStatus; +import java.time.LocalDateTime; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +@TableName("csv_import") +public class CsvImportPo extends BasePo { + @TableId private String id; + private Integer datasourceId; + private String displayName; + private String description; + private String originalFilename; + private String fileSha256; + private Long fileSize; + private String physicalTableName; + + /** Identifies one execution, including a sync-only retry. */ + private String attemptId; + + private Integer submittedBy; + private String parseOptionsJson; + private String columnsJson; + private CsvImportStatus status; + private Long processedRows; + private Long rowCount; + private String errorMessage; + private LocalDateTime expiresAt; +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/po/DatasourcePo.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/po/DatasourcePo.java index 9fb4a5f5..d794dc2f 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/model/po/DatasourcePo.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/po/DatasourcePo.java @@ -33,6 +33,7 @@ public class DatasourcePo extends BasePo { private String name; private String type; + private String sourceType; private String host; private Integer port; private String databaseName; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/CsvImportPageVo.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/CsvImportPageVo.java new file mode 100644 index 00000000..0a035dee --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/CsvImportPageVo.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.vo; + +import java.util.List; + +public record CsvImportPageVo(List items, long total, int page, int pageSize) + implements BaseVo {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/CsvImportVo.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/CsvImportVo.java new file mode 100644 index 00000000..0e6d2e87 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/CsvImportVo.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.model.vo; + +import io.github.malonetalk.enums.CsvImportStatus; +import io.github.malonetalk.model.dto.CsvColumnDto; +import io.github.malonetalk.model.dto.CsvParseOptions; +import java.time.LocalDateTime; +import java.util.List; +import lombok.Builder; + +@Builder +public record CsvImportVo( + String id, + Integer datasourceId, + String displayName, + String description, + String originalFilename, + String physicalTableName, + CsvImportStatus status, + Long processedRows, + Long rowCount, + String errorMessage, + LocalDateTime createTime, + List columns, + CsvParseOptions options, + List> preview) + implements BaseVo {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/DatasourceVo.java b/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/DatasourceVo.java index 6d77e1ee..54e04a4c 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/DatasourceVo.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/model/vo/DatasourceVo.java @@ -25,6 +25,7 @@ public record DatasourceVo( Integer id, String name, String type, + String sourceType, String host, Integer port, String databaseName, diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/DataPermissionService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/DataPermissionService.java index 746bdbe3..b9ff5419 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/DataPermissionService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/DataPermissionService.java @@ -28,6 +28,7 @@ import io.github.malonetalk.model.bo.UserContextBo; import io.github.malonetalk.model.po.DatasourcePo; import io.github.malonetalk.model.po.RoleTablePermission; +import io.github.malonetalk.service.csv.CsvAccessService; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -43,9 +44,21 @@ public class DataPermissionService { private final RoleTablePermissionMapper tables; private final RoleHiddenColumnMapper columns; private final DatasourceService datasources; + private final CsvAccessService csvAccess; @Transactional(readOnly = true, isolation = Isolation.REPEATABLE_READ) public DataPermissionScopeBo loadScope(UserContextBo user, Integer datasourceId) { + return datasources + .findById(datasourceId) + .map( + datasource -> + csvAccess.restrict( + datasource, loadGrantedScope(user, datasourceId))) + .orElseGet(() -> new DataPermissionScopeBo(false, Map.of(), Map.of())); + } + + /** Role grants for management operations; analysis must use loadScope to enforce CSV readiness. */ + public DataPermissionScopeBo loadGrantedScope(UserContextBo user, Integer datasourceId) { if (Boolean.TRUE.equals(user.superAdmin())) return new DataPermissionScopeBo(true, Map.of(), Map.of()); if (user.roleId() == null || user.roleId() <= 0) @@ -59,23 +72,9 @@ public DataPermissionScopeBo loadScope(UserContextBo user, Integer datasourceId) return new DataPermissionScopeBo(false, granted, hiddenByTable(user, datasourceId)); } - /** - * Returns the granted physical name, never the client-supplied spelling. - */ - public void requireTable(UserContextBo user, Integer datasourceId, String table) { - authorizedTableName(user, datasourceId, table) - .orElseThrow( - () -> BusinessException.of(ErrorCode.FORBIDDEN, "Table access denied.")); - } - - private Optional authorizedTableName( - UserContextBo user, Integer datasourceId, String table) { - if (Boolean.TRUE.equals(user.superAdmin())) return Optional.of(table); - if (user.roleId() == null || user.roleId() <= 0) return Optional.empty(); - return tables.selectByRoleAndDatasource(user.roleId(), datasourceId).stream() - .map(RoleTablePermission::getTableName) - .filter(name -> normalizeIdentifier(name).equals(normalizeIdentifier(table))) - .findFirst(); + /** Returns the granted physical name, never the client-supplied spelling. */ + public String requireTable(UserContextBo user, Integer datasourceId, String table) { + return loadScope(user, datasourceId).requireTable(table); } public boolean canDatasource(UserContextBo user, Integer datasourceId) { diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceServiceImpl.java index bbe20673..76f6b6e4 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceServiceImpl.java @@ -20,6 +20,7 @@ import com.baomidou.mybatisplus.core.toolkit.Wrappers; import io.github.malonetalk.agent.datasource.DataSourceType; import io.github.malonetalk.agent.datasource.DynamicDataSourceManager; +import io.github.malonetalk.enums.DatasourceSourceType; import io.github.malonetalk.enums.Status; import io.github.malonetalk.exception.BusinessException; import io.github.malonetalk.exception.ErrorCode; @@ -59,6 +60,7 @@ public void save(DatasourceDto request) { DataSourceType type = requireDatasourceType(request.type()); DatasourcePo datasource = converter.toPo(request); datasource.setType(type.getCode()); + datasource.setSourceType(DatasourceSourceType.DATABASE.name()); datasource.setStatus(Status.INACTIVE.getCode()); requireOperationSuccess( dataSourceMapper.insert(datasource), "Failed to save the datasource."); @@ -68,6 +70,7 @@ public void save(DatasourceDto request) { public void update(Integer id, DatasourceDto request) { DataSourceType type = requireDatasourceType(request.type()); DatasourcePo datasource = requireById(id); + requireEditable(datasource); converter.update(request, datasource); datasource.setType(type.getCode()); // 与旧接口保持一致:null 不覆盖已有字段,空密码表示继续使用原密码。 @@ -81,7 +84,7 @@ public void update(Integer id, DatasourceDto request) { @Override public void deleteById(Integer id) { - requireById(id); + requireEditable(requireById(id)); requireOperationSuccess( dataSourceMapper.deleteById(id), "Failed to delete the datasource."); dynamicDataSourceManager.removeDataSource(id); @@ -133,6 +136,12 @@ private DataSourceType requireDatasourceType(String type) { "Unsupported datasource type: " + type)); } + private void requireEditable(DatasourcePo datasource) { + if (DatasourceSourceType.CSV.name().equals(datasource.getSourceType())) { + throw BusinessException.of(ErrorCode.BAD_REQUEST, "CSV 托管数据源的连接由管理员配置,不能修改或删除。"); + } + } + private void requireOperationSuccess(int affectedRows, String message) { if (affectedRows == 0) { throw BusinessException.of(ErrorCode.OPERATION_FAILED, message); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceWriteLockService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceWriteLockService.java new file mode 100644 index 00000000..c4a2f66c --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceWriteLockService.java @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.mapper.DatasourceMapper; +import io.github.malonetalk.model.po.DatasourcePo; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +public class DatasourceWriteLockService { + private final DatasourceMapper datasources; + + // ponytail: serialize per datasource; use per-table coordination if contention warrants it. + @Transactional(propagation = Propagation.MANDATORY) + public DatasourcePo lock(Integer datasourceId) { + return Optional.ofNullable( + datasources.selectOne( + Wrappers.lambdaQuery() + .eq(DatasourcePo::getId, datasourceId) + .last("FOR UPDATE"))) + .orElseThrow( + () -> + BusinessException.of( + ErrorCode.RESOURCE_NOT_FOUND, + "Datasource does not exist: " + datasourceId)); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/MetricServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/MetricServiceImpl.java index d8a6778d..50dbc656 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/MetricServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/MetricServiceImpl.java @@ -31,7 +31,9 @@ import io.github.malonetalk.model.dto.MetricCreateDto; import io.github.malonetalk.model.dto.MetricQueryDto; import io.github.malonetalk.model.dto.MetricUpdateDto; +import io.github.malonetalk.model.po.DatasourcePo; import io.github.malonetalk.model.po.MetricInfoPo; +import io.github.malonetalk.service.csv.CsvAccessService; import io.github.malonetalk.utils.SemanticUtils; import java.math.BigDecimal; import java.util.Arrays; @@ -41,10 +43,13 @@ import java.util.Locale; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Isolation; +import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; @Slf4j @@ -59,7 +64,9 @@ public class MetricServiceImpl implements MetricService { private final MetricInfoMapper metricInfoMapper; private final MetricConverter metricConverter; private final MetricPromptConverter metricPromptConverter; - private final DatasourceService datasourceService; + private final DatasourceWriteLockService datasourceLocks; + private final CsvAccessService csvAccess; + private final MetricTableReferenceService metricReferences; @Override public MetricLookup lookupCaliber(Integer datasourceId, String hint) { @@ -129,8 +136,10 @@ boolean matched() { } @Override + @Transactional public MetricInfoBo create(MetricCreateDto dto) { - Integer dsId = requireDatasource(dto.datasourceId()); + DatasourcePo datasource = datasourceLocks.lock(dto.datasourceId()); + Integer dsId = datasource.getId(); String key = SemanticUtils.normalizeObjectName(dto.metricKey(), "指标 key 不能为空"); if (metricInfoMapper.selectByKey(dsId, key) != null) { throw new IllegalArgumentException("指标 key 已存在: " + key); @@ -139,21 +148,32 @@ public MetricInfoBo create(MetricCreateDto dto) { po.setDatasourceId(dsId); po.setMetricKey(key); normalizeBusinessProperties(po); + requireReadyCsvReferences(po, datasource); metricInfoMapper.insert(po); return metricConverter.toBo(po); } @Override + @Transactional(isolation = Isolation.READ_COMMITTED) public MetricInfoBo update(Integer id, MetricUpdateDto dto) { if (id == null) { throw new IllegalArgumentException("id 不能为空"); } - MetricInfoPo po = requireById(id); + // Re-read after locking: another edit may have committed while locating the datasource. + DatasourcePo datasource = datasourceLocks.lock(requireById(id).getDatasourceId()); + MetricInfoPo po = + Optional.ofNullable( + metricInfoMapper.selectOne( + Wrappers.lambdaQuery() + .eq(MetricInfoPo::getId, id) + .last("FOR UPDATE"))) + .orElseThrow(() -> new IllegalArgumentException("指标不存在: id=" + id)); metricConverter.toPoForUpdate(dto, po); normalizeBusinessProperties(po); if (!StringUtils.hasText(dto.name())) { po.setName(null); } + requireReadyCsvReferences(po, datasource); metricInfoMapper.updateById(po); return metricConverter.toBo(po); } @@ -189,11 +209,13 @@ public IPage page(MetricQueryDto dto) { return page.convert(metricConverter::toBo); } - private Integer requireDatasource(Integer datasourceId) { - if (datasourceService.findById(datasourceId).isEmpty()) { - throw BusinessException.of(ErrorCode.BAD_REQUEST, "数据源不存在: " + datasourceId); - } - return datasourceId; + private void requireReadyCsvReferences(MetricInfoPo metric, DatasourcePo datasource) { + if (!csvAccess.managed(datasource)) return; + var readyTables = + csvAccess.ready(datasource).stream() + .map(task -> SemanticUtils.normalizeIdentifier(task.getPhysicalTableName())) + .collect(Collectors.toSet()); + metricReferences.requireReadyTables(metric, readyTables); } private MetricInfoPo requireById(Integer id) { diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/MetricTableReferenceService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/MetricTableReferenceService.java new file mode 100644 index 00000000..d626bc48 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/MetricTableReferenceService.java @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service; + +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.model.po.MetricInfoPo; +import io.github.malonetalk.utils.SemanticUtils; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Stream; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.select.AllTableColumns; +import org.springframework.stereotype.Service; + +/** Conservative expression lineage for deletion and CSV metric reference validation. */ +@Service +public class MetricTableReferenceService { + public boolean mayReference(MetricInfoPo metric, String table, Set knownTables) { + return references(metric, knownTables) + .map(references -> references.contains(SemanticUtils.normalizeIdentifier(table))) + .orElse(true); + } + + public void requireReadyTables(MetricInfoPo metric, Set readyTables) { + if (references(metric, readyTables).isEmpty()) + throw BusinessException.of( + ErrorCode.CSV_TABLE_NOT_READY, "CSV 指标必须明确引用已就绪的表,请使用表名限定字段并检查相关导入任务。"); + } + + private Optional> references(MetricInfoPo metric, Set knownTables) { + if (Optional.ofNullable(metric.getMeasureExpr()) + .filter(value -> !value.isBlank()) + .isEmpty()) return Optional.empty(); + Set references = new HashSet<>(); + Consumer collect = + source -> { + String name = + Optional.ofNullable(source) + .filter(table -> table.getSchemaName() == null) + .filter( + table -> + table.getDatabase() == null + || table.getDatabase().getDatabaseName() + == null) + .map(Table::getName) + .map(SemanticUtils::normalizeIdentifier) + .filter(knownTables::contains) + .orElseThrow( + () -> + new IllegalArgumentException( + "Unknown metric table")); + references.add(name); + }; + var inspector = + new SqlExpressionInspector( + column -> collect.accept(column.getTable()), + select -> { + throw new IllegalArgumentException( + "Metric subquery lineage is unknown"); + }, + all -> { + if (all instanceof AllTableColumns qualified) + collect.accept(qualified.getTable()); + else throw new IllegalArgumentException("Unqualified metric wildcard"); + }); + try { + // Parse fields separately: a qualified filter cannot resolve an unqualified measure. + // String literals and comments are never treated as table references. + for (String expression : + Stream.of(metric.getMeasureExpr(), metric.getFilters(), metric.getTimeField()) + .flatMap(value -> Optional.ofNullable(value).stream()) + .filter(value -> !value.isBlank()) + .toList()) { + inspector.inspect(CCJSqlParserUtil.parseExpression(expression, false)); + } + } catch (Exception e) { + return Optional.empty(); + } + return references.isEmpty() ? Optional.empty() : Optional.of(Set.copyOf(references)); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/SqlExpressionInspector.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/SqlExpressionInspector.java index 47711c2c..1a280b11 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/SqlExpressionInspector.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/SqlExpressionInspector.java @@ -85,7 +85,7 @@ /** Only explicitly supported expression nodes are traversed; every other node fails closed. */ @RequiredArgsConstructor -final class SqlExpressionInspector { +public final class SqlExpressionInspector { private static final Set> LITERALS = Set.of( NullValue.class, @@ -130,8 +130,13 @@ final class SqlExpressionInspector { .split(",")); private final Consumer columns; private final Consumer selects) { + this(columns, selects, all -> {}); + } + + public void inspect(Expression expression) { if (expression == null || LITERALS.contains(expression.getClass())) return; if (expression instanceof Column column) { columns.accept(column); @@ -163,6 +168,7 @@ void inspect(Expression expression) { || qualified.getTable().getDatabase() != null && qualified.getTable().getDatabase().getDatabaseName() != null)) throw denied(); + allColumns.accept(all); } else if (expression instanceof TimeKeyExpression time) { if (!Set.of( "current_date", diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/TableExportServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/TableExportServiceImpl.java index 4f3eacdf..ee7ccaf8 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/TableExportServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/TableExportServiceImpl.java @@ -20,6 +20,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import io.github.malonetalk.agent.SessionService; +import io.github.malonetalk.agent.datasource.DataSourceType; import io.github.malonetalk.agent.datasource.DynamicDataSourceManager; import io.github.malonetalk.agent.datasource.SqlExecutor; import io.github.malonetalk.dto.TableExportPageQuery; @@ -141,7 +142,7 @@ private CsvResult writeCsv(DatasourcePo datasource, String sql) { new OutputStreamWriter(output, StandardCharsets.UTF_8))) { stmt.setQueryTimeout(QUERY_TIMEOUT_SECONDS); stmt.setFetchSize( - "mysql".equalsIgnoreCase(datasource.getType()) + DataSourceType.MYSQL.getCode().equalsIgnoreCase(datasource.getType()) ? Integer.MIN_VALUE : FETCH_SIZE); writer.write('\ufeff'); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvAccessService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvAccessService.java new file mode 100644 index 00000000..ab904e05 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvAccessService.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.csv; + +import static io.github.malonetalk.enums.CsvImportStatus.READY; +import static io.github.malonetalk.enums.CsvImportStatus.SYNCING; +import static io.github.malonetalk.enums.DatasourceSourceType.CSV; +import static io.github.malonetalk.utils.SemanticUtils.normalizeIdentifier; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.mapper.CsvImportMapper; +import io.github.malonetalk.model.bo.DataPermissionScopeBo; +import io.github.malonetalk.model.po.CsvImportPo; +import io.github.malonetalk.model.po.DatasourcePo; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class CsvAccessService { + private final CsvImportMapper imports; + + public boolean managed(DatasourcePo datasource) { + return CSV.name().equals(datasource.getSourceType()); + } + + public List ready(DatasourcePo datasource) { + return imports.selectList( + Wrappers.lambdaQuery() + .eq(CsvImportPo::getDatasourceId, datasource.getId()) + .eq(CsvImportPo::getStatus, READY)); + } + + public DataPermissionScopeBo restrict(DatasourcePo datasource, DataPermissionScopeBo scope) { + if (!managed(datasource)) return scope; + Map tables = + ready(datasource).stream() + .filter(row -> scope.canTable(row.getPhysicalTableName())) + .collect( + Collectors.toMap( + row -> normalizeIdentifier(row.getPhysicalTableName()), + CsvImportPo::getPhysicalTableName)); + return new DataPermissionScopeBo(false, tables, scope.hiddenColumns()); + } + + public Predicate readyFilter(DatasourcePo datasource) { + if (!managed(datasource)) return table -> true; + Set names = + ready(datasource).stream() + .map(row -> normalizeIdentifier(row.getPhysicalTableName())) + .collect(Collectors.toSet()); + return table -> names.contains(normalizeIdentifier(table)); + } + + public boolean isReady(DatasourcePo datasource, String table) { + if (!managed(datasource)) return true; + return imports.selectCount( + Wrappers.lambdaQuery() + .eq(CsvImportPo::getDatasourceId, datasource.getId()) + .eq(CsvImportPo::getPhysicalTableName, table) + .eq(CsvImportPo::getStatus, READY)) + > 0; + } + + public void requireReady(DatasourcePo datasource, String table) { + if (!isReady(datasource, table)) + throw BusinessException.of(ErrorCode.CSV_TABLE_NOT_READY, "CSV 表尚未就绪或不属于当前数据源。"); + } + + public void requireImportTable(DatasourcePo datasource, CsvImportPo task) { + if (!managed(datasource) + || !datasource.getId().equals(task.getDatasourceId()) + || task.getStatus() != SYNCING + || imports.selectCount( + Wrappers.lambdaQuery() + .eq(CsvImportPo::getId, task.getId()) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .eq(CsvImportPo::getStatus, task.getStatus()) + .eq( + CsvImportPo::getPhysicalTableName, + task.getPhysicalTableName())) + != 1) + throw BusinessException.of(ErrorCode.CSV_TASK_STATE_CONFLICT, "CSV 任务不能同步。"); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvDatasourceService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvDatasourceService.java new file mode 100644 index 00000000..e1aa8df8 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvDatasourceService.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.csv; + +import io.github.malonetalk.agent.datasource.DataSourceType; +import io.github.malonetalk.config.CsvImportConfig; +import io.github.malonetalk.enums.DatasourceSourceType; +import io.github.malonetalk.enums.Status; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.mapper.DatasourceMapper; +import io.github.malonetalk.model.po.DatasourcePo; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** Shared validation of the configured CSV datasource for requests and background workers. */ +@Service +@RequiredArgsConstructor +public class CsvDatasourceService { + private final CsvImportConfig config; + private final DatasourceMapper datasources; + + public DatasourcePo requireActive() { + DatasourcePo datasource = + Optional.ofNullable(datasources.selectById(config.getDatasourceId())) + .filter( + ds -> + DatasourceSourceType.CSV.name().equals(ds.getSourceType()) + && DataSourceType.MYSQL + .getCode() + .equals(ds.getType())) + .filter( + ds -> + ds.getDatabaseName() != null + && !ds.getDatabaseName().isBlank()) + .orElseThrow( + () -> + BusinessException.of( + ErrorCode.CSV_NOT_CONFIGURED, + "CSV 导入尚未配置,请联系管理员设置 CSV_DATASOURCE_ID 和导入库连接。")); + if (!Status.ACTIVE.getCode().equals(datasource.getStatus())) + throw BusinessException.of(ErrorCode.CSV_DATASOURCE_DISABLED, "CSV 数据源已禁用。"); + return datasource; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvFileService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvFileService.java new file mode 100644 index 00000000..c6e75a3d --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvFileService.java @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.csv; + +import static io.github.malonetalk.common.CsvLimits.MAX_COLUMNS; +import static io.github.malonetalk.common.CsvLimits.MAX_NAME_LENGTH; + +import io.github.malonetalk.config.CsvImportConfig; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.model.bo.CsvPreviewBo; +import io.github.malonetalk.model.converter.CsvSchemaConverter; +import io.github.malonetalk.model.dto.CsvColumnDto; +import io.github.malonetalk.model.dto.CsvParseOptions; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PushbackReader; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; +import lombok.RequiredArgsConstructor; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +@Service +@RequiredArgsConstructor +public class CsvFileService { + private static final long MAX_FILE_BYTES = 50L * 1024 * 1024; + private static final int MAX_CELL_BYTES = 16 * 1024; + private static final int INFERENCE_ROWS = 1000; + private static final int PREVIEW_ROWS = 20; + private final CsvImportConfig config; + private final CsvSchemaConverter converter; + + public String save(String id, MultipartFile file) throws Exception { + if (file.isEmpty() || file.getSize() > MAX_FILE_BYTES) + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "文件必须在 1 字节到 50 MiB 之间。"); + Files.createDirectories(config.getDirectory()); + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + try (var input = new DigestInputStream(file.getInputStream(), digest); + var output = Files.newOutputStream(path(id))) { + byte[] buffer = new byte[8192]; + long count = 0; + for (int length; (length = input.read(buffer)) != -1; ) { + count += length; + if (count > MAX_FILE_BYTES) + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "文件超过 50 MiB。"); + output.write(buffer, 0, length); + } + } + return HexFormat.of().formatHex(digest.digest()); + } + + public void verify(String id, String expectedHash) throws Exception { + if (Files.size(path(id)) > MAX_FILE_BYTES) + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "文件超过 50 MiB。"); + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + try (var input = new DigestInputStream(Files.newInputStream(path(id)), digest)) { + input.transferTo(OutputStream.nullOutputStream()); + } + if (!expectedHash.equals(HexFormat.of().formatHex(digest.digest()))) + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "上传文件发生变化,请重新上传。"); + } + + public Path path(String id) { + if (!id.matches("[a-f0-9]{32}")) + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "无效的导入 ID。"); + return config.getDirectory().toAbsolutePath().normalize().resolve(id + ".csv"); + } + + public void delete(String id) throws IOException { + Files.deleteIfExists(path(id)); + } + + public CSVParser open(String id, CsvParseOptions options) throws IOException { + if (options.encoding() == null + || options.delimiter() == null + || !Set.of("UTF-8", "GB18030").contains(options.encoding()) + || !Set.of(",", ";", "\t").contains(options.delimiter())) + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "不支持的编码或分隔符。"); + Reader reader = + new InputStreamReader( + Files.newInputStream(path(id)), + Charset.forName(options.encoding()) + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT)); + PushbackReader input = new PushbackReader(reader, 1); + try { + int first = input.read(); + if (first != -1 && first != 0xFEFF) input.unread(first); + return CSVFormat.RFC4180 + .builder() + .setDelimiter(options.delimiter().charAt(0)) + .setIgnoreEmptyLines(true) + .get() + .parse(input); + } catch (Exception e) { + input.close(); + throw e; + } + } + + public CsvPreviewBo preview(String id, CsvParseOptions options) throws IOException { + try (CSVParser parser = open(id, options)) { + var iterator = parser.iterator(); + if (!iterator.hasNext()) + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "CSV 没有数据。"); + CSVRecord first = iterator.next(); + validateAndMeasure(first, first.size()); + List names = new ArrayList<>(); + List> sample = new ArrayList<>(); + for (int i = 0; i < first.size(); i++) { + String name = + options.header() && !first.get(i).isBlank() ? first.get(i) : "字段" + (i + 1); + if (name.length() > MAX_NAME_LENGTH) + throw BusinessException.of( + ErrorCode.CSV_FILE_INVALID, "表头不能超过 " + MAX_NAME_LENGTH + " 字。"); + names.add(name); + } + if (!options.header()) sample.add(first.toList()); + while (sample.size() < INFERENCE_ROWS && iterator.hasNext()) { + CSVRecord record = iterator.next(); + validateAndMeasure(record, names.size()); + sample.add(record.toList()); + } + if (sample.isEmpty()) + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "CSV 只有表头,没有数据记录。"); + List columns = new ArrayList<>(); + for (int i = 0; i < names.size(); i++) { + final int index = i; + columns.add( + converter.infer( + i, + names.get(i), + sample.stream().map(row -> row.get(index)).toList(), + options.emptyAsNull())); + } + return new CsvPreviewBo(columns, sample.stream().limit(PREVIEW_ROWS).toList()); + } catch (IOException | UncheckedIOException e) { + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "无法解析 CSV,请检查文件编码、分隔符和引号格式。"); + } + } + + /** Validates record limits and returns UTF-8 payload bytes for import batch sizing. */ + public long validateAndMeasure(CSVRecord record, int columns) { + if (columns < 1 || columns > MAX_COLUMNS || record.size() != columns) + throw BusinessException.of( + ErrorCode.CSV_FILE_INVALID, + "第 " + record.getRecordNumber() + " 条记录列数不一致,最多允许 " + MAX_COLUMNS + " 列。"); + long bytes = 0; + for (int i = 0; i < record.size(); i++) { + int cellBytes = record.get(i).getBytes(StandardCharsets.UTF_8).length; + if (cellBytes > MAX_CELL_BYTES) + throw BusinessException.of( + ErrorCode.CSV_FILE_INVALID, + "第 " + record.getRecordNumber() + " 条记录,第 " + (i + 1) + " 列超过 16 KiB。"); + bytes += cellBytes; + } + return bytes; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportCleanupService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportCleanupService.java new file mode 100644 index 00000000..035d0c97 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportCleanupService.java @@ -0,0 +1,223 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.csv; + +import static io.github.malonetalk.enums.CsvImportStatus.CLEANING; +import static io.github.malonetalk.enums.CsvImportStatus.CLEANUP_FAILED; +import static io.github.malonetalk.enums.CsvImportStatus.DELETE_FAILED; +import static io.github.malonetalk.enums.CsvImportStatus.DELETING; +import static io.github.malonetalk.enums.CsvImportStatus.EXPIRED; +import static io.github.malonetalk.enums.CsvImportStatus.IMPORTING; +import static io.github.malonetalk.enums.CsvImportStatus.IMPORT_FAILED; +import static io.github.malonetalk.enums.CsvImportStatus.PREVIEWED; +import static io.github.malonetalk.enums.CsvImportStatus.READY; +import static io.github.malonetalk.enums.CsvImportStatus.SYNCING; +import static io.github.malonetalk.enums.CsvImportStatus.SYNC_FAILED; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import io.github.malonetalk.config.CsvImportConfig; +import io.github.malonetalk.enums.CsvImportStatus; +import io.github.malonetalk.exception.ExceptionResponseMapper; +import io.github.malonetalk.mapper.CsvImportMapper; +import io.github.malonetalk.model.po.CsvImportPo; +import java.nio.file.Files; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class CsvImportCleanupService { + private final CsvImportConfig config; + private final CsvImportMapper imports; + private final CsvDatasourceService datasources; + private final CsvTableImportService tables; + private final CsvFileService files; + private final ExceptionResponseMapper exceptionResponses; + + public void discardStaleTable(CsvImportPo task) { + try { + // Each data retry has its own table, so this cannot drop the new attempt's data. + tables.drop(task, datasources.requireActive()); + } catch (Exception e) { + logTaskFailure(task, "stale table cleanup", e); + } + } + + public void cleanupTable(CsvImportPo task) { + try { + tables.drop(task, datasources.requireActive()); + transition(task, CLEANING, IMPORT_FAILED); + } catch (Exception e) { + logTaskFailure(task, "table cleanup", e); + transition(task, CLEANING, CLEANUP_FAILED); + } + } + + public void removeFile(CsvImportPo task) { + try { + files.delete(task.getId()); + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, task.getId()) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .set(CsvImportPo::getExpiresAt, null)); + } catch (Exception e) { + logTaskFailure(task, "file cleanup", e); + } + } + + @Scheduled(fixedDelay = 60000, initialDelay = 60000) + public void cleanup() { + if (config.getDatasourceId() <= 0) return; + LocalDateTime now = LocalDateTime.now(); + recoverTimedOutTasks(now); + for (CsvImportPo task : cleanupCandidates(now)) { + try { + cleanupTask(task, now); + } catch (Exception e) { + logTaskFailure(task, "recovery", e); + } + } + cleanupOrphanFiles(); + cleanupOrphanTables(); + } + + private void recoverTimedOutTasks(LocalDateTime now) { + // Ten minutes is a recovery threshold, not proof that the old worker has stopped. + // Attempt-guarded updates and distinct retry tables fence workers that later resume. + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getDatasourceId, config.getDatasourceId()) + .in(CsvImportPo::getStatus, IMPORTING, CLEANING) + .lt(CsvImportPo::getUpdateTime, now.minusMinutes(10)) + .set(CsvImportPo::getStatus, CLEANUP_FAILED) + .set(CsvImportPo::getErrorMessage, "导入任务中断,请重新确认导入。")); + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getDatasourceId, config.getDatasourceId()) + .eq(CsvImportPo::getStatus, SYNCING) + .lt(CsvImportPo::getUpdateTime, now.minusMinutes(10)) + .set(CsvImportPo::getStatus, SYNC_FAILED) + .set(CsvImportPo::getErrorMessage, "语义同步中断,请重试同步。")); + } + + private List cleanupCandidates(LocalDateTime now) { + var query = + Wrappers.lambdaQuery() + .eq(CsvImportPo::getDatasourceId, config.getDatasourceId()); + query.and( + filter -> + filter.in(CsvImportPo::getStatus, CLEANUP_FAILED, DELETING) + .or() + .in( + CsvImportPo::getStatus, + PREVIEWED, + IMPORT_FAILED, + READY, + EXPIRED) + .isNotNull(CsvImportPo::getExpiresAt) + .lt(CsvImportPo::getExpiresAt, now)); + return imports.selectList(query.orderByAsc(CsvImportPo::getUpdateTime).last("LIMIT 100")); + } + + private void cleanupTask(CsvImportPo task, LocalDateTime now) { + if (task.getStatus() == CLEANUP_FAILED && transition(task, CLEANUP_FAILED, CLEANING)) + cleanupTable(task); + if (task.getStatus() == DELETING && task.getUpdateTime().isBefore(now.minusMinutes(10))) + transition(task, DELETING, DELETE_FAILED); + // A cleaned failed import can also expire during this pass. + if (task.getStatus() == READY || task.getStatus() == EXPIRED) removeFile(task); + else if ((task.getStatus() == PREVIEWED || task.getStatus() == IMPORT_FAILED) + && task.getExpiresAt().isBefore(now)) { + if (transition(task, task.getStatus(), EXPIRED)) removeFile(task); + } + } + + private void cleanupOrphanTables() { + try { + tables.dropOrphanTables( + datasources.requireActive(), + table -> + imports.selectCount( + Wrappers.lambdaQuery() + .eq(CsvImportPo::getPhysicalTableName, table)) + > 0); + } catch (Exception e) { + logDatasourceFailure("orphan table cleanup", e); + } + } + + private void cleanupOrphanFiles() { + if (!Files.isDirectory(config.getDirectory())) return; + Instant expiresBefore = Instant.now().minus(1, ChronoUnit.DAYS); + try (var paths = Files.list(config.getDirectory())) { + var candidates = + paths.filter( + path -> + path.getFileName() + .toString() + .matches("[a-f0-9]{32}[.]csv")) + .iterator(); + while (candidates.hasNext()) { + var path = candidates.next(); + String id = path.getFileName().toString().substring(0, 32); + if (Files.getLastModifiedTime(path).toInstant().isBefore(expiresBefore) + && Optional.ofNullable(imports.selectById(id)).isEmpty()) files.delete(id); + } + } catch (Exception e) { + logDatasourceFailure("orphan file cleanup", e); + } + } + + private boolean transition(CsvImportPo task, CsvImportStatus from, CsvImportStatus to) { + int changed = + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, task.getId()) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .eq(CsvImportPo::getStatus, from) + .set(CsvImportPo::getStatus, to)); + if (changed == 1) task.setStatus(to); + return changed == 1; + } + + private void logTaskFailure(CsvImportPo task, String phase, Exception error) { + String context = + "CSV %s [taskId=%s, attemptId=%s, status=%s, table=%s]" + .formatted( + phase, + task.getId(), + task.getAttemptId(), + task.getStatus(), + task.getPhysicalTableName()); + exceptionResponses.logMapped(log, error, exceptionResponses.resolve(error), context); + } + + private void logDatasourceFailure(String phase, Exception error) { + String context = "CSV %s [datasourceId=%s]".formatted(phase, config.getDatasourceId()); + exceptionResponses.logMapped(log, error, exceptionResponses.resolve(error), context); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportDeleteService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportDeleteService.java new file mode 100644 index 00000000..7fbed0be --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportDeleteService.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.csv; + +import static io.github.malonetalk.enums.CsvImportStatus.CLEANING; +import static io.github.malonetalk.enums.CsvImportStatus.DELETING; +import static io.github.malonetalk.enums.CsvImportStatus.IMPORTING; +import static io.github.malonetalk.enums.CsvImportStatus.QUEUED; +import static io.github.malonetalk.enums.CsvImportStatus.SYNCING; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.mapper.CsvImportMapper; +import io.github.malonetalk.mapper.LogicalTableRelationMapper; +import io.github.malonetalk.mapper.MetricInfoMapper; +import io.github.malonetalk.mapper.RoleHiddenColumnMapper; +import io.github.malonetalk.mapper.RoleTablePermissionMapper; +import io.github.malonetalk.mapper.TableInfoMapper; +import io.github.malonetalk.model.po.CsvImportPo; +import io.github.malonetalk.service.DatasourceWriteLockService; +import io.github.malonetalk.service.MetricTableReferenceService; +import io.github.malonetalk.utils.SemanticUtils; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Isolation; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +public class CsvImportDeleteService { + private final CsvImportMapper imports; + private final TableInfoMapper tables; + private final LogicalTableRelationMapper relations; + private final MetricInfoMapper metrics; + private final RoleTablePermissionMapper tablePermissions; + private final RoleHiddenColumnMapper hiddenColumns; + private final MetricTableReferenceService metricReferences; + private final DatasourceWriteLockService datasourceLocks; + + @Transactional(isolation = Isolation.READ_COMMITTED) + public CsvImportPo begin(String id) { + // Resolve the immutable datasource before locking; subsequent reads must see its winner. + Integer datasourceId = + Optional.ofNullable(imports.selectById(id)) + .map(CsvImportPo::getDatasourceId) + .orElseThrow( + () -> + BusinessException.of( + ErrorCode.CSV_TASK_NOT_FOUND, "导入记录不存在。")); + datasourceLocks.lock(datasourceId); + CsvImportPo task = + Optional.ofNullable( + imports.selectOne( + Wrappers.lambdaQuery() + .eq(CsvImportPo::getId, id) + .last("FOR UPDATE"))) + .orElseThrow( + () -> + BusinessException.of( + ErrorCode.CSV_TASK_NOT_FOUND, "导入记录不存在。")); + if (Set.of(QUEUED, IMPORTING, SYNCING, CLEANING, DELETING).contains(task.getStatus())) + throw BusinessException.of(ErrorCode.CSV_TASK_STATE_CONFLICT, "任务正在运行,请完成后再删除。"); + requireUnreferenced(task); + // Return the locked pre-delete snapshot for physical-table cleanup. + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, id) + .set(CsvImportPo::getStatus, DELETING)); + return task; + } + + private void requireUnreferenced(CsvImportPo task) { + var table = + Optional.ofNullable( + tables.selectByDatasourceIdAndTableName( + task.getDatasourceId(), task.getPhysicalTableName())); + if (table.isEmpty()) return; + boolean linked = + relations.selectByDatasourceId(task.getDatasourceId()).stream() + .anyMatch( + relation -> + table.get().getId().equals(relation.getSourceTableId()) + || table.get() + .getId() + .equals(relation.getTargetTableId())); + if (linked) + throw BusinessException.of(ErrorCode.CSV_TABLE_REFERENCED, "该表仍有关联关系,请先删除关联关系。"); + Set knownTables = + tables.selectByDatasourceId(task.getDatasourceId()).stream() + .map(row -> SemanticUtils.normalizeIdentifier(row.getTableName())) + .collect(Collectors.toSet()); + boolean referenced = + metrics.selectByDatasource(task.getDatasourceId()).stream() + .anyMatch( + metric -> + metricReferences.mayReference( + metric, task.getPhysicalTableName(), knownTables)); + if (referenced) + throw BusinessException.of( + ErrorCode.CSV_TABLE_REFERENCED, "该表存在指标引用,或数据源含无法确定归属的指标,请先调整指标口径。"); + } + + @Transactional + public void finish(CsvImportPo task) { + datasourceLocks.lock(task.getDatasourceId()); + Optional.ofNullable( + imports.selectOne( + Wrappers.lambdaQuery() + .eq(CsvImportPo::getId, task.getId()) + .last("FOR UPDATE"))) + .filter(current -> current.getStatus() == DELETING) + .orElseThrow( + () -> + BusinessException.of( + ErrorCode.CSV_TASK_STATE_CONFLICT, "删除任务状态已变化。")); + tablePermissions.deleteImportedTable(task.getDatasourceId(), task.getPhysicalTableName()); + hiddenColumns.deleteImportedTable(task.getDatasourceId(), task.getPhysicalTableName()); + tables.deleteImportedTable(task.getDatasourceId(), task.getPhysicalTableName()); + imports.deleteById(task.getId()); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportPublishService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportPublishService.java new file mode 100644 index 00000000..087eadba --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportPublishService.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.csv; + +import static io.github.malonetalk.enums.CsvImportStatus.READY; +import static io.github.malonetalk.enums.CsvImportStatus.SYNCING; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import io.github.malonetalk.agent.datasource.SchemaReader; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.mapper.CsvImportMapper; +import io.github.malonetalk.model.bo.TableSyncSourceBo; +import io.github.malonetalk.model.converter.SemanticSyncConverter; +import io.github.malonetalk.model.po.CsvImportPo; +import io.github.malonetalk.model.po.DatasourcePo; +import io.github.malonetalk.service.DatasourceWriteLockService; +import io.github.malonetalk.service.semantic.sync.SemanticSyncApplyService; +import java.util.List; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +public class CsvImportPublishService { + private final CsvImportMapper imports; + private final SchemaReader schemaReader; + private final SemanticSyncApplyService semanticSync; + private final SemanticSyncConverter converter; + private final DatasourceWriteLockService datasourceLocks; + + public TableSyncSourceBo snapshot(CsvImportPo task, DatasourcePo datasource) { + // Read the external database before opening the metadata publication transaction. + var columns = schemaReader.getImportTableSchema(datasource, task); + if (columns.isEmpty()) + throw BusinessException.of(ErrorCode.CSV_SCHEMA_INVALID, "导入表没有可同步的字段。"); + return converter.toCsvTable(task, columns); + } + + @Transactional + public void publish(CsvImportPo task, TableSyncSourceBo snapshot) { + datasourceLocks.lock(task.getDatasourceId()); + Optional.ofNullable( + imports.selectOne( + Wrappers.lambdaQuery() + .eq(CsvImportPo::getId, task.getId()) + .last("FOR UPDATE"))) + .filter( + current -> + current.getStatus() == SYNCING + && task.getAttemptId().equals(current.getAttemptId()) + && task.getPhysicalTableName() + .equals(current.getPhysicalTableName())) + .orElseThrow( + () -> + BusinessException.of( + ErrorCode.CSV_TASK_STATE_CONFLICT, "CSV 任务状态已变化,请刷新后重试。")); + // Semantic metadata and READY become visible in one business-database transaction. + semanticSync.applyTableSync(task.getDatasourceId(), List.of(snapshot), List.of()); + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, task.getId()) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .set(CsvImportPo::getStatus, READY) + .set(CsvImportPo::getErrorMessage, "")); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportService.java new file mode 100644 index 00000000..e0e3cffb --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportService.java @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.csv; + +import static io.github.malonetalk.enums.CsvImportStatus.DATA_READY; +import static io.github.malonetalk.enums.CsvImportStatus.DELETE_FAILED; +import static io.github.malonetalk.enums.CsvImportStatus.DELETING; +import static io.github.malonetalk.enums.CsvImportStatus.EXPIRED; +import static io.github.malonetalk.enums.CsvImportStatus.IMPORTING; +import static io.github.malonetalk.enums.CsvImportStatus.IMPORT_FAILED; +import static io.github.malonetalk.enums.CsvImportStatus.PREVIEWED; +import static io.github.malonetalk.enums.CsvImportStatus.QUEUED; +import static io.github.malonetalk.enums.CsvImportStatus.READY; +import static io.github.malonetalk.enums.CsvImportStatus.SYNCING; +import static io.github.malonetalk.enums.CsvImportStatus.SYNC_FAILED; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.github.malonetalk.config.CsvImportConfig; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.mapper.CsvImportMapper; +import io.github.malonetalk.model.converter.CsvImportConverter; +import io.github.malonetalk.model.converter.CsvSchemaConverter; +import io.github.malonetalk.model.dto.CsvImportConfirmDto; +import io.github.malonetalk.model.dto.CsvParseOptions; +import io.github.malonetalk.model.holder.UserContextHolder; +import io.github.malonetalk.model.po.CsvImportPo; +import io.github.malonetalk.model.po.DatasourcePo; +import io.github.malonetalk.model.vo.CsvImportPageVo; +import io.github.malonetalk.model.vo.CsvImportVo; +import io.github.malonetalk.service.DataPermissionService; +import io.github.malonetalk.utils.CsvPhysicalNames; +import java.nio.file.Files; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +@Service +@RequiredArgsConstructor +public class CsvImportService { + private final CsvImportConfig config; + private final CsvImportMapper imports; + private final CsvDatasourceService datasources; + private final CsvFileService files; + private final CsvSchemaConverter schema; + private final CsvImportConverter converter; + private final UserContextHolder holder; + private final DataPermissionService dataPermissions; + private final CsvImportDeleteService deletion; + private final CsvTableImportService tables; + + public CsvImportVo preview(MultipartFile file, CsvParseOptions options) throws Exception { + DatasourcePo datasource = datasources.requireActive(); + String filename = Optional.ofNullable(file.getOriginalFilename()).orElse(""); + if (!filename.toLowerCase(Locale.ROOT).endsWith(".csv") || filename.length() > 255) + throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "请选择文件名不超过 255 字的 CSV 文件。"); + String id = UUID.randomUUID().toString().replace("-", ""); + try { + String hash = files.save(id, file); + var preview = files.preview(id, options); + CsvImportPo task = + converter.previewTask( + id, + datasource.getId(), + filename, + hash, + file.getSize(), + options, + preview); + task.setStatus(PREVIEWED); + task.setExpiresAt(LocalDateTime.now().plusHours(24)); + imports.insert(task); + return converter.toVo(task, preview.rows()); + } catch (Exception e) { + files.delete(id); + throw e; + } + } + + public CsvImportVo repreview(String id, CsvParseOptions options) throws Exception { + CsvImportPo task = accessible(id); + requirePreviewable(task); + var preview = files.preview(id, options); + int updated = + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, id) + .in(CsvImportPo::getStatus, PREVIEWED, IMPORT_FAILED) + .set(CsvImportPo::getParseOptionsJson, converter.json(options)) + .set( + CsvImportPo::getColumnsJson, + converter.json(preview.columns()))); + if (updated != 1) + throw BusinessException.of(ErrorCode.CSV_TASK_STATE_CONFLICT, "任务状态已变化,请刷新后重试。"); + return converter.toVo(accessible(id), preview.rows()); + } + + public CsvImportVo confirm(String id, CsvImportConfirmDto request) { + datasources.requireActive(); + CsvImportPo task = accessible(id); + var submitted = Set.of(QUEUED, IMPORTING, DATA_READY, SYNCING, READY, SYNC_FAILED); + if (submitted.contains(task.getStatus())) return converter.toVo(task, List.of()); + requirePreviewable(task); + schema.validate(request.columns(), converter.columns(task.getColumnsJson()).size()); + String attemptId = UUID.randomUUID().toString().replace("-", ""); + int updated = + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, id) + .in(CsvImportPo::getStatus, PREVIEWED, IMPORT_FAILED) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .set(CsvImportPo::getDisplayName, request.displayName()) + .set( + CsvImportPo::getDescription, + Optional.ofNullable(request.description()).orElse("")) + .set(CsvImportPo::getColumnsJson, converter.json(request.columns())) + .set(CsvImportPo::getStatus, QUEUED) + // A new table isolates a resumed worker from the retry's data. + .set( + CsvImportPo::getPhysicalTableName, + CsvPhysicalNames.table(attemptId)) + .set(CsvImportPo::getAttemptId, attemptId) + .set(CsvImportPo::getSubmittedBy, holder.checkAndGet().userId()) + .set(CsvImportPo::getProcessedRows, 0) + .set(CsvImportPo::getRowCount, 0) + .set(CsvImportPo::getErrorMessage, "")); + if (updated != 1) { + // Concurrent confirmations reuse the winning submission without creating another batch. + CsvImportPo current = accessible(id); + if (submitted.contains(current.getStatus())) return converter.toVo(current, List.of()); + throw BusinessException.of(ErrorCode.CSV_TASK_STATE_CONFLICT, "任务状态已变化,请刷新后重试。"); + } + return get(id); + } + + public CsvImportVo get(String id) { + return converter.toVo(accessible(id), List.of()); + } + + public CsvImportPageVo list(int page, int pageSize) { + int size = Math.min(100, Math.max(1, pageSize)); + int number = Math.max(1, page); + var user = holder.checkAndGet(); + var query = Wrappers.lambdaQuery(); + if (!Boolean.TRUE.equals(user.superAdmin())) { + var scope = dataPermissions.loadGrantedScope(user, config.getDatasourceId()); + query.and( + nested -> { + nested.eq(CsvImportPo::getCreatorId, user.userId()); + if (!scope.tables().isEmpty()) + nested.or( + granted -> + granted.in( + CsvImportPo::getPhysicalTableName, + scope.tables().values())); + }); + } + var result = + imports.selectPage( + new Page<>(number, size), query.orderByDesc(CsvImportPo::getCreateTime)); + return new CsvImportPageVo( + result.getRecords().stream().map(row -> converter.toVo(row, List.of())).toList(), + result.getTotal(), + number, + size); + } + + public void retrySync(String id) { + CsvImportPo task = accessible(id); + if (!Set.of(DATA_READY, SYNC_FAILED).contains(task.getStatus())) + throw BusinessException.of(ErrorCode.CSV_TASK_STATE_CONFLICT, "该任务无需重试同步。"); + datasources.requireActive(); + int updated = + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, id) + .eq(CsvImportPo::getStatus, task.getStatus()) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .set(CsvImportPo::getStatus, DATA_READY) + // Preserve the physical data and original creator when taking over + // a retry. + .set( + CsvImportPo::getAttemptId, + UUID.randomUUID().toString().replace("-", "")) + .set(CsvImportPo::getSubmittedBy, holder.checkAndGet().userId()) + .set(CsvImportPo::getErrorMessage, "")); + if (updated != 1) + throw BusinessException.of(ErrorCode.CSV_TASK_STATE_CONFLICT, "任务状态已变化,请刷新后重试。"); + } + + public void delete(String id) throws Exception { + accessible(id); + CsvImportPo task = deletion.begin(id); + try { + if (!Set.of(PREVIEWED, IMPORT_FAILED, EXPIRED).contains(task.getStatus())) + tables.drop(task, datasources.requireActive()); + files.delete(id); + deletion.finish(task); + } catch (Exception e) { + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, id) + .eq(CsvImportPo::getStatus, DELETING) + .set(CsvImportPo::getStatus, DELETE_FAILED) + .set(CsvImportPo::getErrorMessage, "删除未完成,请检查导入库连接后重试。")); + throw BusinessException.of(ErrorCode.CSV_DELETE_FAILED, "删除未完成,请检查导入库连接后重试。", e); + } + } + + private CsvImportPo accessible(String id) { + CsvImportPo task = + Optional.ofNullable(imports.selectById(id)) + .orElseThrow( + () -> + BusinessException.of( + ErrorCode.CSV_TASK_NOT_FOUND, "导入记录不存在。")); + var user = holder.checkAndGet(); + if (!Boolean.TRUE.equals(user.superAdmin()) + && !Long.valueOf(user.userId()).equals(task.getCreatorId())) { + if (!dataPermissions + .loadGrantedScope(user, task.getDatasourceId()) + .canTable(task.getPhysicalTableName())) + throw BusinessException.of(ErrorCode.FORBIDDEN); + } + return task; + } + + private void requirePreviewable(CsvImportPo task) { + if (!Set.of(PREVIEWED, IMPORT_FAILED).contains(task.getStatus()) + || task.getExpiresAt().isBefore(LocalDateTime.now()) + || !Files.isRegularFile(files.path(task.getId()))) + throw BusinessException.of(ErrorCode.CSV_TASK_STATE_CONFLICT, "文件已过期或任务已经提交,请重新上传。"); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportWorker.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportWorker.java new file mode 100644 index 00000000..024f85c6 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvImportWorker.java @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.csv; + +import static io.github.malonetalk.enums.CsvImportStatus.CLEANING; +import static io.github.malonetalk.enums.CsvImportStatus.DATA_READY; +import static io.github.malonetalk.enums.CsvImportStatus.IMPORTING; +import static io.github.malonetalk.enums.CsvImportStatus.QUEUED; +import static io.github.malonetalk.enums.CsvImportStatus.READY; +import static io.github.malonetalk.enums.CsvImportStatus.SYNCING; +import static io.github.malonetalk.enums.CsvImportStatus.SYNC_FAILED; +import static io.github.malonetalk.enums.PermissionEnum.DATA_CSV_IMPORT; +import static io.github.malonetalk.enums.PermissionEnum.SEMANTIC_TABLE_SYNC; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import io.github.malonetalk.config.CsvImportConfig; +import io.github.malonetalk.enums.CsvImportStatus; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ExceptionResponseMapper; +import io.github.malonetalk.mapper.CsvImportMapper; +import io.github.malonetalk.model.holder.UserContextHolder; +import io.github.malonetalk.model.po.CsvImportPo; +import io.github.malonetalk.model.po.DatasourcePo; +import io.github.malonetalk.service.PermissionService; +import jakarta.annotation.Resource; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.task.TaskRejectedException; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class CsvImportWorker { + private final CsvImportConfig config; + private final CsvImportMapper imports; + private final CsvDatasourceService datasources; + private final CsvTableImportService tables; + private final CsvImportPublishService publisher; + private final CsvImportCleanupService cleanup; + private final PermissionService permissions; + private final UserContextHolder holder; + private final ExceptionResponseMapper exceptionResponses; + + @Resource(name = "csvImportExecutor") + private ThreadPoolTaskExecutor executor; + + @Scheduled(fixedDelay = 5000, initialDelay = 5000) + public void dispatch() { + if (config.getDatasourceId() <= 0) return; + var tasks = + imports.selectList( + Wrappers.lambdaQuery() + .eq(CsvImportPo::getDatasourceId, config.getDatasourceId()) + .in(CsvImportPo::getStatus, QUEUED, DATA_READY) + .orderByAsc(CsvImportPo::getCreateTime) + .last("LIMIT " + executor.getMaxPoolSize())); + for (CsvImportPo task : tasks) { + if (executor.getActiveCount() >= executor.getMaxPoolSize()) break; + try { + executor.execute(() -> run(task)); + } catch (TaskRejectedException ignored) { + break; // The durable queue is retried on the next poll. + } + } + } + + private void run(CsvImportPo task) { + boolean importing = task.getStatus() == QUEUED; + if (importing && !transition(task, QUEUED, IMPORTING)) return; + try { + var user = + permissions.require( + Optional.ofNullable(task.getSubmittedBy()) + .orElseGet(() -> Math.toIntExact(task.getCreatorId())), + DATA_CSV_IMPORT, + SEMANTIC_TABLE_SYNC); + holder.set(user); + var datasource = datasources.requireActive(); + if (importing && !importData(task, datasource)) return; + publish(task, datasource); + } catch (Exception e) { + fail(task, e); + } finally { + holder.clear(); + } + } + + private boolean importData(CsvImportPo task, DatasourcePo datasource) throws Exception { + long count = tables.importTable(task, datasource, rows -> updateProgress(task, rows)); + // Complete the data phase atomically under the same attempt and status guards. + int changed = + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, task.getId()) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .eq(CsvImportPo::getStatus, IMPORTING) + .set(CsvImportPo::getRowCount, count) + .set(CsvImportPo::getStatus, DATA_READY)); + if (changed != 1) { + cleanup.discardStaleTable(task); + return false; + } + task.setRowCount(count); + task.setStatus(DATA_READY); + return true; + } + + private void updateProgress(CsvImportPo task, long rows) { + int changed = + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, task.getId()) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .eq(CsvImportPo::getStatus, IMPORTING) + .set(CsvImportPo::getProcessedRows, rows)); + if (changed != 1) throw new IllegalStateException("Import was interrupted"); + } + + private void publish(CsvImportPo task, DatasourcePo datasource) { + if (!transition(task, DATA_READY, SYNCING)) return; + publisher.publish(task, publisher.snapshot(task, datasource)); + task.setStatus(READY); + cleanup.removeFile(task); + } + + private boolean transition(CsvImportPo task, CsvImportStatus from, CsvImportStatus to) { + int changed = + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, task.getId()) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .eq(CsvImportPo::getStatus, from) + .set(CsvImportPo::getStatus, to)); + if (changed == 1) task.setStatus(to); + return changed == 1; + } + + private void fail(CsvImportPo task, Exception error) { + // Log the original cause before a status update or compensation can fail as well. + logTaskFailure(task, "execution", error); + boolean importing = task.getStatus() == IMPORTING; + String message; + if (error instanceof BusinessException) message = error.getMessage(); + else if (importing) message = "CSV 解析或数据库写入失败,请检查文件格式、编码、字段类型和导入库连接。"; + else message = "语义同步失败,请检查分析账号的读取权限后重试同步。"; + CsvImportStatus target = importing ? CLEANING : SYNC_FAILED; + int changed = + imports.update( + Wrappers.lambdaUpdate() + .eq(CsvImportPo::getId, task.getId()) + .eq(CsvImportPo::getAttemptId, task.getAttemptId()) + .eq(CsvImportPo::getStatus, task.getStatus()) + .set(CsvImportPo::getStatus, target) + .set( + CsvImportPo::getErrorMessage, + message.substring(0, Math.min(500, message.length())))); + if (changed == 1 && target == CLEANING) { + task.setStatus(CLEANING); + cleanup.cleanupTable(task); + } else if (changed == 0 && task.getStatus() == IMPORTING) { + cleanup.discardStaleTable(task); + } + } + + private void logTaskFailure(CsvImportPo task, String phase, Exception error) { + String context = + "CSV %s [taskId=%s, attemptId=%s, status=%s, table=%s]" + .formatted( + phase, + task.getId(), + task.getAttemptId(), + task.getStatus(), + task.getPhysicalTableName()); + exceptionResponses.logMapped(log, error, exceptionResponses.resolve(error), context); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvTableImportService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvTableImportService.java new file mode 100644 index 00000000..b1a3609f --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/csv/CsvTableImportService.java @@ -0,0 +1,267 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.csv; + +import io.github.malonetalk.agent.datasource.DataSourceType; +import io.github.malonetalk.config.CsvImportConfig; +import io.github.malonetalk.exception.BusinessException; +import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.model.converter.CsvImportConverter; +import io.github.malonetalk.model.converter.CsvSchemaConverter; +import io.github.malonetalk.model.dto.CsvColumnDto; +import io.github.malonetalk.model.po.CsvImportPo; +import io.github.malonetalk.model.po.DatasourcePo; +import io.github.malonetalk.utils.CsvPhysicalNames; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.function.LongConsumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class CsvTableImportService { + private static final int MAX_IMPORT_ROWS = 100000; + private static final int MAX_BATCH_ROWS = 1000; + private static final long MAX_BATCH_BYTES = 1024 * 1024; + + private final CsvImportConfig config; + private final CsvFileService files; + private final CsvSchemaConverter converter; + private final CsvImportConverter importConverter; + + public long importTable(CsvImportPo task, DatasourcePo datasource, LongConsumer progress) + throws Exception { + if (!datasource.getId().equals(task.getDatasourceId())) + throw BusinessException.of(ErrorCode.CSV_TASK_STATE_CONFLICT, "任务不属于当前 CSV 数据源。"); + files.verify(task.getId(), task.getFileSha256()); + List columns = importConverter.columns(task.getColumnsJson()); + converter.validate(columns, columns.size()); + try (Connection connection = connect(datasource)) { + progress.accept(0); + createTable(connection, task, columns); + connection.setAutoCommit(false); + try { + long count = writeRows(connection, task, columns, progress); + connection.commit(); + return count; + } catch (Exception e) { + connection.rollback(); + throw e; + } + } + } + + private Connection connect(DatasourcePo datasource) throws Exception { + if (!config.getWriterUrl().startsWith(DataSourceType.MYSQL.getUrlPrefix()) + || config.getWriterUsername().isBlank()) + throw BusinessException.of(ErrorCode.CSV_NOT_CONFIGURED, "请先配置 CSV 导入库的写入连接和账号。"); + Properties properties = new Properties(); + properties.setProperty("user", config.getWriterUsername()); + properties.setProperty("password", config.getWriterPassword()); + properties.setProperty("connectTimeout", "10000"); + properties.setProperty("socketTimeout", "30000"); + properties.setProperty("rewriteBatchedStatements", "true"); + properties.setProperty("characterEncoding", "UTF-8"); + Connection connection = DriverManager.getConnection(config.getWriterUrl(), properties); + try { + if (!datasource.getDatabaseName().equals(connection.getCatalog())) + throw BusinessException.of( + ErrorCode.CSV_NOT_CONFIGURED, "CSV 写入连接与分析数据源必须使用同一个数据库。"); + try (var statement = connection.createStatement()) { + statement.setQueryTimeout(30); + statement.execute( + "SET SESSION sql_mode =" + + " 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'"); + } + return connection; + } catch (Exception e) { + connection.close(); + throw e; + } + } + + private void createTable(Connection connection, CsvImportPo task, List columns) + throws Exception { + String definitions = + columns.stream() + .map( + column -> + "`" + + CsvPhysicalNames.column(column.index()) + + "` " + + converter.sqlType(column) + + " NULL COMMENT '" + + comment(column.name()) + + "'") + .collect(Collectors.joining(",")); + try (var statement = connection.createStatement()) { + statement.setQueryTimeout(30); + statement.execute( + "CREATE TABLE " + + table(task) + + " (" + + definitions + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='" + + comment(task.getDisplayName()) + + "'"); + } + } + + private long writeRows( + Connection connection, + CsvImportPo task, + List columns, + LongConsumer progress) + throws Exception { + var options = importConverter.options(task.getParseOptionsJson()); + String placeholders = columns.stream().map(column -> "?").collect(Collectors.joining(",")); + try (CSVParser parser = files.open(task.getId(), options); + PreparedStatement statement = + connection.prepareStatement( + "INSERT INTO " + table(task) + " VALUES (" + placeholders + ")")) { + statement.setQueryTimeout(30); + long count = 0; + int batchRows = 0; + long batchBytes = 0; + for (CSVRecord record : parser) { + long bytes = files.validateAndMeasure(record, columns.size()); + if (record.getRecordNumber() == 1 && options.header()) continue; + if (++count > MAX_IMPORT_ROWS) + throw BusinessException.of( + ErrorCode.CSV_FILE_INVALID, "CSV 最多允许 " + MAX_IMPORT_ROWS + " 条数据记录。"); + task.setProcessedRows(count); + if (batchRows > 0 && batchBytes + bytes > MAX_BATCH_BYTES) { + flush(connection, statement, count - 1, progress); + batchRows = 0; + batchBytes = 0; + } + bind(statement, record, columns, options.emptyAsNull()); + statement.addBatch(); + batchRows++; + batchBytes += bytes; + if (batchRows == MAX_BATCH_ROWS) { + flush(connection, statement, count, progress); + batchRows = 0; + batchBytes = 0; + } + } + if (count == 0) throw BusinessException.of(ErrorCode.CSV_FILE_INVALID, "CSV 没有数据记录。"); + if (batchRows > 0) flush(connection, statement, count, progress); + return count; + } + } + + private void bind( + PreparedStatement statement, + CSVRecord record, + List columns, + boolean emptyAsNull) + throws Exception { + for (CsvColumnDto column : columns) { + try { + statement.setObject( + column.index() + 1, + converter + .value(record.get(column.index()), column, emptyAsNull) + .orElse(null)); + } catch (RuntimeException e) { + // Do not include cell contents in errors: uploads may contain sensitive values. + throw BusinessException.of( + ErrorCode.CSV_SCHEMA_INVALID, + "第 " + + record.getRecordNumber() + + " 条记录,第 " + + (column.index() + 1) + + " 列不能转换为 " + + converter.sqlType(column) + + ",请调整字段类型后重新导入。"); + } + } + } + + private void flush( + Connection connection, PreparedStatement statement, long count, LongConsumer progress) + throws Exception { + statement.executeBatch(); + statement.clearBatch(); + // Batches commit independently; failure recovery drops the whole attempt's table. + // The outer rollback only covers the current, uncommitted batch. + connection.commit(); + progress.accept(count); + } + + public void drop(CsvImportPo task, DatasourcePo datasource) throws Exception { + if (!datasource.getId().equals(task.getDatasourceId())) + throw BusinessException.of(ErrorCode.CSV_TASK_STATE_CONFLICT, "任务不属于当前 CSV 数据源。"); + try (Connection connection = connect(datasource); + var statement = connection.createStatement()) { + statement.setQueryTimeout(30); + statement.execute("DROP TABLE IF EXISTS " + table(task)); + } + } + + private String table(CsvImportPo task) { + if (!CsvPhysicalNames.isManagedTable(task.getPhysicalTableName()) + || !task.getId().matches("[a-f0-9]{32}")) + throw BusinessException.of(ErrorCode.CSV_SCHEMA_INVALID, "CSV 物理表名称无效。"); + return "`" + task.getPhysicalTableName() + "`"; + } + + public void dropOrphanTables(DatasourcePo datasource, Predicate registered) + throws Exception { + try (Connection connection = connect(datasource)) { + List candidates = new ArrayList<>(); + try (var rows = + connection + .getMetaData() + .getTables( + connection.getCatalog(), + null, + "csv%", + new String[] {"TABLE"})) { + while (rows.next()) { + String name = rows.getString("TABLE_NAME"); + if (CsvPhysicalNames.isManagedTable(name)) candidates.add(name); + } + } + try (var statement = connection.createStatement()) { + statement.setQueryTimeout(30); + for (String name : candidates) { + // Registration precedes CREATE, and table names are never reused. + // Check after discovery so a concurrent new import cannot be mistaken for an + // orphan. + if (!registered.test(name)) + statement.execute("DROP TABLE IF EXISTS `" + name + "`"); + } + } + } + } + + private String comment(String text) { + return text.replace("\\", "\\\\").replace("'", "''").replace("\0", ""); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java index a98d0951..5ce4523b 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java @@ -32,6 +32,7 @@ import io.github.malonetalk.mapper.LogicalTableRelationMapper; import io.github.malonetalk.mapper.TableInfoMapper; import io.github.malonetalk.model.po.DatasourcePo; +import io.github.malonetalk.service.csv.CsvAccessService; import io.github.malonetalk.service.semantic.relation.LogicalTableRelationHelper; import io.github.malonetalk.utils.SemanticUtils; import java.util.ArrayList; @@ -55,12 +56,17 @@ public class SemanticMergeService { private final ColumnSemanticInfoMapper columnSemanticInfoMapper; private final LogicalTableRelationMapper logicalTableRelationMapper; private final LogicalTableRelationHelper logicalTableRelationHelper; + private final CsvAccessService csvAccess; public List listVisibleTablesByDomains( DatasourcePo datasource, List domains) { List normalizedDomains = normalizeDomains(domains); // 关系和列按 table_id 对齐;表名只用于最终提示词。 - List tables = tableInfoMapper.selectByDatasourceId(datasource.getId()); + var ready = csvAccess.readyFilter(datasource); + List tables = + tableInfoMapper.selectByDatasourceId(datasource.getId()).stream() + .filter(table -> ready.test(table.getTableName())) + .toList(); Map tablesById = new HashMap<>(); for (TableInfo table : tables) { tablesById.put(table.getId(), table); @@ -100,6 +106,7 @@ public List listVisibleTablesByDomains( } public List getTableSchema(DatasourcePo datasource, String tableName) { + csvAccess.requireReady(datasource, tableName); String normalizedTableName = SemanticUtils.normalizeObjectName( tableName, "Missing tableName for merged table schema lookup."); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java index 62073385..3898211e 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java @@ -39,6 +39,7 @@ import io.github.malonetalk.model.holder.UserContextHolder; import io.github.malonetalk.service.DataPermissionService; import io.github.malonetalk.service.DatasourceService; +import io.github.malonetalk.service.DatasourceWriteLockService; import io.github.malonetalk.service.semantic.SemanticAvailabilityHelper; import io.github.malonetalk.utils.RequestAssert; import io.github.malonetalk.utils.SemanticUtils; @@ -65,6 +66,7 @@ public class RelationSemanticServiceImpl implements RelationSemanticService { private final LogicalTableRelationMapper logicalTableRelationMapper; private final LogicalTableRelationHelper logicalTableRelationHelper; private final SemanticConverter semanticConverter; + private final DatasourceWriteLockService datasourceLocks; @Override public PageResponse getRelationPage( @@ -100,7 +102,8 @@ public PageResponse getRelationPage( @Transactional public LogicalTableRelationResponse createRelationSemantic( String tableName, BindLogicalTableRelationRequest request) { - requireDatasource(request.datasourceId()); + datasourceLocks.lock(request.datasourceId()); + // Load CSV READY visibility only after taking the same lock as import deletion. var scope = dataPermissions.loadScope(holder.checkAndGet(), request.datasourceId()); LogicalTableRelation relation = buildRelation(tableName, request, scope); logicalTableRelationMapper.insert(relation); @@ -111,7 +114,7 @@ public LogicalTableRelationResponse createRelationSemantic( @Transactional public LogicalTableRelationResponse updateRelationSemantic( String tableName, UpdateLogicalTableRelationRequest request) { - requireDatasource(request.datasourceId()); + datasourceLocks.lock(request.datasourceId()); var scope = dataPermissions.loadScope(holder.checkAndGet(), request.datasourceId()); LogicalTableRelation existing = requireRelation(request.datasourceId(), tableName, request.relationId(), scope); @@ -125,7 +128,7 @@ public LogicalTableRelationResponse updateRelationSemantic( @Transactional public boolean updateRelationSemanticEnabled( String tableName, UpdateLogicalTableRelationEnabledRequest request) { - requireDatasource(request.datasourceId()); + datasourceLocks.lock(request.datasourceId()); RequestAssert.requireNonNull(request.enabled(), "enabled cannot be null."); var scope = dataPermissions.loadScope(holder.checkAndGet(), request.datasourceId()); LogicalTableRelation relation = diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java index 9f8aeb0e..5bc83b46 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java @@ -23,7 +23,10 @@ import io.github.malonetalk.entity.TableInfo; import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; import io.github.malonetalk.mapper.TableInfoMapper; +import io.github.malonetalk.model.bo.ColumnSyncSourceBo; import io.github.malonetalk.model.bo.SemanticSyncDiffBo; +import io.github.malonetalk.model.bo.TableSyncSourceBo; +import io.github.malonetalk.model.converter.SemanticSyncConverter; import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; import java.util.ArrayList; @@ -34,6 +37,7 @@ import java.util.Set; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @Service @@ -43,10 +47,14 @@ public class SemanticSyncApplyService { private final TableInfoMapper tableInfoMapper; private final ColumnSemanticInfoMapper columnSemanticInfoMapper; private final SemanticSyncDiffService semanticSyncDiffService; + private final SemanticSyncConverter converter; - @Transactional + // Both interactive sync and CSV publication hold the datasource lock before entering. + @Transactional(propagation = Propagation.MANDATORY) public List applyTableSync( - Integer datasourceId, List presentTables, List missingTables) { + Integer datasourceId, + List presentTables, + List missingTables) { Set tableNames = new LinkedHashSet<>(); presentTables.forEach(table -> tableNames.add(table.tableName())); tableNames.addAll(missingTables); @@ -72,19 +80,14 @@ public List applyTableSync( return diff.results(); } - @Transactional + @Transactional(propagation = Propagation.MANDATORY) public List refreshPhysicalStatus( - Integer datasourceId, + List semanticTables, + List semanticColumns, Set physicalTableNames, Map> physicalColumnNamesByTable) { - // 全量对比语义缓存与物理结构,在内存中收集差异后统一批量更新。 - List semanticTables = tableInfoMapper.selectByDatasourceId(datasourceId); - if (semanticTables.isEmpty()) { - return List.of(); - } - List tableNames = semanticTables.stream().map(TableInfo::getTableName).toList(); - Map> columnsByTableName = - loadSemanticColumnsByTable(datasourceId, tableNames); + // The caller captured and locked this scope before reading the physical schema. + Map> columnsByTableName = indexSemanticColumns(semanticColumns); SemanticSyncDiffBo diff = semanticSyncDiffService.comparePhysicalStatus( semanticTables, @@ -93,24 +96,48 @@ public List refreshPhysicalStatus( physicalColumnNamesByTable); LocalDateTime now = LocalDateTime.now(); - markMissingTables(datasourceId, diff.missingTableNames(), now); + Set missingTableNames = Set.copyOf(diff.missingTableNames()); + List missingTableIds = + semanticTables.stream() + .filter(table -> missingTableNames.contains(tableKey(table.getTableName()))) + .map(TableInfo::getId) + .toList(); + if (!missingTableIds.isEmpty()) + tableInfoMapper.markPhysicalMissingByIds(missingTableIds, now); markMissingColumns(diff.missingColumnIds(), now); return diff.results(); } private void batchSavePresentSchema( Integer datasourceId, - List presentTables, + List presentTables, Map existingTableIndex, Map> columnsByTableName) { if (presentTables.isEmpty()) { return; } + savePresentTables(datasourceId, presentTables, existingTableIndex); + + // New table IDs are generated by the database; reload them before saving columns. + List presentTableNames = + presentTables.stream().map(TableSyncSourceBo::tableName).toList(); + Map persistedTableIndex = + loadSemanticTableIndex(datasourceId, presentTableNames); + savePresentColumns(datasourceId, presentTables, persistedTableIndex, columnsByTableName); + } + + private void savePresentTables( + Integer datasourceId, + List presentTables, + Map existingTableIndex) { List newTables = new ArrayList<>(); - for (TableSyncSource table : presentTables) { + for (TableSyncSourceBo table : presentTables) { String tableKey = tableKey(table.tableName()); - TableInfo tableInfo = buildPhysicalTableInfo(datasourceId, table); + TableInfo tableInfo = converter.toTableInfo(datasourceId, table); + tableInfo.setDomain(SemanticConstants.DEFAULT_DOMAIN); + tableInfo.setIsVisible(Boolean.TRUE); + tableInfo.setPhysicalStatus(Boolean.TRUE); TableInfo existingTable = existingTableIndex.get(tableKey); if (existingTable == null) { newTables.add(tableInfo); @@ -127,22 +154,25 @@ private void batchSavePresentSchema( if (!newTables.isEmpty()) { tableInfoMapper.batchUpsertPhysicalCache(newTables); } + } - // 新表的主键由数据库生成,写入列之前需重新取得 table_id。 - List presentTableNames = - presentTables.stream().map(TableSyncSource::tableName).toList(); - Map persistedTableIndex = - loadSemanticTableIndex(datasourceId, presentTableNames); + private void savePresentColumns( + Integer datasourceId, + List presentTables, + Map persistedTableIndex, + Map> columnsByTableName) { List newColumns = new ArrayList<>(); - for (TableSyncSource table : presentTables) { + for (TableSyncSourceBo table : presentTables) { String tableKey = tableKey(table.tableName()); TableInfo persistedTable = persistedTableIndex.get(tableKey); Map existingColumnIndex = semanticSyncDiffService.indexColumns( columnsByTableName.getOrDefault(tableKey, List.of())); - for (ColumnSyncSource column : table.columns()) { + for (ColumnSyncSourceBo column : table.columns()) { ColumnInfo columnInfo = - buildPhysicalColumnInfo(datasourceId, persistedTable.getId(), column); + converter.toColumnInfo(datasourceId, persistedTable.getId(), column); + columnInfo.setIsVisible(Boolean.TRUE); + columnInfo.setPhysicalStatus(Boolean.TRUE); ColumnInfo existingColumn = existingColumnIndex.get(columnKey(column.columnName())); if (existingColumn == null) { newColumns.add(columnInfo); @@ -203,10 +233,14 @@ private Map loadSemanticTableIndex( private Map> loadSemanticColumnsByTable( Integer datasourceId, List tableNames) { - Map> result = new LinkedHashMap<>(); - for (ColumnInfo column : + return indexSemanticColumns( columnSemanticInfoMapper.selectByDatasourceIdAndTableNames( - datasourceId, tableNames)) { + datasourceId, tableNames)); + } + + private Map> indexSemanticColumns(List columns) { + Map> result = new LinkedHashMap<>(); + for (ColumnInfo column : columns) { String tableName = SemanticUtils.normalizeObjectName( column.getTableName(), "Missing semantic tableName."); @@ -214,42 +248,4 @@ private Map> loadSemanticColumnsByTable( } return result; } - - private TableInfo buildPhysicalTableInfo(Integer datasourceId, TableSyncSource table) { - TableInfo tableInfo = new TableInfo(); - tableInfo.setDatasourceId(datasourceId); - tableInfo.setTableName(table.tableName()); - tableInfo.setPhysicalTableDescription(table.description()); - tableInfo.setTableDescription(table.description()); - tableInfo.setDomain(SemanticConstants.DEFAULT_DOMAIN); - tableInfo.setIsVisible(Boolean.TRUE); - tableInfo.setPhysicalStatus(Boolean.TRUE); - return tableInfo; - } - - private ColumnInfo buildPhysicalColumnInfo( - Integer datasourceId, Integer tableId, ColumnSyncSource column) { - ColumnInfo columnInfo = new ColumnInfo(); - columnInfo.setDatasourceId(datasourceId); - columnInfo.setTableId(tableId); - columnInfo.setColumnName(column.columnName()); - columnInfo.setPhysicalColumnDescription(column.description()); - columnInfo.setColumnDescription(column.description()); - columnInfo.setTypeName(column.typeName()); - columnInfo.setPrimaryKey(column.primaryKey()); - columnInfo.setIndexInfo(column.indexInfo()); - columnInfo.setIsVisible(Boolean.TRUE); - columnInfo.setPhysicalStatus(Boolean.TRUE); - return columnInfo; - } - - public record TableSyncSource( - String tableName, String description, List columns) {} - - public record ColumnSyncSource( - String columnName, - String description, - String typeName, - Boolean primaryKey, - String indexInfo) {} } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncDiffService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncDiffService.java index 29d72681..eb7e084e 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncDiffService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncDiffService.java @@ -20,9 +20,9 @@ import io.github.malonetalk.dto.semantic.SyncTableResult; import io.github.malonetalk.entity.ColumnInfo; import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.model.bo.ColumnSyncSourceBo; import io.github.malonetalk.model.bo.SemanticSyncDiffBo; -import io.github.malonetalk.service.semantic.sync.SemanticSyncApplyService.ColumnSyncSource; -import io.github.malonetalk.service.semantic.sync.SemanticSyncApplyService.TableSyncSource; +import io.github.malonetalk.model.bo.TableSyncSourceBo; import io.github.malonetalk.utils.SemanticUtils; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -39,7 +39,7 @@ public class SemanticSyncDiffService { public SemanticSyncDiffBo compareTables( - List presentTables, + List presentTables, List missingTables, Map existingTableIndex, Map> columnsByTableName) { @@ -47,7 +47,7 @@ public SemanticSyncDiffBo compareTables( List missingTableNames = new ArrayList<>(); List missingColumnIds = new ArrayList<>(); List results = new ArrayList<>(); - for (TableSyncSource table : presentTables) { + for (TableSyncSourceBo table : presentTables) { String tableKey = tableKey(table.tableName()); results.add( buildPresentTableResult( @@ -122,7 +122,7 @@ public SemanticSyncDiffBo comparePhysicalStatus( } private SyncTableResult buildPresentTableResult( - TableSyncSource table, + TableSyncSourceBo table, Optional existingTable, List existingColumns, List missingColumnIds) { @@ -130,13 +130,13 @@ private SyncTableResult buildPresentTableResult( Map existingColumnIndex = indexColumns(existingColumns); Set physicalColumnNames = table.columns().stream() - .map(ColumnSyncSource::columnName) + .map(ColumnSyncSourceBo::columnName) .map(this::columnKey) .collect(Collectors.toSet()); int addedColumns = 0; int reactivatedColumns = 0; int updatedColumns = 0; - for (ColumnSyncSource column : table.columns()) { + for (ColumnSyncSourceBo column : table.columns()) { ColumnInfo existingColumn = existingColumnIndex.get(columnKey(column.columnName())); if (existingColumn == null) { addedColumns++; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java index 382676c2..d4d110dc 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java @@ -18,7 +18,6 @@ package io.github.malonetalk.service.semantic.sync; import io.github.malonetalk.agent.datasource.SchemaReader; -import io.github.malonetalk.dto.datasource.PhysicalColumnInfo; import io.github.malonetalk.dto.datasource.PhysicalTableInfo; import io.github.malonetalk.dto.pagination.PageResponse; import io.github.malonetalk.dto.semantic.PhysicalTableCandidatePageQuery; @@ -27,14 +26,18 @@ import io.github.malonetalk.dto.semantic.SyncTableResult; import io.github.malonetalk.dto.semantic.SyncTableSemanticsRequest; import io.github.malonetalk.dto.semantic.SyncTableSemanticsResponse; +import io.github.malonetalk.entity.ColumnInfo; import io.github.malonetalk.entity.TableInfo; import io.github.malonetalk.exception.BusinessException; import io.github.malonetalk.exception.ErrorCode; +import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; import io.github.malonetalk.mapper.TableInfoMapper; +import io.github.malonetalk.model.bo.TableSyncSourceBo; +import io.github.malonetalk.model.converter.SemanticSyncConverter; import io.github.malonetalk.model.po.DatasourcePo; import io.github.malonetalk.service.DatasourceService; -import io.github.malonetalk.service.semantic.sync.SemanticSyncApplyService.ColumnSyncSource; -import io.github.malonetalk.service.semantic.sync.SemanticSyncApplyService.TableSyncSource; +import io.github.malonetalk.service.DatasourceWriteLockService; +import io.github.malonetalk.service.csv.CsvAccessService; import io.github.malonetalk.utils.SemanticUtils; import java.util.ArrayList; import java.util.Comparator; @@ -46,16 +49,21 @@ import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; @Service @RequiredArgsConstructor public class SemanticSyncServiceImpl implements SemanticSyncService { private final DatasourceService datasourceService; + private final DatasourceWriteLockService datasourceWriteLock; private final TableInfoMapper tableInfoMapper; + private final ColumnSemanticInfoMapper columnSemanticInfoMapper; private final SchemaReader schemaReader; private final SemanticSyncApplyService semanticSyncApplyService; private final SemanticSyncResultService semanticSyncResultService; + private final CsvAccessService csvAccess; + private final SemanticSyncConverter converter; @Override public PageResponse getPhysicalTableCandidates( @@ -111,8 +119,10 @@ public PageResponse getPhysicalTableCandidates( private List loadCandidateTables( DatasourcePo datasource, List semanticTables) { Map candidates = loadPhysicalTableIndex(datasource); + var ready = csvAccess.readyFilter(datasource); // 保留语义缓存中的表,供用户发现已从物理数据源消失的表。 for (TableInfo table : semanticTables) { + if (!ready.test(table.getTableName())) continue; String tableName = SemanticUtils.requireTrimmed( table.getTableName(), "Missing semantic tableName."); @@ -126,22 +136,25 @@ private List loadCandidateTables( } @Override + @Transactional public SyncTableSemanticsResponse syncTables(SyncTableSemanticsRequest request) { - DatasourcePo datasource = requireDatasource(request.datasourceId()); + SemanticUtils.requireDatasourceId(request.datasourceId()); + DatasourcePo datasource = datasourceWriteLock.lock(request.datasourceId()); Map physicalTables = loadPhysicalTableIndex(datasource); Map selectedTableNames = new LinkedHashMap<>(); for (String tableName : request.tableNames()) { String trimmedTableName = SemanticUtils.requireTrimmed(tableName, "Missing physical tableName."); + csvAccess.requireReady(datasource, trimmedTableName); selectedTableNames.putIfAbsent( SemanticUtils.normalizeObjectName( trimmedTableName, "Missing physical tableName."), trimmedTableName); } - List presentTables = new ArrayList<>(); + List presentTables = new ArrayList<>(); List missingTableNames = new ArrayList<>(); - // 先读取并整理全部物理结构,再交给事务方法批量写库,避免事务中访问外部数据源。 + // Keep publication/deletion out of the interval between reading and applying the snapshot. for (Map.Entry selectedTable : selectedTableNames.entrySet()) { PhysicalTableInfo physicalTable = physicalTables.get(selectedTable.getKey()); if (physicalTable == null) { @@ -158,14 +171,24 @@ public SyncTableSemanticsResponse syncTables(SyncTableSemanticsRequest request) } @Override + @Transactional public SyncTableSemanticsResponse refreshPhysicalStatus(RefreshPhysicalStatusRequest request) { - DatasourcePo datasource = requireDatasource(request.datasourceId()); + SemanticUtils.requireDatasourceId(request.datasourceId()); + // Lock before any ordinary reads, so a waiting transaction sees the latest CSV READY state. + DatasourcePo datasource = datasourceWriteLock.lock(request.datasourceId()); + List semanticTables = + tableInfoMapper.selectByDatasourceIdForUpdate(datasource.getId()); + if (semanticTables.isEmpty()) return semanticSyncResultService.summarize(List.of()); + List semanticColumns = + columnSemanticInfoMapper.selectByDatasourceIdForUpdate(datasource.getId()); + // Hold these rows through the physical read; update_time has only second precision. Map physicalTables = loadPhysicalTableIndex(datasource); Map> physicalColumnNamesByTable = loadPhysicalColumnNamesByTable(datasource, physicalTables); List results = semanticSyncApplyService.refreshPhysicalStatus( - request.datasourceId(), + semanticTables, + semanticColumns, physicalTables.keySet(), physicalColumnNamesByTable); return semanticSyncResultService.summarize(results); @@ -182,25 +205,12 @@ private Map loadPhysicalTableIndex(DatasourcePo datas return result; } - private TableSyncSource readTableSyncSource( + private TableSyncSourceBo readTableSyncSource( DatasourcePo datasource, PhysicalTableInfo physicalTable) { - // 在开启写事务前读取物理列快照,避免 JDBC 元数据读取延长事务时间。 - List columns = new ArrayList<>(); - for (PhysicalColumnInfo column : - schemaReader.getTableSchema(datasource, physicalTable.tableName())) { - columns.add( - new ColumnSyncSource( - SemanticUtils.requireTrimmed( - column.columnName(), "Missing physical columnName."), - SemanticUtils.trimToNull(column.remarks()), - column.formattedTypeName(), - column.primaryKey(), - column.formattedIndexInfo())); - } - return new TableSyncSource( + return converter.toTable( physicalTable.tableName(), - SemanticUtils.trimToNull(physicalTable.remarks()), - columns); + physicalTable.remarks(), + schemaReader.getTableSchema(datasource, physicalTable.tableName())); } private Map> loadPhysicalColumnNamesByTable( diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/utils/CsvPhysicalNames.java b/data-agent-backend/src/main/java/io/github/malonetalk/utils/CsvPhysicalNames.java new file mode 100644 index 00000000..e5284d23 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/utils/CsvPhysicalNames.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.utils; + +import java.util.regex.Pattern; + +/** Reserved physical names shared by CSV task creation, table writes and orphan cleanup. */ +public final class CsvPhysicalNames { + private static final String TABLE_PREFIX = "csv_t_"; + private static final Pattern IDENTIFIER = Pattern.compile("[a-f0-9]{32}"); + private static final Pattern TABLE = Pattern.compile(TABLE_PREFIX + IDENTIFIER.pattern()); + + private CsvPhysicalNames() {} + + public static String table(String identifier) { + if (!IDENTIFIER.matcher(identifier).matches()) + throw new IllegalArgumentException("Invalid CSV table identifier."); + return TABLE_PREFIX + identifier; + } + + public static boolean isManagedTable(String name) { + return TABLE.matcher(name).matches(); + } + + public static String column(int index) { + return "c%03d".formatted(index + 1); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/utils/DateTimeUtils.java b/data-agent-backend/src/main/java/io/github/malonetalk/utils/DateTimeUtils.java new file mode 100644 index 00000000..6cb95225 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/utils/DateTimeUtils.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.utils; + +import java.time.DateTimeException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.ResolverStyle; + +/** Shared date parsing; storage-specific limits are explicit so other callers can use historical dates. */ +public final class DateTimeUtils { + public static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai"); + private static final DateTimeFormatter DATE_TIME = + DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss") + .withResolverStyle(ResolverStyle.STRICT); + + private DateTimeUtils() {} + + public static LocalDate parseDate(String text) { + return LocalDate.parse(text); + } + + /** Accepts ISO local date-times and the database-style space separator without applying a timezone. */ + public static LocalDateTime parseDateTime(String text) { + return text.indexOf('T') >= 0 + ? LocalDateTime.parse(text) + : LocalDateTime.parse(text, DATE_TIME); + } + + public static LocalDate parseMysqlDate(String text) { + LocalDate date = parseDate(text); + requireMysqlYear(date.getYear()); + return date; + } + + /** CSV DATE/DATETIME storage supports years 1000–9999 and whole seconds. */ + public static LocalDateTime parseMysqlDateTime(String text) { + LocalDateTime date = parseDateTime(text); + requireMysqlYear(date.getYear()); + if (date.getNano() != 0) + throw new DateTimeException("MySQL date-time must use whole seconds."); + return date; + } + + private static void requireMysqlYear(int year) { + if (year < 1000 || year > 9999) + throw new DateTimeException("Date is outside the MySQL range."); + } +} diff --git a/data-agent-backend/src/main/resources/application.properties b/data-agent-backend/src/main/resources/application.properties index 94ec4817..52df9ff7 100644 --- a/data-agent-backend/src/main/resources/application.properties +++ b/data-agent-backend/src/main/resources/application.properties @@ -50,3 +50,13 @@ management.otlp.metrics.export.url=${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:http:// # 日志导出默认关闭(仅终端);在 Docker 中设置 LOGS_OTLP_EXPORT_ENABLED=true management.logging.export.otlp.enabled=${LOGS_OTLP_EXPORT_ENABLED:false} management.opentelemetry.logging.export.otlp.endpoint=${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT:http://localhost:4318/v1/logs} + +# Configure CSV import through environment variables or explicit values below. +csv-import.datasource-id=${CSV_DATASOURCE_ID:} +csv-import.directory=${CSV_IMPORT_DIRECTORY:} +csv-import.writer-url=${CSV_WRITER_URL:} +csv-import.writer-username=${CSV_WRITER_USERNAME:} +csv-import.writer-password=${CSV_WRITER_PASSWORD:} +spring.servlet.multipart.max-file-size=50MB +spring.servlet.multipart.max-request-size=51MB +spring.task.scheduling.pool.size=2 diff --git a/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml b/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml index a70dd086..bbce343c 100644 --- a/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml +++ b/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml @@ -32,6 +32,13 @@ ORDER BY table_meta.table_name ASC, c.column_name ASC, c.id ASC + + + + SELECT COUNT(1) FROM table_info WHERE LOWER(domain) = LOWER(#{domain}) diff --git a/data-agent-backend/src/test/java/io/github/malonetalk/service/PermissionSimplificationTest.java b/data-agent-backend/src/test/java/io/github/malonetalk/service/PermissionSimplificationTest.java index faa51776..67086e23 100644 --- a/data-agent-backend/src/test/java/io/github/malonetalk/service/PermissionSimplificationTest.java +++ b/data-agent-backend/src/test/java/io/github/malonetalk/service/PermissionSimplificationTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -31,22 +32,28 @@ import io.github.malonetalk.exception.BusinessException; import io.github.malonetalk.exception.ErrorCode; import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; +import io.github.malonetalk.mapper.CsvImportMapper; import io.github.malonetalk.mapper.RoleCapabilityMapper; import io.github.malonetalk.mapper.RoleHiddenColumnMapper; import io.github.malonetalk.mapper.RoleTablePermissionMapper; import io.github.malonetalk.mapper.SysUserMapper; import io.github.malonetalk.model.bo.UserContextBo; import io.github.malonetalk.model.converter.UserConverter; +import io.github.malonetalk.model.po.CsvImportPo; import io.github.malonetalk.model.po.DatasourcePo; import io.github.malonetalk.model.po.RoleHiddenColumn; import io.github.malonetalk.model.po.RoleTablePermission; import io.github.malonetalk.model.po.SysUserPo; +import io.github.malonetalk.service.csv.CsvAccessService; import java.util.List; +import java.util.Optional; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.select.Select; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; class PermissionSimplificationTest { private final SysUserMapper users = mock(SysUserMapper.class); @@ -57,11 +64,18 @@ class PermissionSimplificationTest { private final ColumnSemanticInfoMapper metadata = mock(ColumnSemanticInfoMapper.class); private final PermissionService permissions = new PermissionService(users, capabilities, converter); + private final DatasourceService datasources = mock(DatasourceService.class); + private final CsvImportMapper imports = mock(CsvImportMapper.class); private final DataPermissionService data = - new DataPermissionService(tables, columns, mock(DatasourceService.class)); + new DataPermissionService(tables, columns, datasources, new CsvAccessService(imports)); private final UserContextBo user = UserContextBo.builder().userId(7).roleId(2).superAdmin(false).build(); + @BeforeEach + void configureDatasourceMetadata() { + when(datasources.findById(1)).thenReturn(Optional.of(new DatasourcePo())); + } + @Test void authRejectsMissingOrInactiveUsers() { var mapper = mock(SysUserMapper.class); @@ -177,6 +191,32 @@ void hiddenColumnListIsLoadedOnceForAllReferencedColumns() { verify(columns, times(1)).selectByRoleAndDatasource(2, 1); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void csvScopeOnlyAllowsReadyTables(boolean superAdmin) { + var datasource = new DatasourcePo(); + datasource.setId(1); + datasource.setSourceType("CSV"); + when(datasources.findById(1)).thenReturn(Optional.of(datasource)); + var ready = new CsvImportPo(); + ready.setPhysicalTableName("Csv_Ready"); + when(imports.selectList(any())).thenReturn(List.of(ready)); + var readyGrant = new RoleTablePermission(); + readyGrant.setTableName("Csv_Ready"); + var pendingGrant = new RoleTablePermission(); + pendingGrant.setTableName("Csv_Pending"); + when(tables.selectByRoleAndDatasource(2, 1)).thenReturn(List.of(readyGrant, pendingGrant)); + var context = UserContextBo.builder().userId(7).roleId(2).superAdmin(superAdmin).build(); + + assertEquals("Csv_Ready", data.requireTable(context, 1, "`CSV_READY`")); + assertEquals( + ErrorCode.FORBIDDEN, + assertThrows( + BusinessException.class, + () -> data.requireTable(context, 1, "Csv_Pending")) + .getErrorCode()); + } + @ParameterizedTest @EnumSource( value = PermissionEnum.class, diff --git a/data-agent-backend/src/test/java/io/github/malonetalk/service/SqlAuthorizationRegressionTest.java b/data-agent-backend/src/test/java/io/github/malonetalk/service/SqlAuthorizationRegressionTest.java index 0bfd9a37..8798b5c0 100644 --- a/data-agent-backend/src/test/java/io/github/malonetalk/service/SqlAuthorizationRegressionTest.java +++ b/data-agent-backend/src/test/java/io/github/malonetalk/service/SqlAuthorizationRegressionTest.java @@ -29,6 +29,7 @@ import io.github.malonetalk.exception.BusinessException; import io.github.malonetalk.exception.ErrorCode; import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; +import io.github.malonetalk.mapper.CsvImportMapper; import io.github.malonetalk.mapper.RoleCapabilityMapper; import io.github.malonetalk.mapper.RoleHiddenColumnMapper; import io.github.malonetalk.mapper.RoleTablePermissionMapper; @@ -38,8 +39,10 @@ import io.github.malonetalk.model.po.RoleHiddenColumn; import io.github.malonetalk.model.po.RoleTablePermission; import io.github.malonetalk.model.po.SysUserPo; +import io.github.malonetalk.service.csv.CsvAccessService; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -54,6 +57,8 @@ class SqlAuthorizationRegressionTest { @BeforeEach void setup() { var users = mock(SysUserMapper.class); + var datasources = mock(DatasourceService.class); + when(datasources.findById(1)).thenReturn(Optional.of(datasource)); var capabilities = mock(RoleCapabilityMapper.class); var tables = mock(RoleTablePermissionMapper.class); var columns = mock(RoleHiddenColumnMapper.class); @@ -79,7 +84,11 @@ void setup() { new SqlPermissionService( new PermissionService( users, capabilities, Mappers.getMapper(UserConverter.class)), - new DataPermissionService(tables, columns, mock(DatasourceService.class)), + new DataPermissionService( + tables, + columns, + datasources, + new CsvAccessService(mock(CsvImportMapper.class))), metadata); executor = new SqlExecutor(mock(DynamicDataSourceManager.class), permissions); datasource.setId(1); diff --git a/data-agent-backend/src/test/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyServiceTest.java b/data-agent-backend/src/test/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyServiceTest.java index 532cd868..4903a962 100644 --- a/data-agent-backend/src/test/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyServiceTest.java +++ b/data-agent-backend/src/test/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyServiceTest.java @@ -25,11 +25,13 @@ import io.github.malonetalk.entity.TableInfo; import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; import io.github.malonetalk.mapper.TableInfoMapper; -import io.github.malonetalk.service.semantic.sync.SemanticSyncApplyService.ColumnSyncSource; -import io.github.malonetalk.service.semantic.sync.SemanticSyncApplyService.TableSyncSource; +import io.github.malonetalk.model.bo.ColumnSyncSourceBo; +import io.github.malonetalk.model.bo.TableSyncSourceBo; +import io.github.malonetalk.model.converter.SemanticSyncConverter; import java.util.List; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mapstruct.factory.Mappers; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Spy; @@ -41,6 +43,7 @@ class SemanticSyncApplyServiceTest { @Mock private TableInfoMapper tableInfoMapper; @Mock private ColumnSemanticInfoMapper columnSemanticInfoMapper; @Spy private SemanticSyncDiffService semanticSyncDiffService; + @Spy private SemanticSyncConverter converter = Mappers.getMapper(SemanticSyncConverter.class); @InjectMocks private SemanticSyncApplyService service; @Test @@ -58,11 +61,11 @@ void associatesNewColumnWithPersistedTableId() { service.applyTableSync( 7, List.of( - new TableSyncSource( + new TableSyncSourceBo( "orders", "orders table", List.of( - new ColumnSyncSource( + new ColumnSyncSourceBo( "id", "order id", "BIGINT", true, "PRIMARY")))), List.of()); diff --git a/data-agent-frontend/src/api/csvImport.ts b/data-agent-frontend/src/api/csvImport.ts new file mode 100644 index 00000000..0367fc2b --- /dev/null +++ b/data-agent-frontend/src/api/csvImport.ts @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import request from './request'; +import type { ApiResponse } from './request'; + +export const CSV_COLUMN_TYPES = [ + 'VARCHAR', + 'TEXT', + 'BIGINT', + 'DECIMAL', + 'BOOLEAN', + 'DATE', + 'DATETIME', +] as const; + +export type CsvColumnType = (typeof CSV_COLUMN_TYPES)[number]; + +export type CsvImportStatus = + | 'PREVIEWED' + | 'QUEUED' + | 'IMPORTING' + | 'DATA_READY' + | 'SYNCING' + | 'READY' + | 'IMPORT_FAILED' + | 'SYNC_FAILED' + | 'CLEANING' + | 'CLEANUP_FAILED' + | 'DELETING' + | 'DELETE_FAILED' + | 'EXPIRED'; + +export interface CsvParseOptions { + encoding: string; + delimiter: string; + header: boolean; + emptyAsNull: boolean; +} + +export interface CsvColumn { + index: number; + name: string; + type: CsvColumnType; + length: number; + precision: number; + scale: number; +} + +export interface CsvImport { + id: string; + datasourceId: number; + displayName: string; + description: string; + originalFilename: string; + physicalTableName: string; + status: CsvImportStatus; + processedRows: number; + rowCount: number; + errorMessage: string; + createTime: string; + columns: CsvColumn[]; + options: CsvParseOptions; + preview: string[][]; +} + +export interface CsvImportPage { + items: CsvImport[]; + total: number; + page: number; + pageSize: number; +} + +export type CsvImportConfirm = Pick; + +export function previewCsv(file: File, options: CsvParseOptions) { + const data = new FormData(); + data.append('file', file); + Object.entries(options).forEach(([key, value]) => data.append(key, String(value))); + return request + .post>('/csv-imports/preview', data, { timeout: 120000 }) + .then(res => res.data.data); +} + +export function repreviewCsv(id: string, options: CsvParseOptions) { + return request + .put>(`/csv-imports/${id}/preview`, options) + .then(res => res.data.data); +} + +export function confirmCsv(id: string, data: CsvImportConfirm) { + return request + .post>(`/csv-imports/${id}/confirm`, data) + .then(res => res.data.data); +} + +export function getCsvImports(page: number, pageSize: number) { + return request + .get>('/csv-imports', { params: { page, pageSize } }) + .then(res => res.data.data); +} + +export function syncCsv(id: string) { + return request.post(`/csv-imports/${id}/sync`); +} + +export function deleteCsv(id: string) { + return request.delete(`/csv-imports/${id}`); +} diff --git a/data-agent-frontend/src/api/datasource.ts b/data-agent-frontend/src/api/datasource.ts index c3163870..34480468 100644 --- a/data-agent-frontend/src/api/datasource.ts +++ b/data-agent-frontend/src/api/datasource.ts @@ -33,6 +33,7 @@ export interface DatasourceRequest { } export interface DatasourceResponse { + sourceType?: 'DATABASE' | 'CSV'; id: number; name: string; type: string; diff --git a/data-agent-frontend/src/api/semantic.ts b/data-agent-frontend/src/api/semantic.ts index 11db45c3..80c227e2 100644 --- a/data-agent-frontend/src/api/semantic.ts +++ b/data-agent-frontend/src/api/semantic.ts @@ -16,7 +16,6 @@ */ import request, { type ApiResponse } from './request'; -import { getDatasourceOptions } from './datasource'; import type { PageResponse } from './types'; export interface PhysicalTableCandidateResponse { @@ -176,11 +175,6 @@ export interface UpdateLogicalTableRelationEnabledRequest { enabled: boolean; } -export async function getActiveDatasourceId() { - const datasources = await getDatasourceOptions(); - return datasources.find(datasource => datasource.status === 'ACTIVE')?.id ?? null; -} - export function getTableSemanticPage(query: TableSemanticPageQuery) { return request .get>>('/semantic/tables', { diff --git a/data-agent-frontend/src/composables/useAgentChat.ts b/data-agent-frontend/src/composables/useAgentChat.ts index e071dde9..dd57aa15 100644 --- a/data-agent-frontend/src/composables/useAgentChat.ts +++ b/data-agent-frontend/src/composables/useAgentChat.ts @@ -57,8 +57,6 @@ export function useAgentChat(initialSessionId?: string) { const isStreaming = ref(false); const sessionId = ref(initialSessionId || null); let sessionCreation: Promise | null = null; - // 本会话允许/选择的数据源;空数组表示全部可访问数据源。 - const datasourceIds = ref([]); const abortController = shallowRef(null); const pendingQuestion = ref(null); const lastReportContent = ref(null); @@ -153,7 +151,6 @@ export function useAgentChat(initialSessionId?: string) { : { sessionId: currentSessionId, message: text, - datasourceIds: datasourceIds.value.length ? datasourceIds.value : undefined, }; try { @@ -248,14 +245,12 @@ export function useAgentChat(initialSessionId?: string) { stopStreaming(); messages.value = []; sessionId.value = null; - datasourceIds.value = []; } return { messages, isStreaming, sessionId, - datasourceIds, pendingQuestion, lastReportContent, loadHistory, diff --git a/data-agent-frontend/src/views/chat/ChatView.vue b/data-agent-frontend/src/views/chat/ChatView.vue index 6837b3c5..01899166 100644 --- a/data-agent-frontend/src/views/chat/ChatView.vue +++ b/data-agent-frontend/src/views/chat/ChatView.vue @@ -20,8 +20,6 @@ import { useRoute, useRouter } from 'vue-router'; import { useAgentChat } from '@/composables/useAgentChat'; import { useUserStore } from '@/stores/user'; - import { fetchSessionList } from '@/api/agent'; - import ChatDatasourceSelect from '@/views/chat/components/ChatDatasourceSelect.vue'; import ChatMessage from '@/views/chat/components/ChatMessage.vue'; import ChatInput from '@/views/chat/components/ChatInput.vue'; import SessionList from '@/views/chat/components/SessionList.vue'; @@ -39,7 +37,6 @@ messages, isStreaming, sessionId, - datasourceIds, pendingQuestion, lastReportContent, sendMessage, @@ -62,22 +59,9 @@ messages.value.filter(m => m.role === 'user').map(m => m.content), ); - // 重进会话时从会话列表回显数据源范围;无记录时重置为空(表示全部可访问)。 - async function resolveBoundDatasource(sid: string) { - datasourceIds.value = []; - try { - const list = await fetchSessionList(); - const found = list.find(s => s.sessionId === sid); - datasourceIds.value = found?.datasources?.map(d => d.id) ?? []; - } catch { - // 会话列表加载失败不阻断会话展示。 - } - } - async function loadSession(sid: string) { if (canReadSessions.value) { await loadHistory(sid); - await resolveBoundDatasource(sid); } else { newSession(); sessionId.value = sid; @@ -177,7 +161,6 @@
- 会话报告 diff --git a/data-agent-frontend/src/views/chat/components/ChatDatasourceSelect.vue b/data-agent-frontend/src/views/chat/components/ChatDatasourceSelect.vue deleted file mode 100644 index 081e1d62..00000000 --- a/data-agent-frontend/src/views/chat/components/ChatDatasourceSelect.vue +++ /dev/null @@ -1,66 +0,0 @@ - - - - - diff --git a/data-agent-frontend/src/views/data-source/DataSource.vue b/data-agent-frontend/src/views/data-source/DataSource.vue index e66309f2..348651e3 100644 --- a/data-agent-frontend/src/views/data-source/DataSource.vue +++ b/data-agent-frontend/src/views/data-source/DataSource.vue @@ -17,6 +17,7 @@ + + + + diff --git a/data-agent-frontend/src/views/data-source/components/CsvUploadDialog.vue b/data-agent-frontend/src/views/data-source/components/CsvUploadDialog.vue new file mode 100644 index 00000000..f0a2ebb3 --- /dev/null +++ b/data-agent-frontend/src/views/data-source/components/CsvUploadDialog.vue @@ -0,0 +1,282 @@ + + + + + + + diff --git a/data-agent-frontend/src/views/metric/MetricManage.vue b/data-agent-frontend/src/views/metric/MetricManage.vue index 84abd67a..922c4cbc 100644 --- a/data-agent-frontend/src/views/metric/MetricManage.vue +++ b/data-agent-frontend/src/views/metric/MetricManage.vue @@ -16,7 +16,7 @@ -->