Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -111,3 +111,6 @@ yarn-integrity

application-dev.properties
agents

# Local CSV uploaded source files
/data-agent-backend/.local/
5 changes: 5 additions & 0 deletions data-agent-backend/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,11 @@
</dependencyManagement>

<dependencies>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.14.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,11 @@
import io.github.malonetalk.service.DatasourceService;
import io.github.malonetalk.service.PermissionService;
import io.github.malonetalk.service.SessionDatasourceService;
import io.github.malonetalk.utils.DateTimeUtils;
import io.opentelemetry.context.Context;
import io.opentelemetry.instrumentation.reactor.v3_1.ContextPropagationOperator;
import jakarta.annotation.PostConstruct;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
Expand All @@ -74,8 +74,6 @@
@RequiredArgsConstructor
public class AgentService {

private static final ZoneId DEFAULT_ZONE = ZoneId.of("Asia/Shanghai");

private final ModelFactory modelFactory;
private final List<DataAgentTool> allToolBeans;
private final ModelProperties modelProperties;
Expand Down Expand Up @@ -296,7 +294,7 @@ private ReActAgent createAgent(ToolCallContext toolCallContext) {
}

private String systemPrompt() {
LocalDate today = LocalDate.now(DEFAULT_ZONE);
LocalDate today = LocalDate.now(DateTimeUtils.DEFAULT_ZONE);
return """
你是一个数据助手,可以帮助用户查询数据库中的数据。
当前时区是 Asia/Shanghai。
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
import io.github.malonetalk.exception.BusinessException;
import io.github.malonetalk.exception.ErrorCode;
import io.github.malonetalk.model.converter.JdbcColumnMetadataConverter;
import io.github.malonetalk.model.po.CsvImportPo;
import io.github.malonetalk.model.po.DatasourcePo;
import io.github.malonetalk.service.csv.CsvAccessService;
import io.github.malonetalk.utils.SemanticUtils;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
Expand All @@ -48,12 +50,14 @@ public class SchemaReader {

private final DynamicDataSourceManager dynamicDataSourceManager;
private final JdbcColumnMetadataConverter columnMetadataConverter;
private final CsvAccessService csvAccess;

public List<PhysicalTableInfo> getTables(DatasourcePo datasource) {
javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource);

try (Connection conn = ds.getConnection()) {
return getTables(conn);
var ready = csvAccess.readyFilter(datasource);
return getTables(conn).stream().filter(table -> ready.test(table.tableName())).toList();
} catch (SQLException e) {
log.error("Failed to read tables: {}", e.getMessage(), e);
throw BusinessException.of(
Expand All @@ -64,6 +68,17 @@ public List<PhysicalTableInfo> getTables(DatasourcePo datasource) {
}

public List<PhysicalColumnInfo> getTableSchema(DatasourcePo datasource, String tableName) {
csvAccess.requireReady(datasource, tableName);
return readTableSchema(datasource, tableName);
}

public List<PhysicalColumnInfo> getImportTableSchema(
DatasourcePo datasource, CsvImportPo task) {
csvAccess.requireImportTable(datasource, task);
return readTableSchema(datasource, task.getPhysicalTableName());
}

private List<PhysicalColumnInfo> readTableSchema(DatasourcePo datasource, String tableName) {
javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource);

try (Connection conn = ds.getConnection()) {
Expand All @@ -86,7 +101,9 @@ public Map<String, Set<String>> getTableColumnNames(
return Map.of();
}
Map<String, Set<String>> columnNamesByTable = new LinkedHashMap<>();
var ready = csvAccess.readyFilter(datasource);
for (String tableName : tableNames) {
if (!ready.test(tableName)) continue;
columnNamesByTable.putIfAbsent(tableName, new LinkedHashSet<>());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import io.github.malonetalk.agent.tools.ToolResultBoConverter;
import io.github.malonetalk.exception.ExceptionResponseMapper;
import io.github.malonetalk.model.bo.ToolResultBo;
import io.github.malonetalk.utils.DateTimeUtils;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
Expand All @@ -52,7 +53,6 @@
@AllArgsConstructor(access = AccessLevel.PRIVATE)
public class DateInfoTool implements DataAgentTool {

private static final String DEFAULT_TIMEZONE = "Asia/Shanghai";
private static final String HOLIDAY_API_BASE_URL = "https://timor.tech/api/holiday/info";

private final ObjectMapper objectMapper;
Expand Down Expand Up @@ -122,10 +122,12 @@ public ToolResultBo getDateInfo(

DateInfo resolve(String startDateText, String endDateText, String timezoneText) {
ZoneId zoneId =
ZoneId.of(StringUtils.hasText(timezoneText) ? timezoneText : DEFAULT_TIMEZONE);
StringUtils.hasText(timezoneText)
? ZoneId.of(timezoneText)
: DateTimeUtils.DEFAULT_ZONE;
LocalDate startDate =
StringUtils.hasText(startDateText)
? LocalDate.parse(startDateText)
? DateTimeUtils.parseDate(startDateText)
: LocalDate.now(clock.withZone(zoneId));
Optional<HolidayApiResponse> holidayApiResponse = queryHolidayApi(startDate);
HolidayDetail holiday = holidayApiResponse.map(HolidayApiResponse::holiday).orElse(null);
Expand All @@ -144,7 +146,7 @@ DateInfo resolve(String startDateText, String endDateText, String timezoneText)
.holidayDataAvailable(holidayApiResponse.isPresent())
.dateDiff(
StringUtils.hasText(endDateText)
? resolveDateDiff(startDate, LocalDate.parse(endDateText))
? resolveDateDiff(startDate, DateTimeUtils.parseDate(endDateText))
: null)
.build();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* Copyright (C) 2026 github.com/MaloneTalk
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
* limitations under the License.
*/
package io.github.malonetalk.common;

/** Shared CSV limits keep request validation, file parsing and schema conversion consistent. */
public final class CsvLimits {
public static final int MAX_COLUMNS = 100;
public static final int MAX_NAME_LENGTH = 200;

private CsvLimits() {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Copyright (C) 2026 github.com/MaloneTalk
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
* limitations under the License.
*/
package io.github.malonetalk.config;

import java.nio.file.Path;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

@Data
@Configuration
@EnableScheduling
@ConfigurationProperties(prefix = "csv-import")
public class CsvImportConfig {
private Integer datasourceId = 0;
private Path directory = Path.of(System.getProperty("java.io.tmpdir"), "data-agent-csv");
private String writerUrl = "";
private String writerUsername = "";
private String writerPassword = "";

// Keep Boot's applicationTaskExecutor available for MVC/SSE.
@Bean(value = "csvImportExecutor", defaultCandidate = false)
public ThreadPoolTaskExecutor csvImportExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(2);
executor.setMaxPoolSize(2);
executor.setQueueCapacity(0);
executor.setThreadNamePrefix("csv-import-");
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(40);
return executor;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
* Copyright (C) 2026 github.com/MaloneTalk
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
* limitations under the License.
*/
package io.github.malonetalk.controller;

import io.github.malonetalk.annotation.RequirePermission;
import io.github.malonetalk.enums.PermissionEnum;
import io.github.malonetalk.model.dto.CsvImportConfirmDto;
import io.github.malonetalk.model.dto.CsvParseOptions;
import io.github.malonetalk.model.vo.BooleanVo;
import io.github.malonetalk.model.vo.CsvImportPageVo;
import io.github.malonetalk.model.vo.CsvImportVo;
import io.github.malonetalk.service.csv.CsvImportService;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

@RestController
@RequestMapping("/api/csv-imports")
@RequiredArgsConstructor
public class CsvImportController {
private final CsvImportService service;

@PostMapping("/preview")
@RequirePermission(PermissionEnum.DATA_CSV_IMPORT)
public CsvImportVo preview(
@RequestParam MultipartFile file,
@RequestParam(defaultValue = "UTF-8") String encoding,
@RequestParam(defaultValue = ",") String delimiter,
@RequestParam(defaultValue = "true") boolean header,
@RequestParam(defaultValue = "true") boolean emptyAsNull)
throws Exception {
return service.preview(
file,
CsvParseOptions.builder()
.encoding(encoding)
.delimiter(delimiter)
.header(header)
.emptyAsNull(emptyAsNull)
.build());
}

@PutMapping("/{id}/preview")
@RequirePermission(PermissionEnum.DATA_CSV_IMPORT)
public CsvImportVo repreview(
@PathVariable String id, @Valid @RequestBody CsvParseOptions options) throws Exception {
return service.repreview(id, options);
}

@PostMapping("/{id}/confirm")
@ResponseStatus(HttpStatus.ACCEPTED)
@RequirePermission({PermissionEnum.DATA_CSV_IMPORT, PermissionEnum.SEMANTIC_TABLE_SYNC})
public CsvImportVo confirm(
@PathVariable String id, @Valid @RequestBody CsvImportConfirmDto request) {
return service.confirm(id, request);
}

@GetMapping("/{id}")
@RequirePermission(
value = {
PermissionEnum.DATA_CSV_IMPORT,
PermissionEnum.DATA_CSV_DELETE,
PermissionEnum.DATA_DATASOURCE_READ
},
anyOf = true)
public CsvImportVo get(@PathVariable String id) {
return service.get(id);
}

@GetMapping
@RequirePermission(
value = {
PermissionEnum.DATA_CSV_IMPORT,
PermissionEnum.DATA_CSV_DELETE,
PermissionEnum.DATA_DATASOURCE_READ
},
anyOf = true)
public CsvImportPageVo list(
@RequestParam(defaultValue = "1") int page,
@RequestParam(defaultValue = "10") int pageSize) {
return service.list(page, pageSize);
}

@PostMapping("/{id}/sync")
@ResponseStatus(HttpStatus.ACCEPTED)
@RequirePermission({PermissionEnum.DATA_CSV_IMPORT, PermissionEnum.SEMANTIC_TABLE_SYNC})
public BooleanVo sync(@PathVariable String id) {
service.retrySync(id);
return BooleanVo.TRUE;
}

@DeleteMapping("/{id}")
@RequirePermission(PermissionEnum.DATA_CSV_DELETE)
public BooleanVo delete(@PathVariable String id) throws Exception {
service.delete(id);
return BooleanVo.TRUE;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright (C) 2026 github.com/MaloneTalk
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
* limitations under the License.
*/
package io.github.malonetalk.enums;

public enum CsvImportStatus {
PREVIEWED,
QUEUED,
IMPORTING,
DATA_READY,
SYNCING,
READY,
IMPORT_FAILED,
SYNC_FAILED,
CLEANING,
CLEANUP_FAILED,
DELETING,
DELETE_FAILED,
EXPIRED
}
Loading
Loading