Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -67,24 +67,26 @@ public List<PhysicalTableInfo> getTables(DatasourcePo datasource) {
}
}

public List<PhysicalColumnInfo> getTableSchema(DatasourcePo datasource, String tableName) {
csvAccess.requireReady(datasource, tableName);
return readTableSchema(datasource, tableName);
public List<PhysicalColumnInfo> getTableSchema(
DatasourcePo datasource, PhysicalTableInfo table) {
csvAccess.requireReady(datasource, table.tableName());
return readTableSchema(datasource, table.tableName(), "VIEW".equals(table.objectType()));
}

public List<PhysicalColumnInfo> getImportTableSchema(
DatasourcePo datasource, CsvImportPo task) {
csvAccess.requireImportTable(datasource, task);
return readTableSchema(datasource, task.getPhysicalTableName());
return readTableSchema(datasource, task.getPhysicalTableName(), false);
}

private List<PhysicalColumnInfo> readTableSchema(DatasourcePo datasource, String tableName) {
private List<PhysicalColumnInfo> readTableSchema(
DatasourcePo datasource, String tableName, boolean view) {
javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource);

try (Connection conn = ds.getConnection()) {
List<String> primaryKeys = getPrimaryKeys(conn, tableName);
List<String> primaryKeys = view ? List.of() : getPrimaryKeys(conn, tableName);
Map<String, List<String>> indexesByColumn =
getIndexesByColumn(conn, tableName, primaryKeys);
view ? Map.of() : getIndexesByColumn(conn, tableName, primaryKeys);
return getColumns(conn, tableName, primaryKeys, indexesByColumn);
} catch (SQLException e) {
log.error("Failed to read schema for table {}: {}", tableName, e.getMessage(), e);
Expand Down Expand Up @@ -140,10 +142,13 @@ private List<PhysicalTableInfo> getTables(Connection conn) throws SQLException {
conn.getCatalog(),
schemaPattern(conn).orElse(null),
"%",
new String[] {"TABLE"})) {
new String[] {"TABLE", "VIEW"})) {
while (rs.next()) {
tables.add(
new PhysicalTableInfo(rs.getString("TABLE_NAME"), rs.getString("REMARKS")));
new PhysicalTableInfo(
rs.getString("TABLE_NAME"),
rs.getString("REMARKS"),
rs.getString("TABLE_TYPE")));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,9 @@ private static List<TableUse> mergeTables(List<TableUse> tables) {
(old, next) ->
new TableUse(
old.name(),
next.objectType() == null
? old.objectType()
: next.objectType(),
old.domain(),
old.description(),
mergeDistinct(old.columns(), next.columns())));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ public Lineage extract(String sql, List<TableInfo> tableInfo, List<ColumnInfo> c
tables.add(
new TableUse(
name,
info == null ? null : info.getObjectType(),
info == null ? null : info.getDomain(),
info == null ? null : info.getTableDescription(),
columns));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public class GetTablesTool implements DataAgentTool {
converter = ToolResultBoConverter.class,
description =
"""
Get synced semantic-layer table information, including table name, domain, \
Get synced semantic-layer tables and ordinary views, including name, object type, domain, \
description, data granularity (what one row represents) and enabled relations.\
""")
public ToolResultBo getTables(
Expand Down Expand Up @@ -114,12 +114,6 @@ private TablePromptResponse filterTable(
rel.targetTableName(),
rel.targetColumnNames()))
.toList();
return TablePromptResponse.builder()
.name(table.name())
.domain(table.domain())
.description(table.description())
.dataGranularity(table.dataGranularity())
.relations(relations)
.build();
return table.toBuilder().relations(relations).build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import io.github.malonetalk.model.holder.UserContextHolder;
import io.github.malonetalk.model.vo.BooleanVo;
import io.github.malonetalk.model.vo.DatasourceColumnsVo;
import io.github.malonetalk.model.vo.DatasourceObjectVo;
import io.github.malonetalk.model.vo.DatasourceOptionVo;
import io.github.malonetalk.model.vo.DatasourceVo;
import io.github.malonetalk.model.vo.ListVo;
Expand Down Expand Up @@ -84,7 +85,7 @@ public DatasourceVo findById(@PathVariable Integer id) {
return datasourceConverter.toVo(dataSourceService.requireById(id));
}

/** 返回某数据源下的所有物理表名,供权限配置页使用。 */
/** 保留表和视图名称列表的兼容格式。 */
@GetMapping("/{id}/tables")
@RequirePermission(PermissionEnum.SYS_ROLE_GRANT)
public ListVo<String> listTableNames(@PathVariable Integer id) {
Expand All @@ -96,6 +97,19 @@ public ListVo<String> listTableNames(@PathVariable Integer id) {
return new ListVo<>(tables);
}

@GetMapping("/{id}/objects")
@RequirePermission(PermissionEnum.SYS_ROLE_GRANT)
public ListVo<DatasourceObjectVo> listObjects(@PathVariable Integer id) {
dataSourceService.requireById(id);
return new ListVo<>(
tableInfoMapper.selectByDatasourceId(id).stream()
.map(
table ->
new DatasourceObjectVo(
table.getTableName(), table.getObjectType()))
.toList());
}

/** 返回某数据源下所有表的列名,一次查询,供列级权限配置使用。 */
@GetMapping("/{id}/columns")
@RequirePermission(PermissionEnum.SYS_ROLE_GRANT)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ public static TablePromptResponse mapTablePrompt(
}
return TablePromptResponse.builder()
.name(table.getTableName())
.objectType(table.getObjectType())
.domain(SemanticUtils.normalizeDomain(table.getDomain()))
.description(SemanticUtils.trimToNull(table.getTableDescription()))
.dataGranularity(SemanticUtils.trimToNull(table.getDataGranularity()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public TableSemanticResponse toResponse(TableInfo tableInfo) {
return TableSemanticResponse.builder()
.id(tableInfo.getId())
.tableName(tableInfo.getTableName())
.objectType(tableInfo.getObjectType())
.domain(SemanticUtils.normalizeDomain(tableInfo.getDomain()))
.physicalTableDescription(
SemanticUtils.trimToNull(tableInfo.getPhysicalTableDescription()))
Expand Down Expand Up @@ -105,6 +106,7 @@ public RelationWorkspaceTableResponse toWorkspaceTable(
TableInfo tableInfo, List<ColumnInfo> columns) {
return RelationWorkspaceTableResponse.builder()
.tableName(tableInfo.getTableName())
.objectType(tableInfo.getObjectType())
.domain(SemanticUtils.normalizeDomain(tableInfo.getDomain()))
.description(
SemanticUtils.firstNonBlank(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@
*/
package io.github.malonetalk.dto.datasource;

/** 物理数据源表信息,由 SchemaReader 从 JDBC 元数据读取 */
public record PhysicalTableInfo(String tableName, String remarks) {}
/** 数据库表或普通视图信息,由 SchemaReader 从 JDBC 元数据读取。 */
public record PhysicalTableInfo(String tableName, String remarks, String objectType) {}
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@
import lombok.Builder;

/** Agent-facing DTO for LLM prompt formatting. */
@Builder
@Builder(toBuilder = true)
public record TablePromptResponse(
String name,
String objectType,
String domain,
String description,
String dataGranularity,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@
package io.github.malonetalk.dto.semantic;

public record PhysicalTableCandidateResponse(
String tableName, String physicalTableDescription, Boolean synced) {}
String tableName, String physicalTableDescription, String objectType, Boolean synced) {}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
@Builder
public record RelationWorkspaceTableResponse(
String tableName,
String objectType,
String domain,
String description,
boolean operable,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
public record TableSemanticResponse(
Integer id,
String tableName,
String objectType,
String domain,
String physicalTableDescription,
String tableDescription,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public class TableInfo {

private Integer id;
private String tableName;
private String objectType = "TABLE";
private String physicalTableDescription;
private String tableDescription;
private String dataGranularity;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@

import java.util.List;

/** Physical table snapshot shared by schema conversion, CSV publication and semantic sync. */
/** Table or ordinary view snapshot shared by schema conversion, CSV publication and semantic sync. */
public record TableSyncSourceBo(
String tableName, String description, List<ColumnSyncSourceBo> columns) {}
String tableName,
String description,
String objectType,
List<ColumnSyncSourceBo> columns) {}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public abstract class SemanticSyncConverter {
@BeanMapping(ignoreByDefault = true)
@Mapping(target = "datasourceId", source = "datasourceId")
@Mapping(target = "tableName", source = "table.tableName")
@Mapping(target = "objectType", source = "table.objectType")
@Mapping(target = "physicalTableDescription", source = "table.description")
@Mapping(target = "tableDescription", source = "table.description")
public abstract TableInfo toTableInfo(Integer datasourceId, TableSyncSourceBo table);
Expand All @@ -55,10 +56,11 @@ public abstract ColumnInfo toColumnInfo(
Integer datasourceId, Integer tableId, ColumnSyncSourceBo column);

public TableSyncSourceBo toTable(
String name, String description, List<PhysicalColumnInfo> columns) {
String name, String description, String objectType, List<PhysicalColumnInfo> columns) {
return new TableSyncSourceBo(
name,
SemanticUtils.trimToNull(description),
objectType,
columns.stream().map(this::toColumn).toList());
}

Expand All @@ -71,6 +73,7 @@ public TableSyncSourceBo toCsvTable(CsvImportPo task, List<PhysicalColumnInfo> c
return toTable(
task.getPhysicalTableName(),
description.substring(0, Math.min(500, description.length())),
"TABLE",
columns);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,11 @@ public record MetricUse(String key, String name, String measureExpr, String filt
public record PythonUse(String callId, String code, String status) {}

public record TableUse(
String name, String domain, String description, List<ColumnUse> columns) {}
String name,
String objectType,
String domain,
String description,
List<ColumnUse> columns) {}

public record ColumnUse(String name, String description) {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Copyright (C) 2026 github.com/MaloneTalk
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
* limitations under the License.
*/
package io.github.malonetalk.model.vo;

/** Synced database object available for role authorization. */
public record DatasourceObjectVo(String tableName, String objectType) implements BaseVo {}
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
@Getter
@AllArgsConstructor
public enum TableInvalidReasonEnum {
PHYSICAL_TABLE_NOT_FOUND("物理表不存在"),
TABLE_HIDDEN("表已隐藏");
PHYSICAL_TABLE_NOT_FOUND("数据库对象不存在或当前连接账号不可见"),
TABLE_HIDDEN("表或视图已隐藏");

private final String reason;
}
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,17 @@ private SyncTableResult buildPresentTableResult(
}
int missingColumnsMarked =
collectMissingColumnIds(existingColumns, physicalColumnNames, missingColumnIds);
boolean tableUpdated =
existingTable
.map(
existing ->
!Objects.equals(
existing.getObjectType(),
table.objectType())
|| !Objects.equals(
existing.getPhysicalTableDescription(),
table.description()))
.orElse(false);
return SyncTableResult.builder()
.tableName(table.tableName())
.physicalTableFound(true)
Expand All @@ -163,14 +174,7 @@ private SyncTableResult buildPresentTableResult(
.map(TableInfo::getPhysicalStatus)
.filter(Boolean.FALSE::equals)
.isPresent())
.tableUpdated(
existingTable
.filter(
existing ->
!Objects.equals(
existing.getPhysicalTableDescription(),
table.description()))
.isPresent())
.tableUpdated(tableUpdated)
.addedColumns(addedColumns)
.reactivatedColumns(reactivatedColumns)
.updatedColumns(updatedColumns)
Expand All @@ -196,7 +200,7 @@ private SyncTableResult buildMissingTableResult(
!Boolean.FALSE.equals(existing.getPhysicalStatus()))
.isPresent())
.missingColumnsMarked(missingColumnsMarked)
.message("物理表不存在")
.message("数据库对象不存在或当前连接账号不可见")
.build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public PageResponse<PhysicalTableCandidateResponse> getPhysicalTableCandidates(
table.getTableName(),
"Missing semantic tableName."))
.collect(Collectors.toSet());
// SchemaReader 一次性读取物理表,筛选、排序和分页统一在内存中完成。
// SchemaReader 一次性读取表和普通视图,筛选、排序和分页统一在内存中完成。
List<PhysicalTableInfo> filteredTables =
loadCandidateTables(datasource, semanticTables).stream()
.filter(
Expand All @@ -108,6 +108,7 @@ public PageResponse<PhysicalTableCandidateResponse> getPhysicalTableCandidates(
new PhysicalTableCandidateResponse(
table.tableName(),
SemanticUtils.trimToNull(table.remarks()),
table.objectType(),
syncedTableNames.contains(
SemanticUtils.normalizeObjectName(
table.tableName(),
Expand All @@ -120,7 +121,7 @@ private List<PhysicalTableInfo> loadCandidateTables(
DatasourcePo datasource, List<TableInfo> semanticTables) {
Map<String, PhysicalTableInfo> candidates = loadPhysicalTableIndex(datasource);
var ready = csvAccess.readyFilter(datasource);
// 保留语义缓存中的表,供用户发现已从物理数据源消失的表。
// 保留语义缓存中的对象,供用户发现已从数据源消失的表或视图。
for (TableInfo table : semanticTables) {
if (!ready.test(table.getTableName())) continue;
String tableName =
Expand All @@ -130,7 +131,8 @@ private List<PhysicalTableInfo> loadCandidateTables(
SemanticUtils.normalizeObjectName(tableName, "Missing semantic tableName."),
new PhysicalTableInfo(
tableName,
SemanticUtils.trimToNull(table.getPhysicalTableDescription())));
SemanticUtils.trimToNull(table.getPhysicalTableDescription()),
table.getObjectType()));
}
return List.copyOf(candidates.values());
}
Expand Down Expand Up @@ -210,7 +212,8 @@ private TableSyncSourceBo readTableSyncSource(
return converter.toTable(
physicalTable.tableName(),
physicalTable.remarks(),
schemaReader.getTableSchema(datasource, physicalTable.tableName()));
physicalTable.objectType(),
schemaReader.getTableSchema(datasource, physicalTable));
}

private Map<String, Set<String>> loadPhysicalColumnNamesByTable(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ io.github.malonetalk.model.name=deepseek-v4-flash
io.github.malonetalk.model.base-url=https://api.deepseek.com
io.github.malonetalk.model.api-key=${IO_GITHUB_MALONETALK_MODEL_API_KEY:}
io.github.malonetalk.model.thinking-enabled=${IO_GITHUB_MALONETALK_MODEL_THINKING_ENABLED:true}
dev

spring.config.import=classpath:skill.properties

Expand Down
Loading
Loading