Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ DOCREADER_TRANSPORT=grpc

# ========== B1. 数据库 ⚠️ 必填 ==========
# 主数据库类型:postgres / mysql / sqlite。
# 使用 MySQL 时:DB_DRIVER=mysql、DB_HOST=mysql、DB_PORT=3306,并搭配非 postgres 的 RETRIEVE_DRIVER
# (如 qdrant / milvus / weaviate / elasticsearch_v8 / opensearch / doris / tencent_vectordb)。
DB_DRIVER=postgres
# 数据库主机地址。
DB_HOST=postgres
Expand All @@ -111,6 +113,8 @@ DB_USER=postgres
DB_PASSWORD=postgres123!@#
# 数据库名称。
DB_NAME=WeKnora
# MySQL root 密码(仅 docker-compose 的 mysql 服务使用;留空默认复用 DB_PASSWORD)。
# MYSQL_ROOT_PASSWORD=
# SQLite 驱动时使用(DB_DRIVER=sqlite),postgres/mysql 忽略。
# DB_PATH=./data/weknora.db

Expand Down
29 changes: 29 additions & 0 deletions docker-compose.dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,34 @@ services:
restart: unless-stopped
stop_grace_period: 1m

mysql:
image: mysql:8.4
container_name: WeKnora-mysql-dev
ports:
- "${DB_PORT:-3306}:3306"
environment:
- MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-weknora-root-password}
- MYSQL_DATABASE=${DB_NAME}
- MYSQL_USER=${DB_USER}
- MYSQL_PASSWORD=${DB_PASSWORD}
- TZ=${TZ:-Asia/Shanghai}
command:
- --character-set-server=utf8mb4
- --collation-server=utf8mb4_unicode_ci
volumes:
- mysql-data-dev:/var/lib/mysql
networks:
- WeKnora-network-dev
healthcheck:
test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u$${MYSQL_USER} -p$${MYSQL_PASSWORD} --silent"]
interval: 10s
timeout: 10s
retries: 5
start_period: 30s
restart: unless-stopped
profiles:
- mysql

redis:
image: redis:7.0-alpine
container_name: WeKnora-redis-dev
Expand Down Expand Up @@ -574,6 +602,7 @@ networks:

volumes:
postgres-data-dev:
mysql-data-dev:
redis_data_dev:
minio_data_dev:
neo4j-data-dev:
Expand Down
29 changes: 28 additions & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ services:
- WeKnora-network
restart: unless-stopped

# 修改的PostgreSQL配置
# 修改的PostgreSQL配置(默认主库;DB_DRIVER=mysql 时可改用下方 mysql 服务)
postgres:
image: paradedb/paradedb:v0.22.2-pg17
container_name: WeKnora-postgres
Expand All @@ -521,6 +521,32 @@ services:
# 添加停机时的优雅退出时间
stop_grace_period: 1m

mysql:
image: mysql:8.4
container_name: WeKnora-mysql
environment:
- MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-weknora-root-password}
- MYSQL_DATABASE=${DB_NAME}
- MYSQL_USER=${DB_USER}
- MYSQL_PASSWORD=${DB_PASSWORD}
- TZ=${TZ:-Asia/Shanghai}
command:
- --character-set-server=utf8mb4
- --collation-server=utf8mb4_unicode_ci
volumes:
- mysql-data:/var/lib/mysql
networks:
- WeKnora-network
healthcheck:
test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u$${MYSQL_USER} -p$${MYSQL_PASSWORD} --silent"]
interval: 10s
timeout: 10s
retries: 5
start_period: 30s
restart: unless-stopped
profiles:
- mysql

redis:
image: redis:7.0-alpine
container_name: WeKnora-redis
Expand Down Expand Up @@ -1009,6 +1035,7 @@ networks:

volumes:
postgres-data:
mysql-data:
data-files:
docreader-tmp:
minio_data:
Expand Down
2 changes: 1 addition & 1 deletion docker/Dockerfile.app
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \
apt-get install -y git build-essential libsqlite3-dev

# Install migrate tool
RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
RUN go install -tags 'postgres mysql sqlite3' github.com/golang-migrate/migrate/v4/cmd/migrate@latest

# Copy go mod and sum files
COPY go.mod go.sum ./
Expand Down
42 changes: 42 additions & 0 deletions docs/MySQL.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# MySQL 主数据库部署说明

WeKnora 现在支持使用 MySQL 作为主数据库(`DB_DRIVER=mysql`)。PostgreSQL 仍然可用;MySQL 模式主要替换业务主库,不再使用 PostgreSQL/ParadeDB 的内置向量检索能力。

## 关键限制

- `DB_DRIVER=mysql` 时不能再使用 `RETRIEVE_DRIVER=postgres`。
- 请改用独立检索/向量引擎,例如:`qdrant`、`milvus`、`weaviate`、`elasticsearch_v8`、`opensearch`、`doris` 或 `tencent_vectordb`。

## docker-compose 示例

`.env` 示例:

```env
DB_DRIVER=mysql
DB_HOST=mysql
DB_PORT=3306
DB_USER=weknora
DB_PASSWORD=weknora123!@#
DB_NAME=WeKnora

# MySQL 模式下请选择非 postgres 的检索引擎
RETRIEVE_DRIVER=qdrant
QDRANT_HOST=qdrant
QDRANT_PORT=6334
```

启动 MySQL 与示例 Qdrant 检索引擎:

```bash
docker compose --profile mysql --profile qdrant up -d
```

> 现有 Langfuse 集成依然依赖 PostgreSQL,这是 Langfuse 自身要求;它与 WeKnora 主库可分开配置。

## 手动迁移

```bash
DB_DRIVER=mysql ./scripts/migrate.sh up
```

脚本会自动使用 `migrations/mysql` 目录。应用启动时 `AUTO_MIGRATE=true` 也会自动执行相同的 MySQL 迁移。
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ require (
google.golang.org/protobuf v1.36.11
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/mysql v1.6.0
gorm.io/driver/postgres v1.6.0
gorm.io/driver/sqlite v1.6.0
gorm.io/gorm v1.31.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -3961,6 +3961,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
Expand Down
10 changes: 8 additions & 2 deletions internal/application/repository/chunk.go
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,8 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun
args = append(args, id)
}

isPostgres := r.db.Dialector.Name() == "postgres"
dialect := r.db.Dialector.Name()
isPostgres := dialect == "postgres"

var sql string
if isPostgres {
Expand All @@ -476,21 +477,26 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun
strings.Join(inPlaceholders, ","),
)
} else {
nowExpr := "datetime('now')"
if dialect == "mysql" {
nowExpr = "NOW(3)"
}
sql = fmt.Sprintf(`
UPDATE chunks SET
content = CASE %s END,
is_enabled = CASE %s END,
tag_id = CASE %s END,
flags = CASE %s END,
status = CASE %s END,
updated_at = datetime('now')
updated_at = %s
WHERE id IN (%s)
`,
strings.Join(contentCases, " "),
strings.Join(isEnabledCases, " "),
strings.Join(tagIDCases, " "),
strings.Join(flagsCases, " "),
strings.Join(statusCases, " "),
nowExpr,
strings.Join(inPlaceholders, ","),
)
}
Expand Down
25 changes: 20 additions & 5 deletions internal/application/repository/knowledge.go
Original file line number Diff line number Diff line change
Expand Up @@ -584,10 +584,17 @@ func (r *knowledgeRepository) FindByMetadataKey(
value string,
) (*types.Knowledge, error) {
var knowledge types.Knowledge
err := r.db.WithContext(ctx).
Where("tenant_id = ? AND knowledge_base_id = ? AND deleted_at IS NULL", tenantID, kbID).
Where("metadata->>? = ?", key, value).
First(&knowledge).Error
query := r.db.WithContext(ctx).
Where("tenant_id = ? AND knowledge_base_id = ? AND deleted_at IS NULL", tenantID, kbID)
switch r.db.Dialector.Name() {
case "postgres":
query = query.Where("metadata->>? = ?", key, value)
case "mysql":
query = query.Where("JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) = ?", "$."+key, value)
default:
query = query.Where("json_extract(metadata, ?) = ?", "$."+key, value)
}
err := query.First(&knowledge).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
Expand Down Expand Up @@ -621,7 +628,15 @@ func (r *knowledgeRepository) FindByMetadataKeyPrefix(
// custom-planned with the actual value, so LIKE 'prefix%' still extracts the
// prefix and drives the index. The explicit ESCAPE '\' keeps backslash-escaped
// wildcards (e.g. \_) literal on both PostgreSQL and SQLite.
keyExpr := "metadata->>'" + strings.ReplaceAll(key, "'", "''") + "'"
var keyExpr string
switch r.db.Dialector.Name() {
case "postgres":
keyExpr = "metadata->>'" + strings.ReplaceAll(key, "'", "''") + "'"
case "mysql":
keyExpr = "JSON_UNQUOTE(JSON_EXTRACT(metadata, '$." + strings.ReplaceAll(key, "'", "''") + "'))"
default:
keyExpr = "json_extract(metadata, '$." + strings.ReplaceAll(key, "'", "''") + "')"
}
err := r.db.WithContext(ctx).
Where("tenant_id = ? AND knowledge_base_id = ? AND deleted_at IS NULL", tenantID, kbID).
Where(keyExpr+" LIKE ? ESCAPE ?", escaped+"%", `\`).
Expand Down
6 changes: 5 additions & 1 deletion internal/application/repository/message.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,13 +162,17 @@ func (r *messageRepository) SearchMessagesByKeyword(

var results []*types.MessageWithSession

likeExpr := "LOWER(messages.content) LIKE LOWER(?)"
if r.db.Dialector.Name() == "postgres" {
likeExpr = "messages.content ILIKE ?"
}
query := r.db.WithContext(ctx).
Table("messages").
Select("messages.*, sessions.title as session_title").
Joins("INNER JOIN sessions ON sessions.id = messages.session_id AND sessions.deleted_at IS NULL").
Where("sessions.tenant_id = ?", tenantID).
Where("messages.deleted_at IS NULL").
Where("messages.content ILIKE ?", "%"+escapeLikeKeyword(keyword)+"%")
Where(likeExpr, "%"+escapeLikeKeyword(keyword)+"%")

if len(sessionIDs) > 0 {
query = query.Where("messages.session_id IN ?", sessionIDs)
Expand Down
61 changes: 42 additions & 19 deletions internal/application/repository/model_usage.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ import (
// scopeKnowledgeBasesByModelID filters knowledge_bases rows that reference
// modelID in any model-binding field.
func scopeKnowledgeBasesByModelID(db *gorm.DB, modelID string) *gorm.DB {
if db.Dialector.Name() == "postgres" {
switch db.Dialector.Name() {
case "postgres":
return db.Where(
"embedding_model_id = ? OR summary_model_id = ? OR "+
"image_processing_config->>'model_id' = ? OR "+
Expand All @@ -16,36 +17,58 @@ func scopeKnowledgeBasesByModelID(db *gorm.DB, modelID string) *gorm.DB {
"wiki_config->>'synthesis_model_id' = ?",
modelID, modelID, modelID, modelID, modelID, modelID,
)
case "mysql":
return db.Where(
"embedding_model_id = ? OR summary_model_id = ? OR "+
"JSON_UNQUOTE(JSON_EXTRACT(image_processing_config, '$.model_id')) = ? OR "+
"JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) = ? OR "+
"JSON_UNQUOTE(JSON_EXTRACT(asr_config, '$.model_id')) = ? OR "+
"JSON_UNQUOTE(JSON_EXTRACT(wiki_config, '$.synthesis_model_id')) = ?",
modelID, modelID, modelID, modelID, modelID, modelID,
)
default:
return db.Where(
"embedding_model_id = ? OR summary_model_id = ? OR "+
"json_extract(image_processing_config, '$.model_id') = ? OR "+
"json_extract(vlm_config, '$.model_id') = ? OR "+
"json_extract(asr_config, '$.model_id') = ? OR "+
"json_extract(wiki_config, '$.synthesis_model_id') = ?",
modelID, modelID, modelID, modelID, modelID, modelID,
)
}
return db.Where(
"embedding_model_id = ? OR summary_model_id = ? OR "+
"json_extract(image_processing_config, '$.model_id') = ? OR "+
"json_extract(vlm_config, '$.model_id') = ? OR "+
"json_extract(asr_config, '$.model_id') = ? OR "+
"json_extract(wiki_config, '$.synthesis_model_id') = ?",
modelID, modelID, modelID, modelID, modelID, modelID,
)
}

// scopeCustomAgentsByModelID filters custom_agents rows whose config JSON
// references modelID in any model-binding field.
func scopeCustomAgentsByModelID(db *gorm.DB, modelID string) *gorm.DB {
if db.Dialector.Name() == "postgres" {
switch db.Dialector.Name() {
case "postgres":
return db.Where(
"config->>'model_id' = ? OR config->>'rerank_model_id' = ? OR "+
"config->>'vlm_model_id' = ? OR config->>'asr_model_id' = ? OR "+
"config->>'query_understand_model_id' = ? OR "+
"config->'question_suggestions'->'follow_ups'->>'model_id' = ?",
modelID, modelID, modelID, modelID, modelID, modelID,
)
case "mysql":
return db.Where(
"JSON_UNQUOTE(JSON_EXTRACT(config, '$.model_id')) = ? OR "+
"JSON_UNQUOTE(JSON_EXTRACT(config, '$.rerank_model_id')) = ? OR "+
"JSON_UNQUOTE(JSON_EXTRACT(config, '$.vlm_model_id')) = ? OR "+
"JSON_UNQUOTE(JSON_EXTRACT(config, '$.asr_model_id')) = ? OR "+
"JSON_UNQUOTE(JSON_EXTRACT(config, '$.query_understand_model_id')) = ? OR "+
"JSON_UNQUOTE(JSON_EXTRACT(config, '$.question_suggestions.follow_ups.model_id')) = ?",
modelID, modelID, modelID, modelID, modelID, modelID,
)
default:
return db.Where(
"json_extract(config, '$.model_id') = ? OR "+
"json_extract(config, '$.rerank_model_id') = ? OR "+
"json_extract(config, '$.vlm_model_id') = ? OR "+
"json_extract(config, '$.asr_model_id') = ? OR "+
"json_extract(config, '$.query_understand_model_id') = ? OR "+
"json_extract(config, '$.question_suggestions.follow_ups.model_id') = ?",
modelID, modelID, modelID, modelID, modelID, modelID,
)
}
return db.Where(
"json_extract(config, '$.model_id') = ? OR "+
"json_extract(config, '$.rerank_model_id') = ? OR "+
"json_extract(config, '$.vlm_model_id') = ? OR "+
"json_extract(config, '$.asr_model_id') = ? OR "+
"json_extract(config, '$.query_understand_model_id') = ? OR "+
"json_extract(config, '$.question_suggestions.follow_ups.model_id') = ?",
modelID, modelID, modelID, modelID, modelID, modelID,
)
}
6 changes: 5 additions & 1 deletion internal/application/repository/organization.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,11 @@ func (r *organizationRepository) ListSearchable(ctx context.Context, query strin
if query != "" {
pattern := "%" + query + "%"
// 支持按名称、描述或空间 ID 搜索,便于区分同名空间
q = q.Where("name ILIKE ? OR description ILIKE ? OR id::text ILIKE ?", pattern, pattern, pattern)
if r.db.Dialector.Name() == "postgres" {
q = q.Where("name ILIKE ? OR description ILIKE ? OR id::text ILIKE ?", pattern, pattern, pattern)
} else {
q = q.Where("LOWER(name) LIKE LOWER(?) OR LOWER(description) LIKE LOWER(?) OR LOWER(id) LIKE LOWER(?)", pattern, pattern, pattern)
}
}
err := q.Order("created_at DESC").Limit(limit).Find(&orgs).Error
if err != nil {
Expand Down
Loading
Loading