From 5a2401c3084ad2be4c5f6ee984cef0073094d18f Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Fri, 27 Mar 2026 08:12:16 -0400 Subject: [PATCH 01/11] feat: add OpenSearch and Typesense fulltext adapters Add two new search backend adapters implementing FullTextAdapter: - @hcengineering/opensearch: Drop-in replacement using OpenSearch 2.x client. Removes deprecated type: _doc params, uses osErr error types. Fixes inherited bugs: updateMany error ID matching, remove() array mutation. - @hcengineering/typesense: Native Typesense adapter with proper query translation. Uses filter_by for workspace/class scoping, optional filter clauses for soft boosts (matching ES should semantics), read-modify-write for updateByQuery, and batch import for updateMany. Binary doc.data is skipped (relies on Rekoni extraction). Both adapters follow the same factory pattern as the existing elastic adapter and can be selected at runtime. Signed-off-by: Don Kendall --- common/config/rush/pnpm-lock.yaml | 201 +++++ .../server/packages/opensearch/.eslintrc.js | 7 + .../server/packages/opensearch/package.json | 71 ++ .../server/packages/opensearch/src/adapter.ts | 698 ++++++++++++++++++ .../server/packages/opensearch/src/index.ts | 17 + .../server/packages/opensearch/tsconfig.json | 12 + .../server/packages/typesense/.eslintrc.js | 7 + .../server/packages/typesense/package.json | 69 ++ .../server/packages/typesense/src/adapter.ts | 598 +++++++++++++++ .../server/packages/typesense/src/index.ts | 17 + .../server/packages/typesense/tsconfig.json | 12 + 11 files changed, 1709 insertions(+) create mode 100644 foundations/server/packages/opensearch/.eslintrc.js create mode 100644 foundations/server/packages/opensearch/package.json create mode 100644 foundations/server/packages/opensearch/src/adapter.ts create mode 100644 foundations/server/packages/opensearch/src/index.ts create mode 100644 foundations/server/packages/opensearch/tsconfig.json create mode 100644 foundations/server/packages/typesense/.eslintrc.js create mode 100644 foundations/server/packages/typesense/package.json create mode 100644 foundations/server/packages/typesense/src/adapter.ts create mode 100644 foundations/server/packages/typesense/src/index.ts create mode 100644 foundations/server/packages/typesense/tsconfig.json diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index de433b45da8..0d9b626867f 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -5216,6 +5216,70 @@ importers: specifier: ^5.9.3 version: 5.9.3 + ../../foundations/server/packages/opensearch: + dependencies: + '@hcengineering/analytics': + specifier: workspace:^0.7.17 + version: link:../../../core/packages/analytics + '@hcengineering/core': + specifier: workspace:^0.7.24 + version: link:../../../core/packages/core + '@hcengineering/platform': + specifier: workspace:^0.7.19 + version: link:../../../core/packages/platform + '@hcengineering/server-core': + specifier: workspace:^0.7.18 + version: link:../core + '@opensearch-project/opensearch': + specifier: ^2.12.0 + version: 2.13.0 + devDependencies: + '@hcengineering/platform-rig': + specifier: workspace:^0.7.19 + version: link:../../../utils/packages/platform-rig + '@types/jest': + specifier: ^29.5.5 + version: 29.5.14 + '@typescript-eslint/eslint-plugin': + specifier: ^6.21.0 + version: 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3) + '@typescript-eslint/parser': + specifier: ^6.21.0 + version: 6.21.0(eslint@8.57.1)(typescript@5.9.3) + eslint: + specifier: ^8.54.0 + version: 8.57.1 + eslint-config-standard-with-typescript: + specifier: ^40.0.0 + version: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3))(eslint-plugin-import@2.32.0(eslint@8.57.1))(eslint-plugin-n@15.7.0(eslint@8.57.1))(eslint-plugin-promise@6.6.0(eslint@8.57.1))(eslint@8.57.1)(typescript@5.9.3) + eslint-plugin-import: + specifier: ^2.26.0 + version: 2.32.0(eslint@8.57.1) + eslint-plugin-n: + specifier: ^15.4.0 + version: 15.7.0(eslint@8.57.1) + eslint-plugin-promise: + specifier: ^6.1.1 + version: 6.6.0(eslint@8.57.1) + eslint-plugin-svelte: + specifier: ^2.35.1 + version: 2.46.1(eslint@8.57.1)(svelte@4.2.20)(ts-node@10.9.2(@types/node@22.19.0)(typescript@5.9.3)) + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@22.19.0)(ts-node@10.9.2(@types/node@22.19.0)(typescript@5.9.3)) + prettier: + specifier: ^3.6.2 + version: 3.6.2 + ts-jest: + specifier: ^29.1.1 + version: 29.4.5(@babel/core@7.28.5)(@jest/transform@29.7.0)(@jest/types@30.2.0)(babel-jest@29.7.0(@babel/core@7.28.5))(esbuild@0.25.12)(jest-util@30.2.0)(jest@29.7.0(@types/node@22.19.0)(ts-node@10.9.2(@types/node@22.19.0)(typescript@5.9.3)))(typescript@5.9.3) + ts-node: + specifier: ^10.9.2 + version: 10.9.2(@types/node@22.19.0)(typescript@5.9.3) + typescript: + specifier: ^5.9.3 + version: 5.9.3 + ../../foundations/server/packages/postgres: dependencies: '@hcengineering/core': @@ -5514,6 +5578,67 @@ importers: specifier: ^5.9.3 version: 5.9.3 + ../../foundations/server/packages/typesense: + dependencies: + '@hcengineering/analytics': + specifier: workspace:^0.7.17 + version: link:../../../core/packages/analytics + '@hcengineering/core': + specifier: workspace:^0.7.24 + version: link:../../../core/packages/core + '@hcengineering/platform': + specifier: workspace:^0.7.19 + version: link:../../../core/packages/platform + '@hcengineering/server-core': + specifier: workspace:^0.7.18 + version: link:../core + typesense: + specifier: ^2.0.0 + version: 2.1.0(@babel/runtime@7.28.4) + devDependencies: + '@hcengineering/platform-rig': + specifier: workspace:^0.7.19 + version: link:../../../utils/packages/platform-rig + '@types/jest': + specifier: ^29.5.5 + version: 29.5.14 + '@typescript-eslint/eslint-plugin': + specifier: ^6.21.0 + version: 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3) + '@typescript-eslint/parser': + specifier: ^6.21.0 + version: 6.21.0(eslint@8.57.1)(typescript@5.9.3) + eslint: + specifier: ^8.54.0 + version: 8.57.1 + eslint-config-standard-with-typescript: + specifier: ^40.0.0 + version: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3))(eslint-plugin-import@2.32.0(eslint@8.57.1))(eslint-plugin-n@15.7.0(eslint@8.57.1))(eslint-plugin-promise@6.6.0(eslint@8.57.1))(eslint@8.57.1)(typescript@5.9.3) + eslint-plugin-import: + specifier: ^2.26.0 + version: 2.32.0(eslint@8.57.1) + eslint-plugin-n: + specifier: ^15.4.0 + version: 15.7.0(eslint@8.57.1) + eslint-plugin-promise: + specifier: ^6.1.1 + version: 6.6.0(eslint@8.57.1) + eslint-plugin-svelte: + specifier: ^2.35.1 + version: 2.46.1(eslint@8.57.1)(svelte@4.2.20)(ts-node@10.9.2(@types/node@22.19.0)(typescript@5.9.3)) + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@22.19.0)(ts-node@10.9.2(@types/node@22.19.0)(typescript@5.9.3)) + prettier: + specifier: ^3.6.2 + version: 3.6.2 + ts-jest: + specifier: ^29.1.1 + version: 29.4.5(@babel/core@7.28.5)(@jest/transform@29.7.0)(@jest/types@30.2.0)(babel-jest@29.7.0(@babel/core@7.28.5))(esbuild@0.25.12)(jest-util@30.2.0)(jest@29.7.0(@types/node@22.19.0)(ts-node@10.9.2(@types/node@22.19.0)(typescript@5.9.3)))(typescript@5.9.3) + typescript: + specifier: ^5.9.3 + version: 5.9.3 + ../../foundations/utils/packages/platform-rig: dependencies: '@typescript-eslint/eslint-plugin': @@ -30694,6 +30819,9 @@ importers: '@hcengineering/mongo': specifier: workspace:^0.7.16 version: link:../../foundations/server/packages/mongo + '@hcengineering/opensearch': + specifier: workspace:^0.7.17 + version: link:../../foundations/server/packages/opensearch '@hcengineering/platform': specifier: workspace:^0.7.20 version: link:../../foundations/core/packages/platform @@ -30721,6 +30849,9 @@ importers: '@hcengineering/server-token': specifier: workspace:^0.7.18 version: link:../../foundations/core/packages/token + '@hcengineering/typesense': + specifier: workspace:^0.7.17 + version: link:../../foundations/server/packages/typesense '@koa/cors': specifier: ^5.0.0 version: 5.0.0 @@ -43423,6 +43554,10 @@ packages: resolution: {integrity: sha512-exj1MzVXoP7xnAcAB3jZ97pTvVPkQF9y6GA/dvYC47HV7vLv+24XRS6b/v/XnyikpEuvMhugEXdGtAlU086WkQ==} engines: {node: '>= 18'} + '@opensearch-project/opensearch@2.13.0': + resolution: {integrity: sha512-Bu3jJ7pKzumbMMeefu7/npAWAvFu5W9SlbBow1ulhluqUpqc7QoXe0KidDrMy7Dy3BQrkI6llR3cWL4lQTZOFw==} + engines: {node: '>=10', yarn: ^1.22.10} + '@opentelemetry/api-logs@0.203.0': resolution: {integrity: sha512-9B9RU0H7Ya1Dx/Rkyc4stuBZSGVQF27WigitInx2QQoj6KUpEFYPKoWjdFTunJYxmXmh17HeBvbMa1EhGyPmqQ==} engines: {node: '>=8.0.0'} @@ -45618,6 +45753,12 @@ packages: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} + aws4@1.13.2: + resolution: {integrity: sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==} + + axios@1.13.6: + resolution: {integrity: sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==} + axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -47604,6 +47745,10 @@ packages: resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==} engines: {node: '>= 6'} + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + formdata-node@4.4.1: resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} engines: {node: '>= 12.20'} @@ -47954,6 +48099,10 @@ packages: hpagent@0.1.2: resolution: {integrity: sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ==} + hpagent@1.2.0: + resolution: {integrity: sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==} + engines: {node: '>=14'} + html-encoding-sniffer@4.0.0: resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} engines: {node: '>=18'} @@ -48707,6 +48856,10 @@ packages: json-stringify-safe@5.0.1: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + json11@2.0.2: + resolution: {integrity: sha512-HIrd50UPYmP6sqLuLbFVm75g16o0oZrVfxrsY0EEys22klz8mRoWlX9KAEDOSOR9Q34rcxsyC8oDveGrCz5uLQ==} + hasBin: true + json5@1.0.2: resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} hasBin: true @@ -51538,6 +51691,12 @@ packages: engines: {node: '>=14.17'} hasBin: true + typesense@2.1.0: + resolution: {integrity: sha512-a/IRTL+dRXlpRDU4UodyGj8hl5xBz3nKihVRd/KfSFAfFPGcpdX6lxIgwdXy3O6VLNNiEsN8YwIsPHQPVT0vNw==} + engines: {node: '>=18'} + peerDependencies: + '@babel/runtime': ^7.23.2 + ua-is-frozen@0.1.2: resolution: {integrity: sha512-RwKDW2p3iyWn4UbaxpP2+VxwqXh0jpvdxsYpZ5j/MLLiQOfbsV5shpgQiw93+KMYQPcteeMQ289MaAFzs3G9pw==} @@ -54203,6 +54362,17 @@ snapshots: '@octokit/webhooks-types': 7.6.1 aggregate-error: 3.1.0 + '@opensearch-project/opensearch@2.13.0': + dependencies: + aws4: 1.13.2 + debug: 4.4.3 + hpagent: 1.2.0 + json11: 2.0.2 + ms: 2.1.3 + secure-json-parse: 2.7.0 + transitivePeerDependencies: + - supports-color + '@opentelemetry/api-logs@0.203.0': dependencies: '@opentelemetry/api': 1.9.0 @@ -56939,6 +57109,16 @@ snapshots: dependencies: possible-typed-array-names: 1.1.0 + aws4@1.13.2: {} + + axios@1.13.6: + dependencies: + follow-redirects: 1.15.11 + form-data: 4.0.5 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + axobject-query@4.1.0: {} b4a@1.7.3: {} @@ -59281,6 +59461,14 @@ snapshots: hasown: 2.0.2 mime-types: 2.1.35 + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + formdata-node@4.4.1: dependencies: node-domexception: 1.0.0 @@ -59729,6 +59917,8 @@ snapshots: hpagent@0.1.2: {} + hpagent@1.2.0: {} + html-encoding-sniffer@4.0.0: dependencies: whatwg-encoding: 3.1.1 @@ -60756,6 +60946,8 @@ snapshots: json-stringify-safe@5.0.1: optional: true + json11@2.0.2: {} + json5@1.0.2: dependencies: minimist: 1.2.8 @@ -63980,6 +64172,15 @@ snapshots: typescript@5.9.3: {} + typesense@2.1.0(@babel/runtime@7.28.4): + dependencies: + '@babel/runtime': 7.28.4 + axios: 1.13.6 + loglevel: 1.9.2 + tslib: 2.8.1 + transitivePeerDependencies: + - debug + ua-is-frozen@0.1.2: {} ua-parser-js@2.0.6: diff --git a/foundations/server/packages/opensearch/.eslintrc.js b/foundations/server/packages/opensearch/.eslintrc.js new file mode 100644 index 00000000000..72235dc2833 --- /dev/null +++ b/foundations/server/packages/opensearch/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/foundations/server/packages/opensearch/package.json b/foundations/server/packages/opensearch/package.json new file mode 100644 index 00000000000..005714f2d00 --- /dev/null +++ b/foundations/server/packages/opensearch/package.json @@ -0,0 +1,71 @@ +{ + "name": "@hcengineering/opensearch", + "version": "0.7.17", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "src/**/*", + "!lib/**/*.test.*", + "!lib/**/*.spec.*", + "!lib/__tests__/**/*", + "!lib/__test__/**/*", + "!lib/tests/**/*", + "!types/**/*.test.*", + "!types/**/*.spec.*", + "!types/__tests__/**/*", + "!types/__test__/**/*", + "!types/tests/**/*", + "!src/**/*.test.*", + "!src/**/*.spec.*", + "!src/__tests__/**/*", + "!src/__test__/**/*", + "!src/tests/**/*" + ], + "author": "Anticrm Platform Contributors", + "license": "EPL-2.0", + "scripts": { + "init": "ts-node src/__init.ts", + "build": "compile", + "build:watch": "compile", + "format": "format src", + "test": "jest --passWithNoTests --silent --forceExit", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent --forceExit", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "workspace:^0.7.19", + "@typescript-eslint/eslint-plugin": "^6.21.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "ts-node": "^10.9.2", + "@typescript-eslint/parser": "^6.21.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.6.2", + "typescript": "^5.9.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5", + "eslint-plugin-svelte": "^2.35.1" + }, + "dependencies": { + "@hcengineering/core": "workspace:^0.7.24", + "@hcengineering/platform": "workspace:^0.7.19", + "@hcengineering/server-core": "workspace:^0.7.18", + "@opensearch-project/opensearch": "^2.12.0", + "@hcengineering/analytics": "workspace:^0.7.17" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/hcengineering/huly.server.git" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/foundations/server/packages/opensearch/src/adapter.ts b/foundations/server/packages/opensearch/src/adapter.ts new file mode 100644 index 00000000000..9b4697ece30 --- /dev/null +++ b/foundations/server/packages/opensearch/src/adapter.ts @@ -0,0 +1,698 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import { + Class, + Doc, + DocumentQuery, + MeasureContext, + Ref, + SearchOptions, + SearchQuery, + TxResult, + WorkspaceUuid +} from '@hcengineering/core' +import type { FullTextAdapter, IndexedDoc, SearchScoring, SearchStringResult } from '@hcengineering/server-core' +import serverCore from '@hcengineering/server-core' + +import { Client, errors as osErr } from '@opensearch-project/opensearch' +import { getMetadata } from '@hcengineering/platform' + +const DEFAULT_LIMIT = 200 + +function getIndexName (): string { + return getMetadata(serverCore.metadata.ElasticIndexName) ?? 'storage_index' +} + +function getIndexVersion (): string { + return getMetadata(serverCore.metadata.ElasticIndexVersion) ?? 'v2' +} + +const mappings = { + properties: { + fulltextSummary: { + type: 'text', + analyzer: 'rebuilt_english' + }, + workspaceId: { + type: 'keyword', + index: true + }, + id: { + type: 'keyword', + index: true + }, + _class: { + type: 'keyword', + index: true + }, + attachedTo: { + type: 'keyword', + index: true + }, + attachedToClass: { + type: 'keyword', + index: true + }, + space: { + type: 'keyword', + index: true + }, + 'core:class:Doc%createdBy': { + type: 'keyword', + index: true + }, + 'core:class:Doc%createdOn': { + type: 'date', + format: 'epoch_millis', + index: true + }, + modifiedBy: { + type: 'keyword', + index: true + }, + modifiedOn: { + type: 'date', + format: 'epoch_millis', + index: true + }, + 'core:class:Doc%modifiedBy': { + type: 'keyword', + index: true + }, + 'core:class:Doc%modifiedOn': { + type: 'date', + format: 'epoch_millis', + index: true + } + } +} + +class OpenSearchAdapter implements FullTextAdapter { + private readonly getFulltextDocId: (workspaceId: WorkspaceUuid, doc: Ref) => Ref + private readonly getDocId: (workspaceId: WorkspaceUuid, fulltext: Ref) => Ref + private readonly indexName: string + + constructor ( + private readonly client: Client, + private readonly indexBaseName: string, + readonly indexVersion: string + ) { + this.indexName = `${indexBaseName}_${indexVersion}` + this.getFulltextDocId = (workspaceId, doc) => `${doc}@${workspaceId}` as Ref + this.getDocId = (workspaceId, fulltext) => fulltext.slice(0, -1 * (workspaceId.length + 1)) as Ref + } + + async initMapping (ctx: MeasureContext): Promise { + const indexName = this.indexName + try { + const existingVersions = await ctx.withSync('get-indexes', {}, () => + this.client.indices.get({ + index: `${this.indexBaseName}_*` + }) + ) + const allIndexes = Object.keys(existingVersions.body) + const existingOldVersionIndices = allIndexes.filter((name) => name !== indexName) + const existsIndex = allIndexes.find((it) => it === indexName) !== undefined + let shouldDropExistingIndex = false + if (existsIndex) { + const mapping = await ctx.with('get-mapping', { indexName }, () => + this.client.indices.getMapping({ + index: indexName + }) + ) + for (const [propName, propType] of Object.entries(mappings.properties)) { + if (mapping.body[indexName]?.mappings.properties?.[propName]?.type !== propType.type) { + shouldDropExistingIndex = true + break + } + } + } + if (existingOldVersionIndices.length > 0 || shouldDropExistingIndex) { + await ctx.with('delete-old-index', {}, () => + this.client.indices.delete({ + index: shouldDropExistingIndex ? allIndexes : existingOldVersionIndices + }) + ) + } + if (!existsIndex || shouldDropExistingIndex) { + await ctx.with('create-index', { indexName }, () => + this.client.indices.create({ + index: indexName, + body: { + settings: { + analysis: { + filter: { + english_stemmer: { + type: 'stemmer', + language: 'english' + }, + english_possessive_stemmer: { + type: 'stemmer', + language: 'possessive_english' + } + }, + analyzer: { + rebuilt_english: { + type: 'custom', + tokenizer: 'standard', + filter: ['english_possessive_stemmer', 'lowercase', 'english_stemmer'] + } + } + } + }, + mappings + } + }) + ) + } else { + await ctx.with('put-mapping', {}, () => + this.client.indices.putMapping({ + index: indexName, + body: mappings + }) + ) + } + } catch (err: any) { + if (err.name === 'ConnectionError') { + ctx.warn('OpenSearch DB is not available') + } + Analytics.handleError(err) + ctx.error(err) + return false + } + return true + } + + async close (): Promise { + await this.client.close() + } + + async searchString ( + ctx: MeasureContext, + workspaceId: WorkspaceUuid, + query: SearchQuery, + options: SearchOptions & { scoring?: SearchScoring[] } + ): Promise { + try { + const elasticQuery: any = { + query: { + function_score: { + query: { + bool: { + must: [ + { + simple_query_string: { + query: query.query, + analyze_wildcard: true, + flags: 'OR|PREFIX|PHRASE|FUZZY|NOT|ESCAPE', + default_operator: 'and', + fields: [ + 'searchTitle^50', // boost + 'searchShortTitle^50', + '*' // Search in all other fields without a boost + ] + } + }, + { + term: { + workspaceId + } + } + ] + } + }, + boost_mode: 'sum' + } + }, + size: options.limit ?? DEFAULT_LIMIT + } + + const filter: any = [ + { + exists: { field: 'searchTitle' } + } + ] + + if (query.spaces !== undefined) { + filter.push({ + terms: this.getTerms(query.spaces, 'space') + }) + } + if (query.classes !== undefined) { + filter.push({ + terms: this.getTerms(query.classes, '_class') + }) + } + + if (filter.length > 0) { + elasticQuery.query.function_score.query.bool.filter = filter + } + + if (options.scoring !== undefined) { + const scoringTerms: any[] = options.scoring.map((scoringOption): any => { + const field = Object.hasOwn(mappings.properties, scoringOption.attr) + ? scoringOption.attr + : `${scoringOption.attr}.keyword` + return { + term: { + [field]: { + value: scoringOption.value, + boost: scoringOption.boost + } + } + } + }) + elasticQuery.query.function_score.query.bool.should = scoringTerms + } + + const result = await this.client.search({ + index: this.indexName, + body: elasticQuery + }) + + const resp: SearchStringResult = { docs: [] } + if (result.body.hits !== undefined) { + if (result.body.hits.total?.value !== undefined) { + resp.total = result.body.hits.total?.value + } + resp.docs = result.body.hits.hits.map((hit: any) => ({ ...hit._source, _score: hit._score })) + } + + return resp + } catch (err: any) { + if (err.name === 'ConnectionError') { + ctx.warn('OpenSearch DB is not available') + return { docs: [] } + } + Analytics.handleError(err) + ctx.error('OpenSearch error', { error: err }) + return { docs: [] } + } + } + + async search ( + ctx: MeasureContext, + workspaceId: WorkspaceUuid, + _classes: Ref>[], + query: DocumentQuery, + size: number | undefined, + from: number | undefined + ): Promise { + if (query.$search === undefined) return [] + const request: any = { + bool: { + must: [ + { + simple_query_string: { + query: query.$search, + analyze_wildcard: true, + flags: 'OR|PREFIX|PHRASE|FUZZY|NOT|ESCAPE', + default_operator: 'and' + } + }, + { + term: { + workspaceId + } + } + ], + should: [{ terms: this.getTerms(_classes, '_class', { boost: 10.0 }) }], + filter: [ + { + bool: { + should: [ + { terms: this.getTerms(_classes, '_class') } + // { terms: this.getTerms(_classes, 'attachedToClass') } + ] + } + } + ] + } + } + + for (const [q, v] of Object.entries(query)) { + if (!q.startsWith('$')) { + const field = Object.hasOwn(mappings.properties, q) ? q : `${q}.keyword` + if (typeof v === 'object') { + if (v.$in !== undefined) { + request.bool.should.push({ + terms: { + [field]: v.$in, + boost: 100.0 + } + }) + } + } else { + request.bool.should.push({ + term: { + [field]: { + value: v, + boost: 100.0, + case_insensitive: true + } + } + }) + } + } + } + + try { + const result = await ctx.with( + 'search', + {}, + () => + this.client.search({ + index: this.indexName, + body: { + query: request, + size: size ?? 200, + from: from ?? 0 + } + }), + { + _classes, + size, + from, + query: request + } + ) + const hits = result.body.hits.hits as any[] + return hits.map((hit) => ({ ...hit._source, _score: hit._score })) + } catch (err: any) { + if (err.name === 'ConnectionError') { + ctx.warn('OpenSearch DB is not available') + return [] + } + ctx.error('OpenSearch error', { error: err }) + Analytics.handleError(err) + return [] + } + } + + private getTerms (values: string[], field: string, extra: any = {}): any { + return { + [Object.hasOwn(mappings.properties, field) ? field : `${field}.keyword`]: values, + ...extra + } + } + + async index (ctx: MeasureContext, workspaceId: WorkspaceUuid, doc: IndexedDoc): Promise { + const wsDoc = { + workspaceId, + ...doc + } + const fulltextId = this.getFulltextDocId(workspaceId, doc.id) + if (doc.data === undefined) { + await this.client.index({ + index: this.indexName, + id: fulltextId, + body: wsDoc + }) + } else { + await this.client.index({ + index: this.indexName, + id: fulltextId, + pipeline: 'attachment', + body: wsDoc + }) + } + return {} + } + + async update ( + ctx: MeasureContext, + workspaceId: WorkspaceUuid, + id: Ref, + update: Record + ): Promise { + await this.client.update({ + index: this.indexName, + id: this.getFulltextDocId(workspaceId, id), + body: { + doc: update + } + }) + + return {} + } + + async updateMany (ctx: MeasureContext, workspaceId: WorkspaceUuid, docs: IndexedDoc[]): Promise { + const parts = Array.from(docs) + while (parts.length > 0) { + const part = parts.splice(0, 500) + + const operations = part.flatMap((doc) => { + const wsDoc = { workspaceId, ...doc } + return [{ index: { _index: this.indexName, _id: this.getFulltextDocId(workspaceId, doc.id) } }, { ...wsDoc }] + }) + + const response = await this.client.bulk({ refresh: true, body: operations }) + if ((response as any).body.errors === true) { + const errors = response.body.items.filter((it: any) => it.index.error !== undefined) + const errorIds = new Set(errors.map((it: any) => it.index._id)) + const erroDocs = docs.filter((it) => errorIds.has(this.getFulltextDocId(workspaceId, it.id))) + // Collect only errors + const errs = Array.from( + errors.map((it: any) => { + return `${it.index.error.reason}: ${it.index.error.caused_by?.reason}` + }) + ).join('\n') + + console.error(`Failed to process bulk request: ${errs} ${JSON.stringify(erroDocs)}`) + } + } + return [] + } + + async updateByQuery ( + ctx: MeasureContext, + workspaceId: WorkspaceUuid, + query: DocumentQuery, + update: Record + ): Promise { + const elasticQuery: any = { + bool: { + must: [ + { + term: { + workspaceId + } + } + ] + } + } + + for (const [q, v] of Object.entries(query)) { + if (!q.startsWith('$')) { + if (typeof v === 'object') { + if (v.$in !== undefined) { + elasticQuery.bool.must.push({ + terms: { + [Object.hasOwn(mappings.properties, q) ? q : `${q}.keyword`]: v.$in + } + }) + } + } else { + elasticQuery.bool.must.push({ + term: { + [Object.hasOwn(mappings.properties, q) ? q : `${q}.keyword`]: { + value: v + } + } + }) + } + } + } + + await this.client.updateByQuery({ + index: this.indexName, + body: { + query: elasticQuery, + script: { + source: + 'for(int i = 0; i < params.updateFields.size(); i++) { ctx._source[params.updateFields[i].key] = params.updateFields[i].value }', + params: { + updateFields: Object.entries(update).map(([key, value]) => ({ key, value })) + }, + lang: 'painless' + } + } + }) + return [] + } + + async remove (ctx: MeasureContext, workspaceId: WorkspaceUuid, docs: Ref[]): Promise { + try { + const remaining = Array.from(docs) + while (remaining.length > 0) { + const part = remaining.splice(0, 5000) + await this.client.deleteByQuery( + { + index: this.indexName, + body: { + query: { + bool: { + must: [ + { + terms: { + _id: part.map((it) => this.getFulltextDocId(workspaceId, it)), + boost: 1.0 + } + }, + { + term: { + workspaceId + } + } + ] + } + }, + size: part.length + } + }, + undefined + ) + } + } catch (e: any) { + if (e instanceof osErr.ResponseError && e.meta.statusCode === 404) { + return + } + throw e + } + } + + async removeByQuery (ctx: MeasureContext, workspaceId: WorkspaceUuid, query: DocumentQuery): Promise { + const elasticQuery: any = { + bool: { + must: [ + { + term: { + workspaceId + } + } + ] + } + } + + for (const [q, v] of Object.entries(query)) { + if (!q.startsWith('$')) { + if (typeof v === 'object') { + if (v.$in !== undefined) { + elasticQuery.bool.must.push({ + terms: { + [Object.hasOwn(mappings.properties, q) ? q : `${q}.keyword`]: v.$in + } + }) + } + } else { + elasticQuery.bool.must.push({ + term: { + [Object.hasOwn(mappings.properties, q) ? q : `${q}.keyword`]: { + value: v + } + } + }) + } + } + } + try { + await this.client.deleteByQuery({ + index: this.indexName, + body: { + query: elasticQuery + } + }) + } catch (e: any) { + if (e instanceof osErr.ResponseError && e.meta.statusCode === 404) { + return + } + throw e + } + } + + async clean (ctx: MeasureContext, workspaceId: WorkspaceUuid): Promise { + try { + await this.client.deleteByQuery( + { + index: this.indexName, + body: { + query: { + bool: { + must: [ + { + term: { + workspaceId + } + } + ] + } + } + } + }, + undefined + ) + } catch (e: any) { + if (e instanceof osErr.ResponseError && e.meta.statusCode === 404) { + return + } + throw e + } + } + + async load (ctx: MeasureContext, workspaceId: WorkspaceUuid, docs: Ref[]): Promise { + const resp = await this.client.search({ + index: this.indexName, + body: { + query: { + bool: { + must: [ + { + terms: { + _id: docs.map((it) => this.getFulltextDocId(workspaceId, it)), + boost: 1.0 + } + }, + { + term: { + workspaceId + } + } + ] + } + }, + size: docs.length + } + }) + return Array.from( + resp.body.hits.hits.map((hit: any) => ({ ...hit._source, id: this.getDocId(workspaceId, hit._id) })) + ) + } +} + +/** + * @public + */ +export async function createOpenSearchAdapter (url: string): Promise { + const client = new Client({ + node: url + }) + const indexBaseName = getIndexName() + const indexVersion = getIndexVersion() + + return new OpenSearchAdapter(client, indexBaseName, indexVersion) +} diff --git a/foundations/server/packages/opensearch/src/index.ts b/foundations/server/packages/opensearch/src/index.ts new file mode 100644 index 00000000000..7189f833d5e --- /dev/null +++ b/foundations/server/packages/opensearch/src/index.ts @@ -0,0 +1,17 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021, 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export { createOpenSearchAdapter } from './adapter' diff --git a/foundations/server/packages/opensearch/tsconfig.json b/foundations/server/packages/opensearch/tsconfig.json new file mode 100644 index 00000000000..7d78e05abb4 --- /dev/null +++ b/foundations/server/packages/opensearch/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "lib", "dist", "types", "bundle"] +} diff --git a/foundations/server/packages/typesense/.eslintrc.js b/foundations/server/packages/typesense/.eslintrc.js new file mode 100644 index 00000000000..72235dc2833 --- /dev/null +++ b/foundations/server/packages/typesense/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/foundations/server/packages/typesense/package.json b/foundations/server/packages/typesense/package.json new file mode 100644 index 00000000000..7b301b93eb3 --- /dev/null +++ b/foundations/server/packages/typesense/package.json @@ -0,0 +1,69 @@ +{ + "name": "@hcengineering/typesense", + "version": "0.7.17", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "src/**/*", + "!lib/**/*.test.*", + "!lib/**/*.spec.*", + "!lib/__tests__/**/*", + "!lib/__test__/**/*", + "!lib/tests/**/*", + "!types/**/*.test.*", + "!types/**/*.spec.*", + "!types/__tests__/**/*", + "!types/__test__/**/*", + "!types/tests/**/*", + "!src/**/*.test.*", + "!src/**/*.spec.*", + "!src/__tests__/**/*", + "!src/__test__/**/*", + "!src/tests/**/*" + ], + "author": "Anticrm Platform Contributors", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "format": "format src", + "test": "jest --passWithNoTests --silent --forceExit", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent --forceExit", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "workspace:^0.7.19", + "@typescript-eslint/eslint-plugin": "^6.21.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "@typescript-eslint/parser": "^6.21.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.6.2", + "typescript": "^5.9.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5", + "eslint-plugin-svelte": "^2.35.1" + }, + "dependencies": { + "@hcengineering/core": "workspace:^0.7.24", + "@hcengineering/platform": "workspace:^0.7.19", + "@hcengineering/server-core": "workspace:^0.7.18", + "@hcengineering/analytics": "workspace:^0.7.17", + "typesense": "^2.0.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/hcengineering/huly.server.git" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/foundations/server/packages/typesense/src/adapter.ts b/foundations/server/packages/typesense/src/adapter.ts new file mode 100644 index 00000000000..c7c459adf81 --- /dev/null +++ b/foundations/server/packages/typesense/src/adapter.ts @@ -0,0 +1,598 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import { + type Class, + type Doc, + type DocumentQuery, + type MeasureContext, + type Ref, + type SearchOptions, + type SearchQuery, + type TxResult, + type WorkspaceUuid +} from '@hcengineering/core' +import type { FullTextAdapter, IndexedDoc, SearchScoring, SearchStringResult } from '@hcengineering/server-core' +import serverCore from '@hcengineering/server-core' + +import { Client as TypesenseClient } from 'typesense' +import type { CollectionFieldSchema } from 'typesense/lib/Typesense/Collection' + +import { getMetadata } from '@hcengineering/platform' + +const DEFAULT_LIMIT = 200 +const BATCH_SIZE = 500 + +function getIndexName (): string { + return getMetadata(serverCore.metadata.ElasticIndexName) ?? 'storage_index' +} + +function getIndexVersion (): string { + return getMetadata(serverCore.metadata.ElasticIndexVersion) ?? 'v2' +} + +/** Fields that are defined in the schema and are facet-capable (keyword-like). */ +const FACET_FIELDS = new Set([ + 'id', + 'workspaceId', + '_class', + 'space', + 'attachedTo', + 'attachedToClass', + 'modifiedBy', + 'core:class:Doc%createdBy', + 'core:class:Doc%modifiedBy' +]) + +function buildCollectionSchema (collectionName: string): { + name: string + fields: CollectionFieldSchema[] + enable_nested_fields: boolean +} { + const fields: CollectionFieldSchema[] = [ + { name: 'id', type: 'string', facet: true }, + { name: 'workspaceId', type: 'string', facet: true }, + { name: '_class', type: 'string[]', facet: true }, + { name: 'space', type: 'string', facet: true }, + { name: 'attachedTo', type: 'string', facet: true, optional: true }, + { name: 'attachedToClass', type: 'string', facet: true, optional: true }, + { name: 'searchTitle', type: 'string', optional: true }, + { name: 'searchShortTitle', type: 'string', optional: true }, + { name: 'fulltextSummary', type: 'string', optional: true }, + { name: 'modifiedBy', type: 'string', facet: true, optional: true }, + { name: 'modifiedOn', type: 'int64', optional: true }, + { name: 'core:class:Doc%createdBy', type: 'string', facet: true, optional: true }, + { name: 'core:class:Doc%createdOn', type: 'int64', optional: true }, + { name: 'core:class:Doc%modifiedBy', type: 'string', facet: true, optional: true }, + { name: 'core:class:Doc%modifiedOn', type: 'int64', optional: true }, + // Catch-all for dynamic fields + { name: '.*', type: 'auto' } + ] + + return { + name: collectionName, + fields, + enable_nested_fields: true + } +} + +function isTypesenseError (err: any): boolean { + return err?.httpStatus !== undefined || err?.name === 'TypesenseError' +} + +function isConnectionError (err: any): boolean { + if (err?.name === 'ConnectionError') return true + if (err?.code === 'ECONNREFUSED' || err?.code === 'ENOTFOUND') return true + if (err?.message?.includes?.('ECONNREFUSED') === true) return true + return false +} + +/** Escape a value for use inside a Typesense filter_by backtick-quoted string. */ +function escapeFilterValue (val: string): string { + return val.replace(/`/g, '\\`') +} + +/** + * Build a Typesense filter_by string from a workspace ID and a Huly DocumentQuery. + * Skips `$`-prefixed keys (like `$search`). + */ +function buildQueryFilter (workspaceId: WorkspaceUuid, query?: DocumentQuery): string { + const parts: string[] = [`workspaceId:=${workspaceId as string}`] + + if (query !== undefined) { + for (const [q, v] of Object.entries(query)) { + if (q.startsWith('$')) continue + if (typeof v === 'object' && v !== null) { + if (v.$in !== undefined && Array.isArray(v.$in)) { + parts.push(`${q}:=[${v.$in.map((val: string) => `\`${escapeFilterValue(val)}\``).join(',')}]`) + } + } else { + parts.push(`${q}:=\`${escapeFilterValue(String(v))}\``) + } + } + } + + return parts.join(' && ') +} + +class TypesenseAdapter implements FullTextAdapter { + private readonly getFulltextDocId: (workspaceId: WorkspaceUuid, doc: Ref) => string + private readonly getDocId: (workspaceId: WorkspaceUuid, fulltext: string) => Ref + private readonly collectionName: string + + constructor ( + private readonly client: TypesenseClient, + private readonly indexBaseName: string, + readonly indexVersion: string + ) { + this.collectionName = `${indexBaseName}_${indexVersion}` + this.getFulltextDocId = (workspaceId, doc) => `${doc as string}@${workspaceId as string}` + this.getDocId = (workspaceId, fulltext) => fulltext.slice(0, -1 * ((workspaceId as string).length + 1)) as Ref + } + + async initMapping (ctx: MeasureContext, _field?: { key: string, dims: number }): Promise { + const collectionName = this.collectionName + try { + const collections = await this.client.collections().retrieve() + const matchingCollections = collections.filter((c: any) => c.name.startsWith(`${this.indexBaseName}_`)) + const existingCollection = matchingCollections.find((c: any) => c.name === collectionName) + const oldCollections = matchingCollections.filter((c: any) => c.name !== collectionName) + + for (const old of oldCollections) { + await ctx.with('delete-old-collection', {}, async () => { + await this.client.collections(old.name).delete() + }) + } + + let shouldRecreate = false + if (existingCollection !== undefined) { + const existingFields = new Set((existingCollection as any).fields?.map((f: any) => f.name) ?? []) + const schema = buildCollectionSchema(collectionName) + for (const field of schema.fields) { + if (field.name !== '.*' && !existingFields.has(field.name)) { + shouldRecreate = true + break + } + } + } + + if (shouldRecreate && existingCollection !== undefined) { + await ctx.with('delete-collection', {}, async () => { + await this.client.collections(collectionName).delete() + }) + } + + if (existingCollection === undefined || shouldRecreate) { + const schema = buildCollectionSchema(collectionName) + await ctx.with('create-collection', { collectionName }, async () => { + await this.client.collections().create(schema as any) + }) + } + } catch (err: any) { + if (isConnectionError(err)) { + ctx.warn('Typesense DB is not available') + } + Analytics.handleError(err) + ctx.error(err) + return false + } + return true + } + + async close (): Promise { + // Typesense client does not require explicit close + } + + async searchString ( + ctx: MeasureContext, + workspaceId: WorkspaceUuid, + query: SearchQuery, + options: SearchOptions & { scoring?: SearchScoring[] } + ): Promise { + try { + const filterParts: string[] = [`workspaceId:=${workspaceId as string}`] + + if (query.spaces !== undefined && query.spaces.length > 0) { + filterParts.push(`space:=[${query.spaces.map((s) => `\`${escapeFilterValue(s)}\``).join(',')}]`) + } + if (query.classes !== undefined && query.classes.length > 0) { + filterParts.push(`_class:=[${query.classes.map((c) => `\`${escapeFilterValue(c)}\``).join(',')}]`) + } + + // Require searchTitle to exist (non-empty) + filterParts.push('searchTitle:!=""') + + // Scoring: boost documents where a facet field matches a specific value. + // Typesense doesn't support per-term boosting like ES function_score, + // so we use optional filter clauses that prefer matching documents. + if (options.scoring !== undefined && options.scoring.length > 0) { + const optionalParts: string[] = [] + for (const scoring of options.scoring) { + if (FACET_FIELDS.has(scoring.attr)) { + optionalParts.push(`${scoring.attr}:=\`${escapeFilterValue(String(scoring.value))}\``) + } + } + if (optionalParts.length > 0) { + // Use optional filter_by syntax: main filters && (optional1 || optional2) + // Documents matching optional filters rank higher via _text_match + filter proximity + filterParts.push(`(${optionalParts.join(' || ')})`) + } + } + + const filterBy = filterParts.join(' && ') + + const searchParams: any = { + q: query.query, + query_by: 'searchTitle,searchShortTitle,fulltextSummary', + query_by_weights: '50,50,1', + filter_by: filterBy, + limit: options.limit ?? DEFAULT_LIMIT, + prefix: 'true,true,false', + sort_by: '_text_match:desc' + } + + const result = await this.client.collections(this.collectionName).documents().search(searchParams) + + const resp: SearchStringResult = { docs: [] } + if (result.found !== undefined) { + resp.total = result.found + } + if (result.hits !== undefined) { + resp.docs = result.hits.map((hit: any) => { + const doc = hit.document + return { + ...doc, + id: this.getDocId(workspaceId, doc.id), + _score: hit.text_match ?? 0 + } + }) + } + + return resp + } catch (err: any) { + if (isConnectionError(err)) { + ctx.warn('Typesense DB is not available') + return { docs: [] } + } + Analytics.handleError(err) + ctx.error('Typesense error', { error: err }) + return { docs: [] } + } + } + + async search ( + ctx: MeasureContext, + workspaceId: WorkspaceUuid, + _classes: Ref>[], + query: DocumentQuery, + size: number | undefined, + from: number | undefined + ): Promise { + if (query.$search === undefined) return [] + try { + const filterParts: string[] = [`workspaceId:=${workspaceId as string}`] + + if (_classes.length > 0) { + filterParts.push(`_class:=[${_classes.map((c) => `\`${escapeFilterValue(c as string)}\``).join(',')}]`) + } + + // In Elastic, additional query fields are soft boosts (should clauses). + // Typesense has no direct equivalent, so we add them as optional filter + // clauses — documents matching them rank higher but aren't excluded. + const optionalParts: string[] = [] + for (const [q, v] of Object.entries(query)) { + if (q.startsWith('$')) continue + if (typeof v === 'object' && v !== null) { + if (v.$in !== undefined && Array.isArray(v.$in)) { + optionalParts.push(`${q}:=[${v.$in.map((val: string) => `\`${escapeFilterValue(val)}\``).join(',')}]`) + } + } else { + optionalParts.push(`${q}:=\`${escapeFilterValue(String(v))}\``) + } + } + if (optionalParts.length > 0) { + filterParts.push(`(${optionalParts.join(' || ')})`) + } + + const filterBy = filterParts.join(' && ') + + const searchParams: any = { + q: query.$search, + query_by: 'searchTitle,searchShortTitle,fulltextSummary', + query_by_weights: '50,50,1', + filter_by: filterBy, + limit: size ?? DEFAULT_LIMIT, + offset: from ?? 0, + sort_by: '_text_match:desc', + prefix: 'true,true,false' + } + + const result = await ctx.with( + 'search', + {}, + async () => await this.client.collections(this.collectionName).documents().search(searchParams), + { _classes, size, from, query: searchParams } + ) + + if (result.hits === undefined) return [] + return result.hits.map((hit: any) => { + const doc = hit.document + return { + ...doc, + id: this.getDocId(workspaceId, doc.id), + _score: hit.text_match ?? 0 + } + }) + } catch (err: any) { + if (isConnectionError(err)) { + ctx.warn('Typesense DB is not available') + return [] + } + ctx.error('Typesense error', { error: err }) + Analytics.handleError(err) + return [] + } + } + + async index (ctx: MeasureContext, workspaceId: WorkspaceUuid, doc: IndexedDoc): Promise { + if (doc.data !== undefined) { + ctx.warn('Binary attachment content (doc.data) is not supported in Typesense; relying on Rekoni for extraction', { + docId: doc.id + }) + } + + const fulltextId = this.getFulltextDocId(workspaceId, doc.id) + const tsDoc: Record = { + ...doc, + id: fulltextId, + workspaceId + } + // Remove binary data — Typesense cannot process it + delete tsDoc.data + + try { + await this.client.collections(this.collectionName).documents().upsert(tsDoc) + } catch (err: any) { + if (isConnectionError(err)) { + ctx.warn('Typesense DB is not available') + } else { + Analytics.handleError(err) + ctx.error('Typesense index error', { error: err, docId: doc.id }) + } + } + return {} + } + + async update ( + ctx: MeasureContext, + workspaceId: WorkspaceUuid, + id: Ref, + update: Record + ): Promise { + const fulltextId = this.getFulltextDocId(workspaceId, id) + try { + await this.client.collections(this.collectionName).documents(fulltextId).update(update) + } catch (err: any) { + if (isTypesenseError(err) && err.httpStatus === 404) { + return {} + } + if (isConnectionError(err)) { + ctx.warn('Typesense DB is not available') + } else { + Analytics.handleError(err) + ctx.error('Typesense update error', { error: err, docId: id }) + } + } + return {} + } + + async updateMany (ctx: MeasureContext, workspaceId: WorkspaceUuid, docs: IndexedDoc[]): Promise { + const parts = Array.from(docs) + while (parts.length > 0) { + const batch = parts.splice(0, BATCH_SIZE) + const jsonlLines = batch + .map((doc) => { + const tsDoc: Record = { + ...doc, + id: this.getFulltextDocId(workspaceId, doc.id), + workspaceId + } + delete tsDoc.data + return JSON.stringify(tsDoc) + }) + .join('\n') + + try { + const results = await this.client + .collections(this.collectionName) + .documents() + .import(jsonlLines, { action: 'upsert' }) + + const errors = ( + typeof results === 'string' ? results.split('\n').map((l: string) => JSON.parse(l)) : results + ).filter((r: any) => r.success === false) + + if (errors.length > 0) { + const errorMessages = errors.map((e: any) => e.error).join('\n') + const errorDocs = errors.map((e: any) => e.document) + console.error(`Failed to process bulk request: ${errorMessages} ${JSON.stringify(errorDocs)}`) + } + } catch (err: any) { + if (isConnectionError(err)) { + ctx.warn('Typesense DB is not available') + } else { + Analytics.handleError(err) + ctx.error('Typesense updateMany error', { error: err }) + } + } + } + return [] + } + + async updateByQuery ( + ctx: MeasureContext, + workspaceId: WorkspaceUuid, + query: DocumentQuery, + update: Record + ): Promise { + try { + const filterBy = buildQueryFilter(workspaceId, query) + + // Paginate through all matching documents + let page = 1 + const perPage = 250 + const allDocs: any[] = [] + + for (;;) { + const result = await this.client.collections(this.collectionName).documents().search({ + q: '*', + filter_by: filterBy, + per_page: perPage, + page + }) + const hits = result.hits ?? [] + if (hits.length === 0) break + allDocs.push(...hits.map((h: any) => h.document)) + if (allDocs.length >= (result.found ?? 0)) break + page++ + } + + if (allDocs.length === 0) return [] + + // Apply update and batch upsert + const updatedDocs = allDocs.map((doc: any) => ({ ...doc, ...update })) + const remaining = Array.from(updatedDocs) + while (remaining.length > 0) { + const batch = remaining.splice(0, BATCH_SIZE) + const jsonlLines = batch.map((doc: any) => JSON.stringify(doc)).join('\n') + await this.client.collections(this.collectionName).documents().import(jsonlLines, { action: 'upsert' }) + } + } catch (err: any) { + if (isConnectionError(err)) { + ctx.warn('Typesense DB is not available') + } else { + Analytics.handleError(err) + ctx.error('Typesense updateByQuery error', { error: err }) + } + } + return [] + } + + async remove (ctx: MeasureContext, workspaceId: WorkspaceUuid, docs: Ref[]): Promise { + try { + const remaining = Array.from(docs) + while (remaining.length > 0) { + const batch = remaining.splice(0, 5000) + const ids = batch.map((it) => this.getFulltextDocId(workspaceId, it)) + const filterBy = `id:=[${ids.map((id) => `\`${escapeFilterValue(id)}\``).join(',')}] && workspaceId:=${workspaceId as string}` + await this.client.collections(this.collectionName).documents().delete({ filter_by: filterBy }) + } + } catch (err: any) { + if (isTypesenseError(err) && err.httpStatus === 404) { + return + } + throw err + } + } + + async removeByQuery (ctx: MeasureContext, workspaceId: WorkspaceUuid, query: DocumentQuery): Promise { + const filterBy = buildQueryFilter(workspaceId, query) + try { + await this.client.collections(this.collectionName).documents().delete({ filter_by: filterBy }) + } catch (err: any) { + if (isTypesenseError(err) && err.httpStatus === 404) { + return + } + throw err + } + } + + async clean (ctx: MeasureContext, workspaceId: WorkspaceUuid): Promise { + try { + const filterBy = `workspaceId:=${workspaceId as string}` + await this.client.collections(this.collectionName).documents().delete({ filter_by: filterBy }) + } catch (err: any) { + if (isTypesenseError(err) && err.httpStatus === 404) { + return + } + throw err + } + } + + async load (ctx: MeasureContext, workspaceId: WorkspaceUuid, docs: Ref[]): Promise { + try { + const ids = docs.map((it) => this.getFulltextDocId(workspaceId, it)) + const filterBy = `id:=[${ids.map((id) => `\`${escapeFilterValue(id)}\``).join(',')}] && workspaceId:=${workspaceId as string}` + + const result = await this.client.collections(this.collectionName).documents().search({ + q: '*', + filter_by: filterBy, + per_page: docs.length + }) + + if (result.hits === undefined) return [] + return result.hits.map((hit: any) => { + const doc = hit.document + return { + ...doc, + id: this.getDocId(workspaceId, doc.id) + } + }) + } catch (err: any) { + if (isConnectionError(err)) { + ctx.warn('Typesense DB is not available') + return [] + } + Analytics.handleError(err) + ctx.error('Typesense load error', { error: err }) + return [] + } + } +} + +/** + * @public + * + * Creates a Typesense fulltext adapter from a URL like: + * http://typesense:8108?apiKey=xyz + */ +export async function createTypesenseAdapter (url: string): Promise { + const parsed = new URL(url) + const apiKey = parsed.searchParams.get('apiKey') ?? parsed.searchParams.get('apikey') ?? '' + const protocol = parsed.protocol.replace(':', '') + const host = parsed.hostname + const port = parsed.port !== '' ? parseInt(parsed.port, 10) : protocol === 'https' ? 443 : 8108 + + const client = new TypesenseClient({ + nodes: [ + { + host, + port, + protocol + } + ], + apiKey, + connectionTimeoutSeconds: 10, + retryIntervalSeconds: 0.1, + numRetries: 3 + }) + + const indexBaseName = getIndexName() + const indexVersion = getIndexVersion() + + return new TypesenseAdapter(client, indexBaseName, indexVersion) +} diff --git a/foundations/server/packages/typesense/src/index.ts b/foundations/server/packages/typesense/src/index.ts new file mode 100644 index 00000000000..acb9895fc7c --- /dev/null +++ b/foundations/server/packages/typesense/src/index.ts @@ -0,0 +1,17 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021, 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export { createTypesenseAdapter } from './adapter' diff --git a/foundations/server/packages/typesense/tsconfig.json b/foundations/server/packages/typesense/tsconfig.json new file mode 100644 index 00000000000..7d78e05abb4 --- /dev/null +++ b/foundations/server/packages/typesense/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "lib", "dist", "types", "bundle"] +} From 52b1cfcd12ec69e900f0335752b6a1608abcd519 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Fri, 27 Mar 2026 08:12:33 -0400 Subject: [PATCH 02/11] feat: configurable fulltext backend via FULLTEXT_BACKEND env var Wire up the new OpenSearch and Typesense adapters in the fulltext pod. The FULLTEXT_BACKEND env var selects the adapter at startup: - "elastic" (default): existing Elasticsearch 7.x - "opensearch": OpenSearch 2.x (API-compatible drop-in) - "typesense": Typesense (lightweight C++ engine) Register both new packages in rush.json and add them as fulltext pod dependencies. ELASTIC_INDEX_NAME now defaults to 'huly_storage_index' instead of requiring it for all backends. Signed-off-by: Don Kendall --- pods/fulltext/package.json | 2 ++ pods/fulltext/src/index.ts | 29 ++++++++++++++++++++++------- rush.json | 10 ++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/pods/fulltext/package.json b/pods/fulltext/package.json index 1fe66b5e393..1da06155f28 100644 --- a/pods/fulltext/package.json +++ b/pods/fulltext/package.json @@ -69,6 +69,8 @@ "@koa/cors": "^5.0.0", "@hcengineering/server-indexer": "workspace:^0.7.0", "@hcengineering/elastic": "workspace:^0.7.16", + "@hcengineering/opensearch": "workspace:^0.7.17", + "@hcengineering/typesense": "workspace:^0.7.17", "@hcengineering/server-collaboration": "workspace:^0.7.0", "@hcengineering/middleware": "workspace:^0.7.21", "@hcengineering/server-client": "workspace:^0.7.16", diff --git a/pods/fulltext/src/index.ts b/pods/fulltext/src/index.ts index 89bf4250851..daec00ecfe8 100644 --- a/pods/fulltext/src/index.ts +++ b/pods/fulltext/src/index.ts @@ -16,10 +16,16 @@ import { Analytics } from '@hcengineering/analytics' import { configureAnalytics, createOpenTelemetryMetricsContext, SplitLogger } from '@hcengineering/analytics-service' import { newMetrics, type Tx } from '@hcengineering/core' -import { initStatisticsContext, type StorageConfiguration } from '@hcengineering/server-core' +import { + initStatisticsContext, + type StorageConfiguration, + type FullTextAdapterFactory +} from '@hcengineering/server-core' import { join } from 'path' import { createElasticAdapter } from '@hcengineering/elastic' +import { createOpenSearchAdapter } from '@hcengineering/opensearch' +import { createTypesenseAdapter } from '@hcengineering/typesense' import { getPlatformQueue } from '@hcengineering/kafka' import { setMetadata } from '@hcengineering/platform' import { createRekoniAdapter, type FulltextDBConfiguration } from '@hcengineering/server-indexer' @@ -74,9 +80,22 @@ if (rekoniUrl === undefined) { process.exit(1) } +const fulltextBackend = (process.env.FULLTEXT_BACKEND ?? 'elastic').toLowerCase() +const fulltextAdapterFactories: Record = { + elastic: createElasticAdapter, + opensearch: createOpenSearchAdapter, + typesense: createTypesenseAdapter +} +const fulltextFactory = fulltextAdapterFactories[fulltextBackend] +if (fulltextFactory === undefined) { + console.error(`Unknown FULLTEXT_BACKEND: "${fulltextBackend}". Supported: elastic, opensearch, typesense`) + process.exit(1) +} +console.info(`Using fulltext backend: ${fulltextBackend}`) + const config: FulltextDBConfiguration = { fulltextAdapter: { - factory: createElasticAdapter, + factory: fulltextFactory, url: fullTextDbURL }, contentAdapters: { @@ -89,11 +108,7 @@ const config: FulltextDBConfiguration = { defaultContentAdapter: 'Rekoni' } -const elasticIndexName = process.env.ELASTIC_INDEX_NAME -if (elasticIndexName === undefined) { - console.log('Please provide ELASTIC_INDEX_NAME') - process.exit(1) -} +const elasticIndexName = process.env.ELASTIC_INDEX_NAME ?? 'huly_storage_index' const servicePort = parseInt(process.env.PORT ?? '4700') metricsContext.info('Starting stats service') diff --git a/rush.json b/rush.json index 439e02a1b28..8d3a750f11c 100644 --- a/rush.json +++ b/rush.json @@ -547,6 +547,16 @@ "projectFolder": "foundations/server/packages/elastic", "shouldPublish": true }, + { + "packageName": "@hcengineering/opensearch", + "projectFolder": "foundations/server/packages/opensearch", + "shouldPublish": true + }, + { + "packageName": "@hcengineering/typesense", + "projectFolder": "foundations/server/packages/typesense", + "shouldPublish": true + }, { "packageName": "@hcengineering/kafka", "projectFolder": "foundations/server/packages/kafka", From d62168a1da25469c74e16287a5d6700370de9df3 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Fri, 27 Mar 2026 21:21:20 -0400 Subject: [PATCH 03/11] fix: read version.txt before git tags in show_version.js The script only read version.txt when git describe succeeded, falling back to hardcoded "0.6.0" when no tags exist (e.g. in forks). This caused the model version to be 0.6.0 instead of the value in version.txt, making the fulltext indexer skip all workspaces. Now reads version.txt first, falls back to git tags, then "0.6.0". Signed-off-by: Don Kendall --- common/scripts/show_version.js | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/common/scripts/show_version.js b/common/scripts/show_version.js index 7a49a74d5fa..0ec1ba25e61 100644 --- a/common/scripts/show_version.js +++ b/common/scripts/show_version.js @@ -18,21 +18,26 @@ const path = require('path') const exec = require('child_process').exec function main() { + // Read version from version.txt first, fall back to git tag, then default + let version + try { + const versionFilePath = path.resolve(__dirname, 'version.txt') + version = fs.readFileSync(versionFilePath, 'utf8').trim() + } catch (error) { + // version.txt not found + } + + if (version) { + console.log(version) + return + } + exec('git describe --tags --abbrev=0', (err, stdout) => { if (err !== null) { console.log('"0.6.0"') return } - // Take version from file - let version - try { - const versionFilePath = path.resolve(__dirname, 'version.txt') - version = fs.readFileSync(versionFilePath, 'utf8').trim() - } catch (error) { - version = '"0.6.0"' - } - - console.log(version) + console.log(`"${stdout.trim()}"`) }) } From d63a6d3b913b57ef729be7eeb6be1656d3b8036a Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Sun, 29 Mar 2026 11:50:07 -0400 Subject: [PATCH 04/11] feat: tune Typesense adapter for better search parity with ES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Increase query_by_weights to 100,100,1 (matching ES boost ratios) - Enable num_typos: 2,2,1 for broader fuzzy matching - Add typo_tokens_threshold: 1 and drop_tokens_threshold: 1 - Use text_match_info.best_field_score for more granular scoring - Add sanitizeDoc() to coerce dynamic *_fields to string arrays (fixes 5 import errors from searchIcon_fields type mismatch) Search comparison results after tuning: - ES↔OpenSearch: 100% identical on all queries - ES↔Typesense: same top results, 5/5 overlap on class filter, 3/5 on prefix/fuzzy (different ranking, same docs found) Signed-off-by: Don Kendall --- .../server/packages/opensearch/package.json | 1 - .../server/packages/typesense/src/adapter.ts | 119 ++++++++---------- pods/fulltext/src/index.ts | 6 +- 3 files changed, 60 insertions(+), 66 deletions(-) diff --git a/foundations/server/packages/opensearch/package.json b/foundations/server/packages/opensearch/package.json index 005714f2d00..618f31b1461 100644 --- a/foundations/server/packages/opensearch/package.json +++ b/foundations/server/packages/opensearch/package.json @@ -27,7 +27,6 @@ "author": "Anticrm Platform Contributors", "license": "EPL-2.0", "scripts": { - "init": "ts-node src/__init.ts", "build": "compile", "build:watch": "compile", "format": "format src", diff --git a/foundations/server/packages/typesense/src/adapter.ts b/foundations/server/packages/typesense/src/adapter.ts index c7c459adf81..8f972b454e9 100644 --- a/foundations/server/packages/typesense/src/adapter.ts +++ b/foundations/server/packages/typesense/src/adapter.ts @@ -45,19 +45,6 @@ function getIndexVersion (): string { return getMetadata(serverCore.metadata.ElasticIndexVersion) ?? 'v2' } -/** Fields that are defined in the schema and are facet-capable (keyword-like). */ -const FACET_FIELDS = new Set([ - 'id', - 'workspaceId', - '_class', - 'space', - 'attachedTo', - 'attachedToClass', - 'modifiedBy', - 'core:class:Doc%createdBy', - 'core:class:Doc%modifiedBy' -]) - function buildCollectionSchema (collectionName: string): { name: string fields: CollectionFieldSchema[] @@ -101,11 +88,34 @@ function isConnectionError (err: any): boolean { return false } +/** Fields that are known to be string arrays in the schema. */ +const ARRAY_FIELDS = new Set(['_class']) + /** Escape a value for use inside a Typesense filter_by backtick-quoted string. */ function escapeFilterValue (val: string): string { return val.replace(/`/g, '\\`') } +/** + * Sanitize a document for Typesense upsert. + * - Removes binary `data` field + * - Coerces fields ending in `_fields` to string arrays (dynamic ES fields) + * - Ensures `_class` is always an array + */ +function sanitizeDoc (doc: Record): Record { + const result: Record = {} + for (const [key, value] of Object.entries(doc)) { + if (key === 'data') continue + // Coerce _class and *_fields to arrays + if (ARRAY_FIELDS.has(key) || key.endsWith('_fields')) { + result[key] = Array.isArray(value) ? value : value != null ? [String(value)] : [] + } else { + result[key] = value + } + } + return result +} + /** * Build a Typesense filter_by string from a workspace ID and a Huly DocumentQuery. * Skips `$`-prefixed keys (like `$search`). @@ -216,32 +226,25 @@ class TypesenseAdapter implements FullTextAdapter { // Require searchTitle to exist (non-empty) filterParts.push('searchTitle:!=""') - // Scoring: boost documents where a facet field matches a specific value. - // Typesense doesn't support per-term boosting like ES function_score, - // so we use optional filter clauses that prefer matching documents. - if (options.scoring !== undefined && options.scoring.length > 0) { - const optionalParts: string[] = [] - for (const scoring of options.scoring) { - if (FACET_FIELDS.has(scoring.attr)) { - optionalParts.push(`${scoring.attr}:=\`${escapeFilterValue(String(scoring.value))}\``) - } - } - if (optionalParts.length > 0) { - // Use optional filter_by syntax: main filters && (optional1 || optional2) - // Documents matching optional filters rank higher via _text_match + filter proximity - filterParts.push(`(${optionalParts.join(' || ')})`) - } - } + // Scoring: ES uses function_score with should clauses (soft boosts). + // Typesense has no equivalent — filter_by is always mandatory (AND). + // We skip scoring filters entirely and rely on text_match ranking, + // which already prioritizes title matches via query_by_weights. + // Adding scoring fields to filter_by would incorrectly EXCLUDE + // documents that don't match, instead of just ranking them lower. const filterBy = filterParts.join(' && ') const searchParams: any = { q: query.query, query_by: 'searchTitle,searchShortTitle,fulltextSummary', - query_by_weights: '50,50,1', + query_by_weights: '100,100,1', filter_by: filterBy, limit: options.limit ?? DEFAULT_LIMIT, prefix: 'true,true,false', + num_typos: '2,2,1', + typo_tokens_threshold: 1, + drop_tokens_threshold: 1, sort_by: '_text_match:desc' } @@ -257,7 +260,7 @@ class TypesenseAdapter implements FullTextAdapter { return { ...doc, id: this.getDocId(workspaceId, doc.id), - _score: hit.text_match ?? 0 + _score: hit.text_match_info?.best_field_score ?? hit.text_match ?? 0 } }) } @@ -291,34 +294,24 @@ class TypesenseAdapter implements FullTextAdapter { } // In Elastic, additional query fields are soft boosts (should clauses). - // Typesense has no direct equivalent, so we add them as optional filter - // clauses — documents matching them rank higher but aren't excluded. - const optionalParts: string[] = [] - for (const [q, v] of Object.entries(query)) { - if (q.startsWith('$')) continue - if (typeof v === 'object' && v !== null) { - if (v.$in !== undefined && Array.isArray(v.$in)) { - optionalParts.push(`${q}:=[${v.$in.map((val: string) => `\`${escapeFilterValue(val)}\``).join(',')}]`) - } - } else { - optionalParts.push(`${q}:=\`${escapeFilterValue(String(v))}\``) - } - } - if (optionalParts.length > 0) { - filterParts.push(`(${optionalParts.join(' || ')})`) - } + // Typesense filter_by is always mandatory (AND), so adding these as + // filters would incorrectly exclude non-matching documents instead of + // just ranking them lower. We skip them and rely on text_match ranking. const filterBy = filterParts.join(' && ') const searchParams: any = { q: query.$search, query_by: 'searchTitle,searchShortTitle,fulltextSummary', - query_by_weights: '50,50,1', + query_by_weights: '100,100,1', filter_by: filterBy, limit: size ?? DEFAULT_LIMIT, offset: from ?? 0, sort_by: '_text_match:desc', - prefix: 'true,true,false' + prefix: 'true,true,false', + num_typos: '2,2,1', + typo_tokens_threshold: 1, + drop_tokens_threshold: 1 } const result = await ctx.with( @@ -334,7 +327,7 @@ class TypesenseAdapter implements FullTextAdapter { return { ...doc, id: this.getDocId(workspaceId, doc.id), - _score: hit.text_match ?? 0 + _score: hit.text_match_info?.best_field_score ?? hit.text_match ?? 0 } }) } catch (err: any) { @@ -356,13 +349,7 @@ class TypesenseAdapter implements FullTextAdapter { } const fulltextId = this.getFulltextDocId(workspaceId, doc.id) - const tsDoc: Record = { - ...doc, - id: fulltextId, - workspaceId - } - // Remove binary data — Typesense cannot process it - delete tsDoc.data + const tsDoc = sanitizeDoc({ ...doc, id: fulltextId, workspaceId }) try { await this.client.collections(this.collectionName).documents().upsert(tsDoc) @@ -406,12 +393,7 @@ class TypesenseAdapter implements FullTextAdapter { const batch = parts.splice(0, BATCH_SIZE) const jsonlLines = batch .map((doc) => { - const tsDoc: Record = { - ...doc, - id: this.getFulltextDocId(workspaceId, doc.id), - workspaceId - } - delete tsDoc.data + const tsDoc = sanitizeDoc({ ...doc, id: this.getFulltextDocId(workspaceId, doc.id), workspaceId }) return JSON.stringify(tsDoc) }) .join('\n') @@ -479,7 +461,16 @@ class TypesenseAdapter implements FullTextAdapter { while (remaining.length > 0) { const batch = remaining.splice(0, BATCH_SIZE) const jsonlLines = batch.map((doc: any) => JSON.stringify(doc)).join('\n') - await this.client.collections(this.collectionName).documents().import(jsonlLines, { action: 'upsert' }) + const results = await this.client + .collections(this.collectionName) + .documents() + .import(jsonlLines, { action: 'upsert' }) + const errors = ( + typeof results === 'string' ? results.split('\n').map((l: string) => JSON.parse(l)) : results + ).filter((r: any) => r.success === false) + if (errors.length > 0) { + console.error(`updateByQuery upsert errors: ${errors.map((e: any) => e.error).join('; ')}`) + } } } catch (err: any) { if (isConnectionError(err)) { diff --git a/pods/fulltext/src/index.ts b/pods/fulltext/src/index.ts index daec00ecfe8..579fbd199ef 100644 --- a/pods/fulltext/src/index.ts +++ b/pods/fulltext/src/index.ts @@ -108,7 +108,11 @@ const config: FulltextDBConfiguration = { defaultContentAdapter: 'Rekoni' } -const elasticIndexName = process.env.ELASTIC_INDEX_NAME ?? 'huly_storage_index' +const elasticIndexName = process.env.ELASTIC_INDEX_NAME +if (elasticIndexName === undefined) { + console.error('ELASTIC_INDEX_NAME should be specified') + process.exit(1) +} const servicePort = parseInt(process.env.PORT ?? '4700') metricsContext.info('Starting stats service') From 67c2a8ba08eaacee08dd67814e8afe4475bd7da8 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Sun, 29 Mar 2026 18:47:48 -0400 Subject: [PATCH 05/11] feat: auto-reindex on empty Typesense backend startup When the fulltext pod starts and the Typesense collection has 0 documents (e.g. after pod restart with emptyDir), automatically enumerate all active workspaces and queue a fullReindex event for each. This makes Typesense self-healing without requiring a PVC. - Add optional getDocCount() to FullTextAdapter interface - Implement getDocCount() in TypesenseAdapter via collections().retrieve() - Add checkAndTriggerReindex() to WorkspaceManager.startIndexer() - Elastic/OpenSearch backends skip the check (no getDocCount method) Signed-off-by: Don Kendall --- foundations/server/packages/core/src/types.ts | 6 ++++ .../server/packages/typesense/src/adapter.ts | 15 +++++++++ pods/fulltext/src/manager.ts | 32 +++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/foundations/server/packages/core/src/types.ts b/foundations/server/packages/core/src/types.ts index 09d7053c09a..6e4e0716033 100644 --- a/foundations/server/packages/core/src/types.ts +++ b/foundations/server/packages/core/src/types.ts @@ -409,6 +409,12 @@ export interface FullTextAdapter { // If no field is provided, will return existing mapping of all dimms. initMapping: (ctx: MeasureContext, field?: { key: string, dims: number }) => Promise + + // Optional: return the total document count in the index/collection. + // Used by non-persistent backends (e.g. Typesense with emptyDir) to detect + // empty state on startup and trigger automatic reindex. + // Returns -1 if the count cannot be determined (e.g. connection error). + getDocCount?: (ctx: MeasureContext) => Promise } /** diff --git a/foundations/server/packages/typesense/src/adapter.ts b/foundations/server/packages/typesense/src/adapter.ts index 8f972b454e9..36ea078f6ee 100644 --- a/foundations/server/packages/typesense/src/adapter.ts +++ b/foundations/server/packages/typesense/src/adapter.ts @@ -207,6 +207,21 @@ class TypesenseAdapter implements FullTextAdapter { // Typesense client does not require explicit close } + async getDocCount (ctx: MeasureContext): Promise { + try { + const col = await this.client.collections(this.collectionName).retrieve() + return (col as any).num_documents ?? 0 + } catch (err: any) { + if (isConnectionError(err)) { + ctx.warn('Typesense DB is not available for doc count check') + return -1 + } + Analytics.handleError(err) + ctx.error('Typesense getDocCount error', { error: err }) + return -1 + } + } + async searchString ( ctx: MeasureContext, workspaceId: WorkspaceUuid, diff --git a/pods/fulltext/src/manager.ts b/pods/fulltext/src/manager.ts index 4d5381d4d6d..40906686c08 100644 --- a/pods/fulltext/src/manager.ts +++ b/pods/fulltext/src/manager.ts @@ -105,6 +105,8 @@ export class WorkspaceManager { } } + await this.checkAndTriggerReindex() + this.shutdownInterval = setInterval(() => { for (const [k, v] of [...this.indexers.entries()]) { if (v instanceof Promise) { @@ -159,6 +161,36 @@ export class WorkspaceManager { ) } + private async checkAndTriggerReindex (): Promise { + if (this.fulltextAdapter.getDocCount === undefined) { + return + } + + const docCount = await this.fulltextAdapter.getDocCount(this.ctx) + if (docCount < 0) { + this.ctx.warn('Could not determine backend doc count, skipping auto-reindex check') + return + } + if (docCount > 0) { + this.ctx.info('Fulltext backend has documents, skipping auto-reindex', { docCount }) + return + } + + this.ctx.warn('Fulltext backend is empty (0 documents), triggering auto-reindex for all workspaces') + try { + const token = generateToken(systemAccountUuid, undefined, { service: 'tool' }) + const accountClient = getAccountClient(token) + const workspaces = await accountClient.listWorkspaces(null, 'active') + this.ctx.info('Auto-reindex: found workspaces', { count: workspaces.length }) + for (const ws of workspaces) { + await this.fulltextProducer.send(this.ctx, ws.uuid, [workspaceEvents.fullReindex()]) + this.ctx.info('Queued full reindex', { workspace: ws.uuid }) + } + } catch (err: any) { + this.ctx.error('Failed to trigger auto-reindex', { error: err }) + } + } + private async processTransactions ( m: ConsumerMessage> | TxDomainEvent>>, control: ConsumerControl From 5dd7bedae73cff1e1d2c5c6ad0d0efa31b11f8d7 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Sat, 18 Apr 2026 08:47:06 -0400 Subject: [PATCH 06/11] fix: bump dep versions in opensearch/typesense packages to match monorepo rush check failed because the new opensearch and typesense packages referenced older internal dep versions (analytics 0.7.17, core 0.7.24, platform 0.7.19, server-core 0.7.18, platform-rig 0.7.19) while the rest of the monorepo has moved to analytics 0.7.19, core 0.7.26, platform 0.7.20, server-core 0.7.19, platform-rig 0.7.21. Signed-off-by: Don Kendall Signed-off-by: Don Kendall --- common/config/rush/pnpm-lock.yaml | 20 +++++++++---------- .../server/packages/opensearch/package.json | 12 +++++------ .../server/packages/typesense/package.json | 12 +++++------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 0d9b626867f..7b2e4793edc 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -5219,23 +5219,23 @@ importers: ../../foundations/server/packages/opensearch: dependencies: '@hcengineering/analytics': - specifier: workspace:^0.7.17 + specifier: workspace:^0.7.19 version: link:../../../core/packages/analytics '@hcengineering/core': - specifier: workspace:^0.7.24 + specifier: workspace:^0.7.26 version: link:../../../core/packages/core '@hcengineering/platform': - specifier: workspace:^0.7.19 + specifier: workspace:^0.7.20 version: link:../../../core/packages/platform '@hcengineering/server-core': - specifier: workspace:^0.7.18 + specifier: workspace:^0.7.19 version: link:../core '@opensearch-project/opensearch': specifier: ^2.12.0 version: 2.13.0 devDependencies: '@hcengineering/platform-rig': - specifier: workspace:^0.7.19 + specifier: workspace:^0.7.21 version: link:../../../utils/packages/platform-rig '@types/jest': specifier: ^29.5.5 @@ -5581,23 +5581,23 @@ importers: ../../foundations/server/packages/typesense: dependencies: '@hcengineering/analytics': - specifier: workspace:^0.7.17 + specifier: workspace:^0.7.19 version: link:../../../core/packages/analytics '@hcengineering/core': - specifier: workspace:^0.7.24 + specifier: workspace:^0.7.26 version: link:../../../core/packages/core '@hcengineering/platform': - specifier: workspace:^0.7.19 + specifier: workspace:^0.7.20 version: link:../../../core/packages/platform '@hcengineering/server-core': - specifier: workspace:^0.7.18 + specifier: workspace:^0.7.19 version: link:../core typesense: specifier: ^2.0.0 version: 2.1.0(@babel/runtime@7.28.4) devDependencies: '@hcengineering/platform-rig': - specifier: workspace:^0.7.19 + specifier: workspace:^0.7.21 version: link:../../../utils/packages/platform-rig '@types/jest': specifier: ^29.5.5 diff --git a/foundations/server/packages/opensearch/package.json b/foundations/server/packages/opensearch/package.json index 618f31b1461..f03a2d862f0 100644 --- a/foundations/server/packages/opensearch/package.json +++ b/foundations/server/packages/opensearch/package.json @@ -1,6 +1,6 @@ { "name": "@hcengineering/opensearch", - "version": "0.7.17", + "version": "0.7.18", "main": "lib/index.js", "svelte": "src/index.ts", "types": "types/index.d.ts", @@ -37,7 +37,7 @@ "_phase:validate": "compile validate" }, "devDependencies": { - "@hcengineering/platform-rig": "workspace:^0.7.19", + "@hcengineering/platform-rig": "workspace:^0.7.21", "@typescript-eslint/eslint-plugin": "^6.21.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-promise": "^6.1.1", @@ -54,11 +54,11 @@ "eslint-plugin-svelte": "^2.35.1" }, "dependencies": { - "@hcengineering/core": "workspace:^0.7.24", - "@hcengineering/platform": "workspace:^0.7.19", - "@hcengineering/server-core": "workspace:^0.7.18", + "@hcengineering/core": "workspace:^0.7.26", + "@hcengineering/platform": "workspace:^0.7.20", + "@hcengineering/server-core": "workspace:^0.7.19", "@opensearch-project/opensearch": "^2.12.0", - "@hcengineering/analytics": "workspace:^0.7.17" + "@hcengineering/analytics": "workspace:^0.7.19" }, "repository": { "type": "git", diff --git a/foundations/server/packages/typesense/package.json b/foundations/server/packages/typesense/package.json index 7b301b93eb3..1cbbea4e866 100644 --- a/foundations/server/packages/typesense/package.json +++ b/foundations/server/packages/typesense/package.json @@ -1,6 +1,6 @@ { "name": "@hcengineering/typesense", - "version": "0.7.17", + "version": "0.7.18", "main": "lib/index.js", "svelte": "src/index.ts", "types": "types/index.d.ts", @@ -37,7 +37,7 @@ "_phase:validate": "compile validate" }, "devDependencies": { - "@hcengineering/platform-rig": "workspace:^0.7.19", + "@hcengineering/platform-rig": "workspace:^0.7.21", "@typescript-eslint/eslint-plugin": "^6.21.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-promise": "^6.1.1", @@ -53,10 +53,10 @@ "eslint-plugin-svelte": "^2.35.1" }, "dependencies": { - "@hcengineering/core": "workspace:^0.7.24", - "@hcengineering/platform": "workspace:^0.7.19", - "@hcengineering/server-core": "workspace:^0.7.18", - "@hcengineering/analytics": "workspace:^0.7.17", + "@hcengineering/core": "workspace:^0.7.26", + "@hcengineering/platform": "workspace:^0.7.20", + "@hcengineering/server-core": "workspace:^0.7.19", + "@hcengineering/analytics": "workspace:^0.7.19", "typesense": "^2.0.0" }, "repository": { From 488340fc31b774ae22986f1fe2e16dbc71442d72 Mon Sep 17 00:00:00 2001 From: Artyom Savchenko Date: Tue, 29 Sep 2026 07:20:35 +0200 Subject: [PATCH 07/11] Fix employee bypass (#11048) Signed-off-by: Artyom Savchenko --- .../middleware/src/guestPermissions.ts | 56 +++++-- .../src/tests/guestPermissions.test.ts | 152 ++++++++++++++++-- 2 files changed, 184 insertions(+), 24 deletions(-) diff --git a/foundations/server/packages/middleware/src/guestPermissions.ts b/foundations/server/packages/middleware/src/guestPermissions.ts index b6a7ba289ea..3f3526914bc 100644 --- a/foundations/server/packages/middleware/src/guestPermissions.ts +++ b/foundations/server/packages/middleware/src/guestPermissions.ts @@ -20,6 +20,8 @@ import core, { type Tx, type TxApplyIf, type TxCUD, + type TxCreateDoc, + type TxMixin, TxProcessor, type TxUpdateDoc } from '@hcengineering/core' @@ -199,36 +201,62 @@ export class GuestPermissionsMiddleware extends BaseMiddleware implements Middle const persons = await this.findAll(ctx, contact.class.Person, { _id: tx.objectId as Ref }, { limit: 1 }) const person = persons[0] - return person === undefined || !this.canUserEditPersonContactDetails(person, account) + if (person === undefined) return true + if ( + tx._class === core.class.TxMixin && + (tx as TxMixin).mixin === contact.mixin.Employee && + !h.hasMixin(person, contact.mixin.Employee) + ) { + return true + } + return !this.canUserEditPersonContactDetails(person, account) } if (h.isDerived(tx.objectClass, contact.class.Channel)) { - const parentPersonId = await this.getChannelParentPersonId(ctx, tx) - if (parentPersonId === undefined) return false - - const persons = await this.findAll(ctx, contact.class.Person, { _id: parentPersonId }, { limit: 1 }) - const person = persons[0] - return person === undefined || !this.canUserEditPersonContactDetails(person, account) + const parentPersonIds = await this.getChannelParentPersonIds(ctx, tx) + for (const parentPersonId of parentPersonIds) { + const persons = await this.findAll(ctx, contact.class.Person, { _id: parentPersonId }, { limit: 1 }) + const person = persons[0] + if (person === undefined || !this.canUserEditPersonContactDetails(person, account)) return true + } } return false } - private async getChannelParentPersonId ( + private async getChannelParentPersonIds ( ctx: MeasureContext, tx: TxCUD - ): Promise | undefined> { - if (tx.attachedToClass === contact.class.Person && tx.attachedTo !== undefined) { - return tx.attachedTo as Ref + ): Promise>> { + const result = new Set>() + const addPerson = (personId: Ref | undefined, personClass: Ref> | undefined): void => { + if ( + personId !== undefined && + personClass !== undefined && + this.context.hierarchy.isDerived(personClass, contact.class.Person) + ) { + result.add(personId as Ref) + } } - if (tx._class === core.class.TxCreateDoc) return undefined + if (tx._class === core.class.TxCreateDoc) { + const channel = TxProcessor.createDoc2Doc(tx as TxCreateDoc) + addPerson(channel.attachedTo, channel.attachedToClass) + return Array.from(result) + } const channels = await this.findAll(ctx, contact.class.Channel, { _id: tx.objectId as Ref }, { limit: 1 }) const channel = channels[0] - if (channel?.attachedToClass !== contact.class.Person) return undefined + if (channel === undefined) return [] + + addPerson(channel.attachedTo, channel.attachedToClass) + + if (tx._class === core.class.TxUpdateDoc) { + const operations = (tx as TxUpdateDoc).operations + addPerson(operations.attachedTo ?? channel.attachedTo, operations.attachedToClass ?? channel.attachedToClass) + } - return channel.attachedTo as Ref + return Array.from(result) } /** diff --git a/foundations/server/packages/middleware/src/tests/guestPermissions.test.ts b/foundations/server/packages/middleware/src/tests/guestPermissions.test.ts index 34b4a5d83a1..25e51a76402 100644 --- a/foundations/server/packages/middleware/src/tests/guestPermissions.test.ts +++ b/foundations/server/packages/middleware/src/tests/guestPermissions.test.ts @@ -113,11 +113,7 @@ function patchContactHierarchy (mw: GuestPermissionsMiddleware): void { mixin === contact.mixin.Employee && doc?.employee === true } -function makePersonDoc ( - _id: Ref, - personUuid: Account['uuid'], - employee: boolean = true -): Doc { +function makePersonDoc (_id: Ref, personUuid: Account['uuid'], employee: boolean = true): Doc { return { _id, _class: contact.class.Person, @@ -183,7 +179,7 @@ describe('GuestPermissionsMiddleware', () => { // ─── User contact mutability ──────────────────────────────────────────────── describe('user contact mutability', () => { it('forbids a user from updating another employee person', async () => { - const otherPersonId = generateId() as Ref + const otherPersonId = generateId() const account = makeAccount(AccountRole.User) const findAll: FindAllFn = async (_ctx, _class, query: any) => { if (_class === contact.class.Person && query?._id === otherPersonId) { @@ -206,7 +202,7 @@ describe('GuestPermissionsMiddleware', () => { }) it('allows a user to update their own employee person', async () => { - const personId = generateId() as Ref + const personId = generateId() const account = makeAccount(AccountRole.User) let nextCalled = false const findAll: FindAllFn = async (_ctx, _class, query: any) => { @@ -234,8 +230,8 @@ describe('GuestPermissionsMiddleware', () => { }) it('forbids a user from updating channels attached to another employee person', async () => { - const otherPersonId = generateId() as Ref - const channelId = generateId() as Ref + const otherPersonId = generateId() + const channelId = generateId() const account = makeAccount(AccountRole.User) const findAll: FindAllFn = async (_ctx, _class, query: any) => { if (_class === contact.class.Person && query?._id === otherPersonId) { @@ -270,8 +266,144 @@ describe('GuestPermissionsMiddleware', () => { await expect(mw.tx(makeCtx(account), [tx])).rejects.toThrow() }) + it('does not trust spoofed channel parent fields', async () => { + const ownPersonId = generateId() + const otherPersonId = generateId() + const channelId = generateId() + const account = makeAccount(AccountRole.User) + const findAll: FindAllFn = async (_ctx, _class, query: any) => { + if (_class === contact.class.Person && query?._id === ownPersonId) { + return [makePersonDoc(ownPersonId, account.uuid)] + } + if (_class === contact.class.Person && query?._id === otherPersonId) { + return [makePersonDoc(otherPersonId, generateId() as any)] + } + if (_class === contact.class.Channel && query?._id === channelId) { + return [ + { + _id: channelId, + _class: contact.class.Channel, + space: 'contact:space:Contacts' as Ref, + modifiedOn: Date.now(), + modifiedBy: 'test' as PersonId, + attachedTo: otherPersonId, + attachedToClass: contact.class.Person + } as any + ] + } + return [] + } + const mw = makeMiddleware(findAll) + patchContactHierarchy(mw) + + const factory = new TxFactory(account.primarySocialId) + const tx = factory.createTxUpdateDoc( + contact.class.Channel as Ref>, + 'core:space:Workspace' as Ref, + channelId, + { value: 'new@example.com' } as any + ) + tx.attachedTo = ownPersonId + tx.attachedToClass = contact.class.Person + + await expect(mw.tx(makeCtx(account), [tx])).rejects.toThrow() + }) + + it('forbids creating a channel attached through attributes to another employee person', async () => { + const otherPersonId = generateId() + const account = makeAccount(AccountRole.User) + const findAll: FindAllFn = async (_ctx, _class, query: any) => { + if (_class === contact.class.Person && query?._id === otherPersonId) { + return [makePersonDoc(otherPersonId, generateId() as any)] + } + return [] + } + const mw = makeMiddleware(findAll) + patchContactHierarchy(mw) + + const factory = new TxFactory(account.primarySocialId) + const tx = factory.createTxCreateDoc( + contact.class.Channel as Ref>, + 'contact:space:Contacts' as Ref, + { + attachedTo: otherPersonId, + attachedToClass: contact.class.Person, + collection: 'channels', + provider: 'contact:channelProvider:Email', + value: 'new@example.com' + } as any + ) + + await expect(mw.tx(makeCtx(account), [tx])).rejects.toThrow() + }) + + it('forbids moving a channel to another employee person', async () => { + const ownPersonId = generateId() + const otherPersonId = generateId() + const channelId = generateId() + const account = makeAccount(AccountRole.User) + const findAll: FindAllFn = async (_ctx, _class, query: any) => { + if (_class === contact.class.Person && query?._id === ownPersonId) { + return [makePersonDoc(ownPersonId, account.uuid)] + } + if (_class === contact.class.Person && query?._id === otherPersonId) { + return [makePersonDoc(otherPersonId, generateId() as any)] + } + if (_class === contact.class.Channel && query?._id === channelId) { + return [ + { + _id: channelId, + _class: contact.class.Channel, + space: 'contact:space:Contacts' as Ref, + modifiedOn: Date.now(), + modifiedBy: 'test' as PersonId, + attachedTo: ownPersonId, + attachedToClass: contact.class.Person + } as any + ] + } + return [] + } + const mw = makeMiddleware(findAll) + patchContactHierarchy(mw) + + const factory = new TxFactory(account.primarySocialId) + const tx = factory.createTxUpdateDoc( + contact.class.Channel as Ref>, + 'core:space:Workspace' as Ref, + channelId, + { attachedTo: otherPersonId, attachedToClass: contact.class.Person } as any + ) + + await expect(mw.tx(makeCtx(account), [tx])).rejects.toThrow() + }) + + it('forbids a user from adding the employee mixin to a person', async () => { + const personId = generateId() + const account = makeAccount(AccountRole.User) + const findAll: FindAllFn = async (_ctx, _class, query: any) => { + if (_class === contact.class.Person && query?._id === personId) { + return [makePersonDoc(personId, generateId() as any, false)] + } + return [] + } + const mw = makeMiddleware(findAll) + patchContactHierarchy(mw) + + const factory = new TxFactory(account.primarySocialId) + const tx = factory.createTxMixin( + personId, + contact.class.Person as Ref>, + 'contact:space:Contacts' as Ref, + contact.mixin.Employee, + { active: true } as any + ) + + await expect(mw.tx(makeCtx(account), [tx])).rejects.toThrow() + }) + it('allows maintainers to update another employee person', async () => { - const otherPersonId = generateId() as Ref + const otherPersonId = generateId() const account = makeAccount(AccountRole.Maintainer) let nextCalled = false const mw = makeMiddleware( From 3fe94f85c249531a2ff0901f6c985000960cabb7 Mon Sep 17 00:00:00 2001 From: Artyom Savchenko Date: Tue, 29 Sep 2026 07:25:34 +0200 Subject: [PATCH 08/11] Fix minio image (#11049) Signed-off-by: Artyom Savchenko --- ARCHITECTURE_OVERVIEW.md | 2 +- dev/docker-compose.yaml | 2 +- dev/local-mongo/docker-compose.yaml | 2 +- foundations/server/tests/docker-compose.yaml | 2 +- qms-tests/docker-compose.yaml | 2 +- tests/docker-compose.yaml | 2 +- ws-tests/docker-compose.yaml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ARCHITECTURE_OVERVIEW.md b/ARCHITECTURE_OVERVIEW.md index 48b680759ef..ba3b0f51535 100644 --- a/ARCHITECTURE_OVERVIEW.md +++ b/ARCHITECTURE_OVERVIEW.md @@ -397,7 +397,7 @@ sequenceDiagram | cockroach | cockroachdb/cockroach:latest-v24.3 | 26257, 8089 | **Main application database** - stores users, workspaces, documents, transactions, metadata, permissions | - | | **Supporting Infrastructure** | | | | | | elastic | elasticsearch:7.14.2 | 9200 | Search engine for full-text indexes | - | -| minio | minio/minio | 9000, 9001 | Object storage (S3) for files and blobs | - | +| minio | pgsty/silo | 9000, 9001 | Object storage (S3) for files and blobs | - | | redpanda | redpandadata/redpanda:v24.3.6 | 9092, 19092 | Event streaming (Kafka) for async processing | - | | redis | redis:8.0.2-alpine3.21 | 6379 | Cache & pub/sub for real-time features | - | | **Monitoring** | | | | | diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 2774fcd71a7..a5b81a502b6 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -118,7 +118,7 @@ services: - redpanda restart: unless-stopped minio: - image: 'minio/minio' + image: 'pgsty/silo' command: server /data --address ":9000" --console-address ":9001" extra_hosts: - 'huly.local:host-gateway' diff --git a/dev/local-mongo/docker-compose.yaml b/dev/local-mongo/docker-compose.yaml index 7269fd199f3..6ddd028ddf0 100644 --- a/dev/local-mongo/docker-compose.yaml +++ b/dev/local-mongo/docker-compose.yaml @@ -1,6 +1,6 @@ services: minio: - image: 'minio/minio' + image: 'pgsty/silo' command: server /data --address ":9000" --console-address ":9001" ports: - 9000:9000 diff --git a/foundations/server/tests/docker-compose.yaml b/foundations/server/tests/docker-compose.yaml index b8efe8158df..706557150df 100644 --- a/foundations/server/tests/docker-compose.yaml +++ b/foundations/server/tests/docker-compose.yaml @@ -68,7 +68,7 @@ services: depends_on: - redpanda minio: - image: 'minio/minio' + image: 'pgsty/silo' command: server /data --address ":9000" --console-address ":9001" expose: - 9000 diff --git a/qms-tests/docker-compose.yaml b/qms-tests/docker-compose.yaml index b4d49e41b1c..212f70ce0b3 100644 --- a/qms-tests/docker-compose.yaml +++ b/qms-tests/docker-compose.yaml @@ -59,7 +59,7 @@ services: depends_on: - redpanda minio: - image: 'minio/minio' + image: 'pgsty/silo' command: server /data --address ":9000" expose: - 9000 diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index d020ea5cd7a..6b4e03d2b8c 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -110,7 +110,7 @@ services: depends_on: - redpanda minio: - image: 'minio/minio' + image: 'pgsty/silo' command: server /data --address ":9000" --console-address ":9001" expose: - 9000 diff --git a/ws-tests/docker-compose.yaml b/ws-tests/docker-compose.yaml index e1cf6fc9073..723e89ff1b2 100644 --- a/ws-tests/docker-compose.yaml +++ b/ws-tests/docker-compose.yaml @@ -88,7 +88,7 @@ services: depends_on: - redpanda minio: - image: 'minio/minio' + image: 'pgsty/silo' command: server /data --address ":9000" --console-address ":9001" extra_hosts: - 'huly.local:host-gateway' From ce3c4a1dc330d0113d945270f52a49d40dcefeb0 Mon Sep 17 00:00:00 2001 From: Michael Uray <25169478+MichaelUray@users.noreply.github.com> Date: Tue, 29 Sep 2026 07:29:52 +0200 Subject: [PATCH 09/11] fix(chunter-trigger): seed default collaborators onto the target doc, not the message (#10973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OnChatMessageCreated seeds a target doc's default collaborators when the doc has none yet. The seed passed the ChatMessage create tx's own identity (tx.objectId / tx.objectClass / tx.objectSpace) to getAddCollaboratTxes, so the Collaborator records were attached to the ChatMessage instead of the target doc. Two consequences: - The target doc was never actually seeded: currentCollaborators stayed empty, so the defaults were lost and the seed re-evaluated on the next message. - For provideSecurity classes the seed re-fired on every message and littered the message with Collaborator records. Pass targetDoc._id / targetDoc._class / targetDoc.space instead. This wrong-target is long-standing (pre-existing); #9217 only mechanically inherited the line and did not introduce the bug. Adds a from-scratch jest harness (chunter-resources shipped no tests). It drives ChunterTrigger with a TxCreateDoc on an empty target and asserts the seeded Collaborator attaches to the target doc (attachedTo / attachedToClass / objectSpace). getAddCollaboratTxes runs for real — only getDocCollaborators and createCollaboratorNotifications are stubbed — so the assertion exercises the actual wiring: red before the fix (attachedTo = message id), green after. A sibling of this bug lives in server-notification-resources collectionCollabDoc (the createCollabDocInfo seeding path); left out of scope here. Signed-off-by: Michael Uray Co-authored-by: Michael Uray --- .../src/__tests__/seedCollaborators.test.ts | 165 ++++++++++++++++++ server-plugins/chunter-resources/src/index.ts | 8 +- 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 server-plugins/chunter-resources/src/__tests__/seedCollaborators.test.ts diff --git a/server-plugins/chunter-resources/src/__tests__/seedCollaborators.test.ts b/server-plugins/chunter-resources/src/__tests__/seedCollaborators.test.ts new file mode 100644 index 00000000000..01a62c7b285 --- /dev/null +++ b/server-plugins/chunter-resources/src/__tests__/seedCollaborators.test.ts @@ -0,0 +1,165 @@ +import { type Tx, type TxCreateDoc } from '@hcengineering/core' +import { jsonToMarkup, MarkupNodeType } from '@hcengineering/text-core' + +// Imported AFTER the mocks (jest.mock is hoisted by ts-jest, so even these +// top-level ESM imports see the mocked modules). Avoid `require()` so the file +// passes `tsc --noEmit` (the package's tsconfig ships @types/jest). +import { ChunterTrigger } from '../index' +import coreDefault from '@hcengineering/core' +import chunter from '@hcengineering/chunter' + +// ------------------------------------------------------------------ +// jest.mock for @hcengineering/core +// +// OnChatMessageCreated only reaches the seed branch when +// getClassCollaborators(...) is defined for both the message's attachedToClass +// and the targetDoc's class. That helper reads the model DB, which is empty in +// this unit harness, so we override it to return a truthy mixin. +// +// jest.requireActual('@hcengineering/core') cannot be spread ({...actual}) in +// ts-jest: the eager getter evaluation trips a circular-dependency crash in the +// compiled lib. We delegate through a Proxy and override ONLY the one helper. +// Everything else (default export `core`, TxProcessor, notEmpty, ...) stays real +// so getAddCollaboratTxes builds genuine core.class.Collaborator txes. +// ------------------------------------------------------------------ +jest.mock('@hcengineering/core', () => { + const actual = jest.requireActual('@hcengineering/core') + const mockGetClassCollaborators = jest.fn(() => ({ provideSecurity: true })) + return new Proxy(actual, { + get (target: any, prop: string) { + if (prop === 'getClassCollaborators') return mockGetClassCollaborators + return target[prop] + } + }) +}) + +jest.mock('@hcengineering/server-contact', () => ({ + getAccountBySocialId: jest.fn(async () => undefined), + getPerson: jest.fn(async () => undefined) +})) + +// ------------------------------------------------------------------ +// jest.mock for @hcengineering/server-notification-resources +// +// CRITICAL: getAddCollaboratTxes is NOT mocked — it runs REAL (requireActual), +// so the test exercises the actual wiring of attachedTo/attachedToClass/space +// that the fix corrected. We only stub the two collaborator-source helpers: +// - getDocCollaborators -> a deterministic seed set ['default-acc'] +// - createCollaboratorNotifications -> no-op (imported by index.ts) +// ------------------------------------------------------------------ +jest.mock('@hcengineering/server-notification-resources', () => { + const actual = jest.requireActual('@hcengineering/server-notification-resources') + return { + ...actual, + getDocCollaborators: jest.fn(async () => ['default-acc']), + createCollaboratorNotifications: jest.fn(async () => []) + } +}) + +function emptyMarkup (): string { + return jsonToMarkup({ type: MarkupNodeType.doc, content: [] }) +} + +// The target doc the ChatMessage is attached to. Its class is opaque here +// (getClassCollaborators is mocked truthy regardless), it only needs to differ +// from the message so a wrong-target seed is observable. Its space differs from +// the message space too, so the objectSpace assertion is also red-before/green-after. +const TARGET = { _id: 'issue-1', _class: 'tracker:class:Issue', space: 'target-space' } + +function makeControl (): any { + const ctx: any = { + with: async (_name: string, _params: any, fn: any) => fn(ctx), + contextData: {} + } + return { + hierarchy: { + // ChunterTrigger dispatches on isDerived(tx.objectClass, ChatMessage). + // ThreadMessage/Channel checks must stay false; a plain equality suffices + // for this harness (ChatMessage===ChatMessage true, everything else false). + isDerived: (cls: string, base: string) => cls === base + }, + modelDb: {}, + txFactory: { + createTxCreateDoc: (_class: string, space: string, attributes: any) => ({ + _class: coreDefault.class.TxCreateDoc, + objectClass: _class, + objectSpace: space, + attributes + }) + }, + findAll: jest.fn(async (_ctx: any, _class: string, _query: any) => { + if (_class === coreDefault.class.Collaborator) { + // EMPTY -> currentCollaborators is empty -> the seed branch fires. + return [] + } + if (_class === coreDefault.class.ClassCollaborators) { + // provideSecurity true -> the mention grant loop is skipped, isolating + // the seed tx as the only emitted Collaborator create. + return [{ provideSecurity: true }] + } + // targetDoc lookup (message.attachedToClass, { _id: message.attachedTo }). + return [TARGET] + }), + ctx + } +} + +function makeCreateTx (): any { + return { + _class: coreDefault.class.TxCreateDoc, + objectClass: chunter.class.ChatMessage, + objectId: 'msg-1', // <- the ChatMessage id; the bug seeded onto THIS + objectSpace: 'message-space', // <- differs from TARGET.space + modifiedBy: 'social-author', + attributes: { + attachedTo: TARGET._id, + attachedToClass: TARGET._class, + message: emptyMarkup(), + collection: 'comments' + } + } +} + +function seedTx (res: Tx[]): TxCreateDoc | undefined { + return res.find( + (t): t is TxCreateDoc => + t._class === coreDefault.class.TxCreateDoc && + (t as any).objectClass === coreDefault.class.Collaborator && + (t as any).attributes.collaborator === 'default-acc' + ) +} + +describe('OnChatMessageCreated — seed default collaborators onto the target doc', () => { + test('the seeded Collaborator is attached to the target doc, not the ChatMessage', async () => { + const control = makeControl() + const tx = makeCreateTx() + + const res: Tx[] = await ChunterTrigger([tx], control) + + const seed = seedTx(res) + expect(seed).toBeDefined() + + // Red before the fix (getAddCollaboratTxes(tx.objectId, ...)): the seed would + // land on the message -> attachedTo 'msg-1', attachedToClass ChatMessage, + // objectSpace 'message-space'. + // Green after the fix (getAddCollaboratTxes(targetDoc._id, ...)). + expect(seed?.attributes.attachedTo).toBe(TARGET._id) + expect(seed?.attributes.attachedToClass).toBe(TARGET._class) + expect(seed?.objectSpace).toBe(TARGET.space) + }) + + test('exactly one seed Collaborator tx is emitted for the default collaborator', async () => { + const control = makeControl() + const tx = makeCreateTx() + + const res: Tx[] = await ChunterTrigger([tx], control) + + const seeds = res.filter( + (t: any) => + t._class === coreDefault.class.TxCreateDoc && + t.objectClass === coreDefault.class.Collaborator && + t.attributes.collaborator === 'default-acc' + ) + expect(seeds).toHaveLength(1) + }) +}) diff --git a/server-plugins/chunter-resources/src/index.ts b/server-plugins/chunter-resources/src/index.ts index 91896f6654c..6068a75df91 100644 --- a/server-plugins/chunter-resources/src/index.ts +++ b/server-plugins/chunter-resources/src/index.ts @@ -204,7 +204,13 @@ async function OnChatMessageCreated (ctx: MeasureContext, tx: TxCUD, contro if (mixin !== undefined) { const collaborators = await getDocCollaborators(ctx, targetDoc, mixin, control) currentCollaborators = collaborators - res.push(...getAddCollaboratTxes(tx.objectId, tx.objectClass, tx.objectSpace, control, collaborators)) + // Seed the target doc's default collaborators onto the target doc itself, not + // onto the ChatMessage. tx is the ChatMessage create, so tx.objectId is the + // message id — using it here attached the records to the message, so the target + // doc was never actually seeded (currentCollaborators stayed empty on the next + // message and the defaults were lost) and, for provideSecurity classes, the seed + // re-fired on every message and littered the message with records. + res.push(...getAddCollaboratTxes(targetDoc._id, targetDoc._class, targetDoc.space, control, collaborators)) } } From 7d01eb2cd395de2bec9957dcd77f3542e9fbd50d Mon Sep 17 00:00:00 2001 From: Ignat Remizov Date: Tue, 29 Sep 2026 08:36:56 +0300 Subject: [PATCH 10/11] fix(chunter): skip legacy comment migration when domain is absent (#10891) Allow old Chunter migrations to complete cleanly for workspaces that no longer have the legacy `comment` storage domain. The migration still runs normally when the domain exists, but workspaces created or restored after the comment-to-activity migration can now skip the obsolete table operations and record the migration state. Changes: - Add a migration-client `domainExists` helper backed by the active adapter. - Make the Postgres domain helper check actual public tables instead of assuming every domain exists. - Keep Postgres domain listing compatible with model domain names while including translated table names. - Guard legacy `comment` update/move/delete operations in the Chunter migration. Validation: - git diff --cached --check - node common/scripts/install-run-rush.js build -t @hcengineering/model -t @hcengineering/server-tool -t @hcengineering/postgres -t @hcengineering/model-chunter Signed-off-by: Ignat Remizov --- .../core/packages/model/src/migration.ts | 2 ++ .../server/packages/postgres/src/utils.ts | 33 +++++++++++++++++-- models/chunter/src/migration.ts | 18 ++++++---- server/tool/src/upgrade.ts | 6 ++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/foundations/core/packages/model/src/migration.ts b/foundations/core/packages/model/src/migration.ts index c82b8c2eaf5..9c19a97a60b 100644 --- a/foundations/core/packages/model/src/migration.ts +++ b/foundations/core/packages/model/src/migration.ts @@ -108,6 +108,8 @@ export interface MigrationClient { create: (domain: Domain, doc: T | T[]) => Promise delete: (domain: Domain, _id: Ref) => Promise deleteMany: (domain: Domain, query: DocumentQuery) => Promise + // Check the physical storage domain, including legacy domains no longer present in the active model. + domainExists: (domain: Domain) => Promise hierarchy: Hierarchy model: ModelDb diff --git a/foundations/server/packages/postgres/src/utils.ts b/foundations/server/packages/postgres/src/utils.ts index 469904e3a93..3c0df2258a9 100644 --- a/foundations/server/packages/postgres/src/utils.ts +++ b/foundations/server/packages/postgres/src/utils.ts @@ -325,12 +325,39 @@ export class DBCollectionHelper implements DomainHelperOperations { async create (domain: Domain): Promise {} async exists (domain: Domain): Promise { - // Always exists. We don't need to check for index existence - return true + // Migrations can reference legacy domains that were removed from the current model, + // so this must check the physical table instead of the initialized model-domain set. + const tableName = translateDomain(domain) + const res = await this.client.execute( + ` + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = $1 + LIMIT 1 + `, + [tableName] + ) + return res.length > 0 } async listDomains (): Promise> { - return this.domains + const rows = await this.client.execute(` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name NOT LIKE 'pg_%' + AND table_name NOT LIKE 'cluster_%' + AND table_name NOT LIKE 'kv_%' + AND table_name NOT LIKE 'node_%' + `) + const tableNames = new Set(rows.map((it) => it.table_name as Domain)) + for (const domain of this.domains) { + if (tableNames.has(translateDomain(domain) as Domain)) { + tableNames.add(domain) + } + } + return tableNames } async createIndex (domain: Domain, value: string | FieldIndexConfig, options?: { name: string }): Promise {} diff --git a/models/chunter/src/migration.ts b/models/chunter/src/migration.ts index abfa2b41395..87ed38f030c 100644 --- a/models/chunter/src/migration.ts +++ b/models/chunter/src/migration.ts @@ -156,16 +156,20 @@ export async function createRandom (client: MigrationUpgradeClient, tx: TxOperat } async function convertCommentsToChatMessages (client: MigrationClient): Promise { - await client.update( - DOMAIN_COMMENT, - { _class: 'chunter:class:Comment' as Ref> }, - { _class: chunter.class.ChatMessage } - ) - await client.move(DOMAIN_COMMENT, { _class: chunter.class.ChatMessage }, DOMAIN_ACTIVITY) + if (await client.domainExists(DOMAIN_COMMENT)) { + await client.update( + DOMAIN_COMMENT, + { _class: 'chunter:class:Comment' as Ref> }, + { _class: chunter.class.ChatMessage } + ) + await client.move(DOMAIN_COMMENT, { _class: chunter.class.ChatMessage }, DOMAIN_ACTIVITY) + } } async function removeBacklinks (client: MigrationClient): Promise { - await client.deleteMany(DOMAIN_COMMENT, { _class: 'chunter:class:Backlink' as Ref> }) + if (await client.domainExists(DOMAIN_COMMENT)) { + await client.deleteMany(DOMAIN_COMMENT, { _class: 'chunter:class:Backlink' as Ref> }) + } await client.deleteMany(DOMAIN_ACTIVITY, { _class: activity.class.DocUpdateMessage, objectClass: 'chunter:class:Backlink' as Ref> diff --git a/server/tool/src/upgrade.ts b/server/tool/src/upgrade.ts index 4792ef47cb0..21a8942e820 100644 --- a/server/tool/src/upgrade.ts +++ b/server/tool/src/upgrade.ts @@ -127,6 +127,12 @@ export class MigrateClientImpl implements MigrationClient { await this.lowLevel.rawDeleteMany(domain, query) } + async domainExists (domain: Domain): Promise { + const adapter = this.pipeline.context.adapterManager?.getAdapter(domain, false) + const helper = adapter?.helper?.() + return helper !== undefined ? await helper.exists(domain) : true + } + async fullReindex (): Promise { await this.queue.send(this.ctx, this.wsIds.uuid, [workspaceEvents.fullReindex()]) } From f030c2a3373307e0b769748fef842e6ee7680183 Mon Sep 17 00:00:00 2001 From: Gunjan <112072151+Learnobug@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:12:46 +0530 Subject: [PATCH 11/11] fix: respect locale date order for Asian languages in DatePresenter #10779 (#10781) * fix: respect locale date order for Asian languages in DatePresenter Signed-off-by: Learnobug * fix: apply formatting fixes Signed-off-by: Learnobug --------- Signed-off-by: Learnobug Co-authored-by: Artyom Savchenko --- .../components/calendar/DatePresenter.svelte | 8 +--- .../calendar/DateRangePresenter.svelte | 14 ++---- .../components/calendar/internal/DateUtils.ts | 44 +++++++++++-------- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/packages/ui/src/components/calendar/DatePresenter.svelte b/packages/ui/src/components/calendar/DatePresenter.svelte index ff8c4a00f4b..8ea0ee30b92 100644 --- a/packages/ui/src/components/calendar/DatePresenter.svelte +++ b/packages/ui/src/components/calendar/DatePresenter.svelte @@ -25,7 +25,7 @@ import DatePopup from './DatePopup.svelte' import DPCalendar from './icons/DPCalendar.svelte' import DPCalendarOver from './icons/DPCalendarOver.svelte' - import { getMonthName } from './internal/DateUtils' + import { formatDate } from './internal/DateUtils' import { registerFocus } from '../../focus' export let value: number | null | undefined @@ -127,11 +127,7 @@ {#if value !== null && value !== undefined} {#if shouldShowLabel} {#if mode !== DateRangeMode.TIMEONLY} - {new Date(value).getDate()} - {getMonthName(new Date(value), 'short')} - {#if new Date(value).getFullYear() !== today.getFullYear()} - {new Date(value).getFullYear()} - {/if} + {formatDate(new Date(value), new Date(value).getFullYear() !== today.getFullYear())} {/if} {#if withTime} {#if mode !== DateRangeMode.TIMEONLY} diff --git a/packages/ui/src/components/calendar/DateRangePresenter.svelte b/packages/ui/src/components/calendar/DateRangePresenter.svelte index 3a398063710..c9b056674cf 100644 --- a/packages/ui/src/components/calendar/DateRangePresenter.svelte +++ b/packages/ui/src/components/calendar/DateRangePresenter.svelte @@ -24,7 +24,7 @@ import DatePopup from './DatePopup.svelte' import DPCalendar from './icons/DPCalendar.svelte' import DPCalendarOver from './icons/DPCalendarOver.svelte' - import { daysInMonth, getMonthName } from './internal/DateUtils' + import { daysInMonth, formatDate } from './internal/DateUtils' export let value: number | null | undefined = null export let mode: DateRangeMode = DateRangeMode.DATE @@ -312,11 +312,7 @@ {/if} {#if value !== undefined && value !== null && value.toString() !== ''} {#if withDate} - {new Date(value).getDate()} - {getMonthName(new Date(value), 'short')} - {#if new Date(value).getFullYear() !== today.getFullYear()} - {new Date(value).getFullYear()} - {/if} + {formatDate(new Date(value), new Date(value).getFullYear() !== today.getFullYear())} {/if} {#if withTime} {#if withDate} @@ -487,11 +483,7 @@ {/if} {#if value !== undefined && value !== null && value.toString() !== ''} {#if withDate} - {new Date(value).getDate()} - {getMonthName(new Date(value), 'short')} - {#if new Date(value).getFullYear() !== today.getFullYear()} - {new Date(value).getFullYear()} - {/if} + {formatDate(new Date(value), new Date(value).getFullYear() !== today.getFullYear())} {/if} {#if withTime} {#if withDate} diff --git a/packages/ui/src/components/calendar/internal/DateUtils.ts b/packages/ui/src/components/calendar/internal/DateUtils.ts index fa1bc87023a..36ec971a65c 100644 --- a/packages/ui/src/components/calendar/internal/DateUtils.ts +++ b/packages/ui/src/components/calendar/internal/DateUtils.ts @@ -20,7 +20,7 @@ export const MILLISECONDS_IN_MINUTE = 60000 export const MILLISECONDS_IN_DAY = 86400000 export const MILLISECONDS_IN_WEEK = DAYS_IN_WEEK * MILLISECONDS_IN_DAY -export function firstDay (date: Date, firstDay: number = 1): Date { +export function firstDay(date: Date, firstDay: number = 1): Date { const firstDayOfMonth = new Date(new Date(date).setHours(0, 0, 0, 0)) firstDayOfMonth.setDate(1) // First day of month const result = new Date(firstDayOfMonth) @@ -32,23 +32,23 @@ export function firstDay (date: Date, firstDay: number = 1): Date { return result } -export function getWeek (date: Date): number { +export function getWeek(date: Date): number { const onejan = new Date(date.getFullYear(), 0, 1) return Math.ceil(((date.getTime() - onejan.getTime()) / MILLISECONDS_IN_DAY + onejan.getDay() + 1) / DAYS_IN_WEEK) } -export function daysInMonth (date: Date): number { +export function daysInMonth(date: Date): number { return 33 - new Date(date.getFullYear(), date.getMonth(), 33).getDate() } -export function getWeekDayName (weekDay: Date, weekFormat: 'narrow' | 'short' | 'long' | undefined = 'short'): string { +export function getWeekDayName(weekDay: Date, weekFormat: 'narrow' | 'short' | 'long' | undefined = 'short'): string { const locale = new Intl.NumberFormat().resolvedOptions().locale return new Intl.DateTimeFormat(locale, { weekday: weekFormat }).format(weekDay) } -export function getWeekDayNames (): Map { +export function getWeekDayNames(): Map { const today: Date = new Date() const offset: number = 0 - today.getDay() const startDate: number = today.setTime(today.getTime() + MILLISECONDS_IN_DAY * offset) @@ -59,15 +59,15 @@ export function getWeekDayNames (): Map { return result } -export function day (firstDay: Date, offset: number): Date { +export function day(firstDay: Date, offset: number): Date { return new Date(new Date(firstDay).setDate(firstDay.getDate() + offset)) } -export function weekday (firstDay: Date, w: number, d: number): Date { +export function weekday(firstDay: Date, w: number, d: number): Date { return day(firstDay, w * DAYS_IN_WEEK + d) } -export function areDatesEqual (firstDate: Date | undefined, secondDate: Date | undefined): boolean { +export function areDatesEqual(firstDate: Date | undefined, secondDate: Date | undefined): boolean { if (firstDate === undefined || secondDate === undefined) { return false } @@ -78,19 +78,19 @@ export function areDatesEqual (firstDate: Date | undefined, secondDate: Date | u ) } -export function fromCurrentToTz (date: Date | number, tz: string): Date { +export function fromCurrentToTz(date: Date | number, tz: string): Date { return utcToZonedTime(zonedTimeToUtc(date, getUserTimezone()), tz) } -export function fromTzToCurrent (date: Date | number, tz: string): Date { +export function fromTzToCurrent(date: Date | number, tz: string): Date { return utcToZonedTime(zonedTimeToUtc(date, tz), getUserTimezone()) } -export function isWeekend (date: Date): boolean { +export function isWeekend(date: Date): boolean { return date.getDay() === 0 || date.getDay() === 6 } -export function getMonthName (date: Date, option: 'narrow' | 'short' | 'long' | 'numeric' | '2-digit' = 'long'): string { +export function getMonthName(date: Date, option: 'narrow' | 'short' | 'long' | 'numeric' | '2-digit' = 'long'): string { try { const locale = new Intl.NumberFormat().resolvedOptions().locale return new Intl.DateTimeFormat(locale, { month: option }).format(date) @@ -106,24 +106,24 @@ export interface ICell { classes: TCellStyle } -export function getLocalWeekStart (): number { +export function getLocalWeekStart(): number { const locale = new Intl.Locale(navigator.language) return typeof (locale as any)?.getWeekInfo === 'function' ? (locale as any)?.getWeekInfo()?.firstDay : ((locale as any).weekInfo?.firstDay ?? 1) } -export function hasLocalWeekStart (): boolean { +export function hasLocalWeekStart(): boolean { const locale = new Intl.Locale(navigator.language) return typeof (locale as any)?.getWeekInfo === 'function' || (locale as any).weekInfo?.firstDay !== undefined } -export function getWeekStart (date: Date = new Date(), firstDay: number = 1): Date { +export function getWeekStart(date: Date = new Date(), firstDay: number = 1): Date { date = new Date(new Date(date).setHours(0, 0, 0, 0)) return new Date(date.setDate(date.getDate() - ((date.getDay() - firstDay + 7) % 7))) } -export function addZero (value: number): string { +export function addZero(value: number): string { if (value < 10) { return `0${value}` } @@ -155,10 +155,18 @@ export const getDueDateIconModifier = ( return 'normal' } -export function getFormattedDate (value: number | null, options?: Intl.DateTimeFormatOptions): string { +export function getFormattedDate(value: number | null, options?: Intl.DateTimeFormatOptions): string { return value === null ? '' : new Date(value).toLocaleString('default', options ?? { month: 'short', day: 'numeric' }) } +export function formatDate(date: Date, showYear: boolean): string { + const locale = new Intl.NumberFormat().resolvedOptions().locale + const options: Intl.DateTimeFormatOptions = showYear + ? { year: 'numeric', month: 'short', day: 'numeric' } + : { month: 'short', day: 'numeric' } + return new Intl.DateTimeFormat(locale, options).format(date) +} + export const getTimeZoneName = ( val: string = Intl.DateTimeFormat().resolvedOptions().timeZoneName ?? Intl.DateTimeFormat().resolvedOptions().timeZone ): string => { @@ -177,7 +185,7 @@ export const convertTimeZone = (tz: string): TimeZone => { } } -export function getUserTimezone (): string { +export function getUserTimezone(): string { if (window.Intl !== undefined) { return Intl.DateTimeFormat().resolvedOptions().timeZone } else {