diff --git a/.gitignore b/.gitignore
index dcfc4b40f..c8450e952 100644
--- a/.gitignore
+++ b/.gitignore
@@ -134,3 +134,6 @@ Untitled*
postgres_metadata
postgres_superset
node_modules
+
+# Local Claude Code workspace state (large; never committed)
+.claude/
diff --git a/datajunction-clients/python/datajunction/client.py b/datajunction-clients/python/datajunction/client.py
index ef9867264..f236ca422 100644
--- a/datajunction-clients/python/datajunction/client.py
+++ b/datajunction-clients/python/datajunction/client.py
@@ -207,31 +207,7 @@ def plan(
include_temporal_filters: bool = False,
lookback_window: str | None = None,
):
- """
- Returns a query execution plan for the given metrics and dimensions.
-
- The plan shows:
- - grain_groups: How metrics are grouped and their intermediate SQL
- - metric_formulas: How each metric combines its components
- - requested_dimensions: The dimensions being queried
-
- This is useful for understanding how DJ decomposes metrics into
- atomic aggregations and how multiple fact tables are joined together.
-
- Args:
- metrics: List of metric names to include
- dimensions: List of dimensions to group by
- filters: List of filter expressions
- cube: Optional cube node name. When provided, the cube's stored
- filters are automatically prepended to the query filters.
- dialect: SQL dialect (e.g., 'spark', 'trino'). Defaults to engine dialect.
- use_materialized: Whether to use materialized tables when available
- include_temporal_filters: Whether to include temporal partition filters.
- Only applies if the metrics and dimensions resolve to a cube with
- temporal partitions.
- lookback_window: Lookback window for temporal filters (e.g., '3 DAY',
- '1 WEEK'). Only applicable when include_temporal_filters is True.
- """
+ """Returns a query execution plan for the given metrics and dimensions."""
params: dict = {
"metrics": metrics,
"dimensions": dimensions or [],
diff --git a/datajunction-clients/python/datajunction/deployment.py b/datajunction-clients/python/datajunction/deployment.py
index 596d23e96..68142ac7c 100644
--- a/datajunction-clients/python/datajunction/deployment.py
+++ b/datajunction-clients/python/datajunction/deployment.py
@@ -342,30 +342,7 @@ def build_codeowners(
default_owner: str | None = None,
exclude_dirs: list[str] | None = None,
) -> int:
- """
- Generate a CODEOWNERS file from the owners fields in DJ node YAML files.
-
- Walks base_dir recursively, reads every *.yaml file (skipping dj.yaml),
- and maps each file path to its owners list. Files with no owners are
- omitted. Paths in the output are relative to base_dir and prefixed with
- / so GitHub resolves them from the repo root.
-
- If github_api_url is provided (and GITHUB_TOKEN / github_token_env is set),
- email addresses in owners fields are resolved to GitHub usernames via the
- search API. Unresolvable emails are emitted as-is with a warning comment.
-
- ``default_owner``, when set, emits a leading ``* `` rule so
- unmatched files (and any excluded directories) fall through to it. Because
- CODEOWNERS is last-match-wins, the per-file rules below it still take
- precedence for the files they name.
-
- ``exclude_dirs`` lists directories (relative to base_dir) whose nodes are
- NOT given per-file owners — they fall through to ``default_owner`` instead.
- This is for machine-generated trees (e.g. ``nodes/generated``) that have no
- individual human owner and should be team-owned as a block.
-
- Returns the number of per-file entries written (excludes the default rule).
- """
+ """Generate a CODEOWNERS file from the owners fields in DJ node YAML files."""
base = Path(base_dir).resolve()
excluded_dirs = [(base / d).resolve() for d in (exclude_dirs or [])]
diff --git a/datajunction-query/uv.lock b/datajunction-query/uv.lock
index 085cb0ecd..a166f9f24 100644
--- a/datajunction-query/uv.lock
+++ b/datajunction-query/uv.lock
@@ -1,10 +1,11 @@
version = 1
revision = 3
-requires-python = ">=3.10, <4.0"
+requires-python = ">=3.11, <4.0"
resolution-markers = [
- "python_full_version >= '3.12'",
- "python_full_version == '3.11.*'",
- "python_full_version < '3.11'",
+ "python_full_version >= '3.14'",
+ "python_full_version == '3.13.*'",
+ "python_full_version == '3.12.*'",
+ "python_full_version < '3.12'",
]
[[package]]
@@ -23,7 +24,6 @@ source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "mako" },
{ name = "sqlalchemy" },
- { name = "tomli", marker = "python_full_version < '3.11'" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/49/cc/aca263693b2ece99fa99a09b6d092acb89973eb2bb575faef1777e04f8b4/alembic-1.18.1.tar.gz", hash = "sha256:83ac6b81359596816fb3b893099841a0862f2117b2963258e965d70dc62fb866", size = 2044319, upload-time = "2026-01-14T18:53:14.907Z" }
@@ -54,7 +54,6 @@ name = "anyio"
version = "4.12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
{ name = "idna" },
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
@@ -76,23 +75,11 @@ wheels = [
name = "astroid"
version = "4.0.3"
source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "typing-extensions", marker = "python_full_version < '3.11'" },
-]
sdist = { url = "https://files.pythonhosted.org/packages/a1/ca/c17d0f83016532a1ad87d1de96837164c99d47a3b6bbba28bd597c25b37a/astroid-4.0.3.tar.gz", hash = "sha256:08d1de40d251cc3dc4a7a12726721d475ac189e4e583d596ece7422bc176bda3", size = 406224, upload-time = "2026-01-03T22:14:26.096Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ce/66/686ac4fc6ef48f5bacde625adac698f41d5316a9753c2b20bb0931c9d4e2/astroid-4.0.3-py3-none-any.whl", hash = "sha256:864a0a34af1bd70e1049ba1e61cee843a7252c826d97825fcee9b2fcbd9e1b14", size = 276443, upload-time = "2026-01-03T22:14:24.412Z" },
]
-[[package]]
-name = "backports-asyncio-runner"
-version = "1.2.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8e/ff/70dca7d7cb1cbc0edb2c6cc0c38b65cba36cccc491eca64cabd5fe7f8670/backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162", size = 69893, upload-time = "2025-07-02T02:27:15.685Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" },
-]
-
[[package]]
name = "boto3"
version = "1.42.34"
@@ -127,10 +114,8 @@ version = "1.4.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "colorama", marker = "os_name == 'nt'" },
- { name = "importlib-metadata", marker = "python_full_version < '3.10.2'" },
{ name = "packaging" },
{ name = "pyproject-hooks" },
- { name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/42/18/94eaffda7b329535d91f00fe605ab1f1e5cd68b2074d03f255c7d250687d/build-1.4.0.tar.gz", hash = "sha256:f1b91b925aa322be454f8330c6fb48b465da993d1e7e7e6fa35027ec49f3c936", size = 50054, upload-time = "2026-01-08T16:41:47.696Z" }
wheels = [
@@ -164,18 +149,6 @@ dependencies = [
]
sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/93/d7/516d984057745a6cd96575eea814fe1edd6646ee6efd552fb7b0921dec83/cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", size = 184283, upload-time = "2025-09-08T23:22:08.01Z" },
- { url = "https://files.pythonhosted.org/packages/9e/84/ad6a0b408daa859246f57c03efd28e5dd1b33c21737c2db84cae8c237aa5/cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", size = 180504, upload-time = "2025-09-08T23:22:10.637Z" },
- { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" },
- { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" },
- { url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" },
- { url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" },
- { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" },
- { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" },
- { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" },
- { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" },
- { url = "https://files.pythonhosted.org/packages/e2/cc/027d7fb82e58c48ea717149b03bcadcbdc293553edb283af792bd4bcbb3f/cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", size = 172184, upload-time = "2025-09-08T23:22:23.328Z" },
- { url = "https://files.pythonhosted.org/packages/33/fa/072dd15ae27fbb4e06b437eb6e944e75b068deb09e2a2826039e49ee2045/cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", size = 182790, upload-time = "2025-09-08T23:22:24.752Z" },
{ url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" },
{ url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" },
{ url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" },
@@ -252,22 +225,6 @@ version = "3.4.4"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/1f/b8/6d51fc1d52cbd52cd4ccedd5b5b2f0f6a11bbf6765c782298b0f3e808541/charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d", size = 209709, upload-time = "2025-10-14T04:40:11.385Z" },
- { url = "https://files.pythonhosted.org/packages/5c/af/1f9d7f7faafe2ddfb6f72a2e07a548a629c61ad510fe60f9630309908fef/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8", size = 148814, upload-time = "2025-10-14T04:40:13.135Z" },
- { url = "https://files.pythonhosted.org/packages/79/3d/f2e3ac2bbc056ca0c204298ea4e3d9db9b4afe437812638759db2c976b5f/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad", size = 144467, upload-time = "2025-10-14T04:40:14.728Z" },
- { url = "https://files.pythonhosted.org/packages/ec/85/1bf997003815e60d57de7bd972c57dc6950446a3e4ccac43bc3070721856/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8", size = 162280, upload-time = "2025-10-14T04:40:16.14Z" },
- { url = "https://files.pythonhosted.org/packages/3e/8e/6aa1952f56b192f54921c436b87f2aaf7c7a7c3d0d1a765547d64fd83c13/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d", size = 159454, upload-time = "2025-10-14T04:40:17.567Z" },
- { url = "https://files.pythonhosted.org/packages/36/3b/60cbd1f8e93aa25d1c669c649b7a655b0b5fb4c571858910ea9332678558/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313", size = 153609, upload-time = "2025-10-14T04:40:19.08Z" },
- { url = "https://files.pythonhosted.org/packages/64/91/6a13396948b8fd3c4b4fd5bc74d045f5637d78c9675585e8e9fbe5636554/charset_normalizer-3.4.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e", size = 151849, upload-time = "2025-10-14T04:40:20.607Z" },
- { url = "https://files.pythonhosted.org/packages/b7/7a/59482e28b9981d105691e968c544cc0df3b7d6133152fb3dcdc8f135da7a/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93", size = 151586, upload-time = "2025-10-14T04:40:21.719Z" },
- { url = "https://files.pythonhosted.org/packages/92/59/f64ef6a1c4bdd2baf892b04cd78792ed8684fbc48d4c2afe467d96b4df57/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0", size = 145290, upload-time = "2025-10-14T04:40:23.069Z" },
- { url = "https://files.pythonhosted.org/packages/6b/63/3bf9f279ddfa641ffa1962b0db6a57a9c294361cc2f5fcac997049a00e9c/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84", size = 163663, upload-time = "2025-10-14T04:40:24.17Z" },
- { url = "https://files.pythonhosted.org/packages/ed/09/c9e38fc8fa9e0849b172b581fd9803bdf6e694041127933934184e19f8c3/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e", size = 151964, upload-time = "2025-10-14T04:40:25.368Z" },
- { url = "https://files.pythonhosted.org/packages/d2/d1/d28b747e512d0da79d8b6a1ac18b7ab2ecfd81b2944c4c710e166d8dd09c/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db", size = 161064, upload-time = "2025-10-14T04:40:26.806Z" },
- { url = "https://files.pythonhosted.org/packages/bb/9a/31d62b611d901c3b9e5500c36aab0ff5eb442043fb3a1c254200d3d397d9/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6", size = 155015, upload-time = "2025-10-14T04:40:28.284Z" },
- { url = "https://files.pythonhosted.org/packages/1f/f3/107e008fa2bff0c8b9319584174418e5e5285fef32f79d8ee6a430d0039c/charset_normalizer-3.4.4-cp310-cp310-win32.whl", hash = "sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f", size = 99792, upload-time = "2025-10-14T04:40:29.613Z" },
- { url = "https://files.pythonhosted.org/packages/eb/66/e396e8a408843337d7315bab30dbf106c38966f1819f123257f5520f8a96/charset_normalizer-3.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d", size = 107198, upload-time = "2025-10-14T04:40:30.644Z" },
- { url = "https://files.pythonhosted.org/packages/b5/58/01b4f815bf0312704c267f2ccb6e5d42bcc7752340cd487bc9f8c3710597/charset_normalizer-3.4.4-cp310-cp310-win_arm64.whl", hash = "sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69", size = 100262, upload-time = "2025-10-14T04:40:32.108Z" },
{ url = "https://files.pythonhosted.org/packages/ed/27/c6491ff4954e58a10f69ad90aca8a1b6fe9c5d3c6f380907af3c37435b59/charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8", size = 206988, upload-time = "2025-10-14T04:40:33.79Z" },
{ url = "https://files.pythonhosted.org/packages/94/59/2e87300fe67ab820b5428580a53cad894272dbb97f38a7a814a2a1ac1011/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0", size = 147324, upload-time = "2025-10-14T04:40:34.961Z" },
{ url = "https://files.pythonhosted.org/packages/07/fb/0cf61dc84b2b088391830f6274cb57c82e4da8bbc2efeac8c025edb88772/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3", size = 142742, upload-time = "2025-10-14T04:40:36.105Z" },
@@ -371,18 +328,6 @@ version = "7.13.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/23/f9/e92df5e07f3fc8d4c7f9a0f146ef75446bf870351cd37b788cf5897f8079/coverage-7.13.1.tar.gz", hash = "sha256:b7593fe7eb5feaa3fbb461ac79aac9f9fc0387a5ca8080b0c6fe2ca27b091afd", size = 825862, upload-time = "2025-12-28T15:42:56.969Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/2d/9a/3742e58fd04b233df95c012ee9f3dfe04708a5e1d32613bd2d47d4e1be0d/coverage-7.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e1fa280b3ad78eea5be86f94f461c04943d942697e0dac889fa18fff8f5f9147", size = 218633, upload-time = "2025-12-28T15:40:10.165Z" },
- { url = "https://files.pythonhosted.org/packages/7e/45/7e6bdc94d89cd7c8017ce735cf50478ddfe765d4fbf0c24d71d30ea33d7a/coverage-7.13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c3d8c679607220979434f494b139dfb00131ebf70bb406553d69c1ff01a5c33d", size = 219147, upload-time = "2025-12-28T15:40:12.069Z" },
- { url = "https://files.pythonhosted.org/packages/f7/38/0d6a258625fd7f10773fe94097dc16937a5f0e3e0cdf3adef67d3ac6baef/coverage-7.13.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:339dc63b3eba969067b00f41f15ad161bf2946613156fb131266d8debc8e44d0", size = 245894, upload-time = "2025-12-28T15:40:13.556Z" },
- { url = "https://files.pythonhosted.org/packages/27/58/409d15ea487986994cbd4d06376e9860e9b157cfbfd402b1236770ab8dd2/coverage-7.13.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db622b999ffe49cb891f2fff3b340cdc2f9797d01a0a202a0973ba2562501d90", size = 247721, upload-time = "2025-12-28T15:40:15.37Z" },
- { url = "https://files.pythonhosted.org/packages/da/bf/6e8056a83fd7a96c93341f1ffe10df636dd89f26d5e7b9ca511ce3bcf0df/coverage-7.13.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1443ba9acbb593fa7c1c29e011d7c9761545fe35e7652e85ce7f51a16f7e08d", size = 249585, upload-time = "2025-12-28T15:40:17.226Z" },
- { url = "https://files.pythonhosted.org/packages/f4/15/e1daff723f9f5959acb63cbe35b11203a9df77ee4b95b45fffd38b318390/coverage-7.13.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c832ec92c4499ac463186af72f9ed4d8daec15499b16f0a879b0d1c8e5cf4a3b", size = 246597, upload-time = "2025-12-28T15:40:19.028Z" },
- { url = "https://files.pythonhosted.org/packages/74/a6/1efd31c5433743a6ddbc9d37ac30c196bb07c7eab3d74fbb99b924c93174/coverage-7.13.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:562ec27dfa3f311e0db1ba243ec6e5f6ab96b1edfcfc6cf86f28038bc4961ce6", size = 247626, upload-time = "2025-12-28T15:40:20.846Z" },
- { url = "https://files.pythonhosted.org/packages/6d/9f/1609267dd3e749f57fdd66ca6752567d1c13b58a20a809dc409b263d0b5f/coverage-7.13.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4de84e71173d4dada2897e5a0e1b7877e5eefbfe0d6a44edee6ce31d9b8ec09e", size = 245629, upload-time = "2025-12-28T15:40:22.397Z" },
- { url = "https://files.pythonhosted.org/packages/e2/f6/6815a220d5ec2466383d7cc36131b9fa6ecbe95c50ec52a631ba733f306a/coverage-7.13.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:a5a68357f686f8c4d527a2dc04f52e669c2fc1cbde38f6f7eb6a0e58cbd17cae", size = 245901, upload-time = "2025-12-28T15:40:23.836Z" },
- { url = "https://files.pythonhosted.org/packages/ac/58/40576554cd12e0872faf6d2c0eb3bc85f71d78427946ddd19ad65201e2c0/coverage-7.13.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:77cc258aeb29a3417062758975521eae60af6f79e930d6993555eeac6a8eac29", size = 246505, upload-time = "2025-12-28T15:40:25.421Z" },
- { url = "https://files.pythonhosted.org/packages/3b/77/9233a90253fba576b0eee81707b5781d0e21d97478e5377b226c5b096c0f/coverage-7.13.1-cp310-cp310-win32.whl", hash = "sha256:bb4f8c3c9a9f34423dba193f241f617b08ffc63e27f67159f60ae6baf2dcfe0f", size = 221257, upload-time = "2025-12-28T15:40:27.217Z" },
- { url = "https://files.pythonhosted.org/packages/e0/43/e842ff30c1a0a623ec80db89befb84a3a7aad7bfe44a6ea77d5a3e61fedd/coverage-7.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:c8e2706ceb622bc63bac98ebb10ef5da80ed70fbd8a7999a5076de3afaef0fb1", size = 222191, upload-time = "2025-12-28T15:40:28.916Z" },
{ url = "https://files.pythonhosted.org/packages/b4/9b/77baf488516e9ced25fc215a6f75d803493fc3f6a1a1227ac35697910c2a/coverage-7.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a55d509a1dc5a5b708b5dad3b5334e07a16ad4c2185e27b40e4dba796ab7f88", size = 218755, upload-time = "2025-12-28T15:40:30.812Z" },
{ url = "https://files.pythonhosted.org/packages/d7/cd/7ab01154e6eb79ee2fab76bf4d89e94c6648116557307ee4ebbb85e5c1bf/coverage-7.13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4d010d080c4888371033baab27e47c9df7d6fb28d0b7b7adf85a4a49be9298b3", size = 219257, upload-time = "2025-12-28T15:40:32.333Z" },
{ url = "https://files.pythonhosted.org/packages/01/d5/b11ef7863ffbbdb509da0023fad1e9eda1c0eaea61a6d2ea5b17d4ac706e/coverage-7.13.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d938b4a840fb1523b9dfbbb454f652967f18e197569c32266d4d13f37244c3d9", size = 249657, upload-time = "2025-12-28T15:40:34.1Z" },
@@ -475,7 +420,6 @@ version = "46.0.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cffi", marker = "platform_python_implementation != 'PyPy'" },
- { name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/9f/33/c00162f49c0e2fe8064a62cb92b93e50c74a72bc370ab92f86112b33ff62/cryptography-46.0.3.tar.gz", hash = "sha256:a8b17438104fed022ce745b362294d9ce35b4c2e45c1d958ad4a4b019285f4a1", size = 749258, upload-time = "2025-10-15T23:18:31.74Z" }
wheels = [
@@ -524,8 +468,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0a/6e/1c8331ddf91ca4730ab3086a0f1be19c65510a33b5a441cb334e7a2d2560/cryptography-46.0.3-cp38-abi3-win32.whl", hash = "sha256:6276eb85ef938dc035d59b87c8a7dc559a232f954962520137529d77b18ff1df", size = 3036695, upload-time = "2025-10-15T23:18:08.672Z" },
{ url = "https://files.pythonhosted.org/packages/90/45/b0d691df20633eff80955a0fc7695ff9051ffce8b69741444bd9ed7bd0db/cryptography-46.0.3-cp38-abi3-win_amd64.whl", hash = "sha256:416260257577718c05135c55958b674000baef9a1c7d9e8f306ec60d71db850f", size = 3501720, upload-time = "2025-10-15T23:18:10.632Z" },
{ url = "https://files.pythonhosted.org/packages/e8/cb/2da4cc83f5edb9c3257d09e1e7ab7b23f049c7962cae8d842bbef0a9cec9/cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372", size = 2918740, upload-time = "2025-10-15T23:18:12.277Z" },
- { url = "https://files.pythonhosted.org/packages/d9/cd/1a8633802d766a0fa46f382a77e096d7e209e0817892929655fe0586ae32/cryptography-46.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a23582810fedb8c0bc47524558fb6c56aac3fc252cb306072fd2815da2a47c32", size = 3689163, upload-time = "2025-10-15T23:18:13.821Z" },
- { url = "https://files.pythonhosted.org/packages/4c/59/6b26512964ace6480c3e54681a9859c974172fb141c38df11eadd8416947/cryptography-46.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e7aec276d68421f9574040c26e2a7c3771060bc0cff408bae1dcb19d3ab1e63c", size = 3429474, upload-time = "2025-10-15T23:18:15.477Z" },
{ url = "https://files.pythonhosted.org/packages/06/8a/e60e46adab4362a682cf142c7dcb5bf79b782ab2199b0dcb81f55970807f/cryptography-46.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7ce938a99998ed3c8aa7e7272dca1a610401ede816d36d0693907d863b10d9ea", size = 3698132, upload-time = "2025-10-15T23:18:17.056Z" },
{ url = "https://files.pythonhosted.org/packages/da/38/f59940ec4ee91e93d3311f7532671a5cef5570eb04a144bf203b58552d11/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:191bb60a7be5e6f54e30ba16fdfae78ad3a342a0599eb4193ba88e3f3d6e185b", size = 4243992, upload-time = "2025-10-15T23:18:18.695Z" },
{ url = "https://files.pythonhosted.org/packages/b0/0c/35b3d92ddebfdfda76bb485738306545817253d0a3ded0bfe80ef8e67aa5/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c70cc23f12726be8f8bc72e41d5065d77e4515efae3690326764ea1b07845cfb", size = 4409944, upload-time = "2025-10-15T23:18:20.597Z" },
@@ -544,6 +486,7 @@ dependencies = [
{ name = "duckdb" },
{ name = "duckdb-engine" },
{ name = "fastapi" },
+ { name = "google-cloud-bigquery" },
{ name = "importlib-metadata" },
{ name = "msgpack" },
{ name = "psycopg", extra = ["pool"] },
@@ -592,12 +535,13 @@ requires-dist = [
{ name = "accept-types", specifier = "==0.4.1" },
{ name = "alembic", specifier = ">=1.7.7" },
{ name = "cachelib", specifier = ">=0.4.0" },
- { name = "duckdb", specifier = "==0.8.1" },
+ { name = "duckdb", specifier = ">=1.0.0" },
{ name = "duckdb-engine" },
{ name = "fastapi", specifier = ">=0.79.0" },
+ { name = "google-cloud-bigquery", specifier = ">=3.11.0" },
{ name = "importlib-metadata" },
{ name = "msgpack", specifier = ">=1.0.3" },
- { name = "psycopg", extras = ["async", "pool"], specifier = ">=3.2.1" },
+ { name = "psycopg", extras = ["pool"], specifier = ">=3.2.1" },
{ name = "pytest-asyncio", specifier = ">=0.24.0" },
{ name = "pytest-integration", specifier = ">=0.2.3" },
{ name = "python-dotenv", specifier = ">=0.21.0" },
@@ -621,7 +565,7 @@ test = [
{ name = "httpx", specifier = ">=0.24.1" },
{ name = "pip-tools", specifier = ">=6.4.0" },
{ name = "pre-commit", specifier = ">=3.2.2" },
- { name = "psycopg", extras = ["async", "pool"], specifier = ">=3.2.1" },
+ { name = "psycopg", extras = ["pool"], specifier = ">=3.2.1" },
{ name = "pydruid", specifier = ">=0.6.4" },
{ name = "pyfakefs", specifier = ">=4.5.1" },
{ name = "pylint", specifier = ">=2.15.3" },
@@ -669,30 +613,38 @@ wheels = [
[[package]]
name = "duckdb"
-version = "0.8.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/cc/69/49e495d5e75d60410144efa50134713b1695a588e36dd8af72a78eef9f03/duckdb-0.8.1.tar.gz", hash = "sha256:a54d37f4abc2afc4f92314aaa56ecf215a411f40af4bffe1e86bd25e62aceee9", size = 10421102, upload-time = "2023-06-13T12:29:04.295Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/fd/c1/3d4cef11f02af5c25d1d4d6dc8f15967ac3e4a2a806adce84cb695440b37/duckdb-0.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:14781d21580ee72aba1f5dcae7734674c9b6c078dd60470a08b2b420d15b996d", size = 27317784, upload-time = "2023-06-13T10:35:42.348Z" },
- { url = "https://files.pythonhosted.org/packages/65/05/cd7088defc381df9ac63fa379f4f7ce7f7bc100679eabbdd6013c8dc3d9a/duckdb-0.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f13bf7ab0e56ddd2014ef762ae4ee5ea4df5a69545ce1191b8d7df8118ba3167", size = 14664958, upload-time = "2023-06-13T10:35:47.034Z" },
- { url = "https://files.pythonhosted.org/packages/ec/9e/c8724532dc5b57c1336b5b45250c6a101a4d338560d16336c9f9fcc1764c/duckdb-0.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4032042d8363e55365bbca3faafc6dc336ed2aad088f10ae1a534ebc5bcc181", size = 12632512, upload-time = "2023-06-13T10:35:51.942Z" },
- { url = "https://files.pythonhosted.org/packages/c0/77/6ca11ed3e339d7fe8acd38ad75eca886ce2403f9fddf9f4f503d42d7093e/duckdb-0.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a71bd8f0b0ca77c27fa89b99349ef22599ffefe1e7684ae2e1aa2904a08684", size = 14798677, upload-time = "2023-06-13T15:46:23.659Z" },
- { url = "https://files.pythonhosted.org/packages/df/c6/e30f5a865d23f94d903f9731bb1fc9f113e329f0862564fd5363cd55d039/duckdb-0.8.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24568d6e48f3dbbf4a933109e323507a46b9399ed24c5d4388c4987ddc694fd0", size = 17713728, upload-time = "2023-06-13T13:11:28.127Z" },
- { url = "https://files.pythonhosted.org/packages/0c/a3/4e52ef89606292b26864bcc3be3e36e1345ba4bb8a6df5b2fa36dfc01fd7/duckdb-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:297226c0dadaa07f7c5ae7cbdb9adba9567db7b16693dbd1b406b739ce0d7924", size = 15880589, upload-time = "2023-06-13T12:42:16.21Z" },
- { url = "https://files.pythonhosted.org/packages/80/4d/140619aa73d3de8ae5ea903dba34257ce129c01e77f0db1f18ffaf9dfe71/duckdb-0.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5792cf777ece2c0591194006b4d3e531f720186102492872cb32ddb9363919cf", size = 18267381, upload-time = "2023-06-13T13:11:31.764Z" },
- { url = "https://files.pythonhosted.org/packages/7a/73/edfe5e3f845eda8cf3db91af7550e9dbfc9e57ce0b2990e4c693fcfeb364/duckdb-0.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:12803f9f41582b68921d6b21f95ba7a51e1d8f36832b7d8006186f58c3d1b344", size = 16476321, upload-time = "2023-06-13T12:42:20.412Z" },
- { url = "https://files.pythonhosted.org/packages/9e/13/ab717b2924313c51e631e1fde8fb2cd5d3daa87a051fbd046d05bc7a22da/duckdb-0.8.1-cp310-cp310-win32.whl", hash = "sha256:d0953d5a2355ddc49095e7aef1392b7f59c5be5cec8cdc98b9d9dc1f01e7ce2b", size = 8754102, upload-time = "2023-06-13T16:38:56.889Z" },
- { url = "https://files.pythonhosted.org/packages/6d/14/12d9c847a39c1d15dd3bf853d5e1d9f333dd3f97e1876750808d2e3bfe6e/duckdb-0.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:6e6583c98a7d6637e83bcadfbd86e1f183917ea539f23b6b41178f32f813a5eb", size = 9815141, upload-time = "2023-06-13T16:39:00.067Z" },
- { url = "https://files.pythonhosted.org/packages/83/35/acc4056703190e3203bfa7cc06bb000afaf24e8b0e63614901bebf20ae2b/duckdb-0.8.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fad7ed0d4415f633d955ac24717fa13a500012b600751d4edb050b75fb940c25", size = 27317508, upload-time = "2023-06-13T10:37:54.974Z" },
- { url = "https://files.pythonhosted.org/packages/75/da/81565f1cd3210442fc130389da0627e2e6b72db2dd173ab03cd277f64916/duckdb-0.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:81ae602f34d38d9c48dd60f94b89f28df3ef346830978441b83c5b4eae131d08", size = 14664994, upload-time = "2023-06-13T10:37:58.539Z" },
- { url = "https://files.pythonhosted.org/packages/af/af/7fc3aca20d65d9851cdf7e25b5c0ca41954c0155c4f4d907c1cfd3382411/duckdb-0.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d75cfe563aaa058d3b4ccaaa371c6271e00e3070df5de72361fd161b2fe6780", size = 12632544, upload-time = "2023-06-13T10:38:01.202Z" },
- { url = "https://files.pythonhosted.org/packages/b2/80/207a50e4dcfa2c7e04b4bb7b4ccd2dc27a19e7ef9f74677f1de5be4e85d7/duckdb-0.8.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8dbb55e7a3336f2462e5e916fc128c47fe1c03b6208d6bd413ac11ed95132aa0", size = 14799218, upload-time = "2023-06-13T14:14:23.605Z" },
- { url = "https://files.pythonhosted.org/packages/d4/5f/bafa326bb094b4b9400810a3654a21ada3ec78648ac7028d24c62b09ff8e/duckdb-0.8.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6df53efd63b6fdf04657385a791a4e3c4fb94bfd5db181c4843e2c46b04fef5", size = 17712863, upload-time = "2023-06-13T13:17:04.758Z" },
- { url = "https://files.pythonhosted.org/packages/79/22/f2b229ec92d559a30bba996e85feba966fe287c6fe837b9b388ddf921c1e/duckdb-0.8.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b188b80b70d1159b17c9baaf541c1799c1ce8b2af4add179a9eed8e2616be96", size = 15880481, upload-time = "2023-06-13T12:29:21.683Z" },
- { url = "https://files.pythonhosted.org/packages/2e/66/f66d99eea100eaa656862f396eead4fb01b0d487aef4c7acfac993393c47/duckdb-0.8.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5ad481ee353f31250b45d64b4a104e53b21415577943aa8f84d0af266dc9af85", size = 18266623, upload-time = "2023-06-13T13:17:10.66Z" },
- { url = "https://files.pythonhosted.org/packages/2f/5c/935f12ce8fb1f1bf546ef939254213d361d1faf55f1f310c9d34fbdc0d4a/duckdb-0.8.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d1d1b1729993611b1892509d21c21628917625cdbe824a61ce891baadf684b32", size = 16477164, upload-time = "2023-06-13T12:29:24.937Z" },
- { url = "https://files.pythonhosted.org/packages/76/5f/36ff597cacb235884d3c19cb182facb86a1502e08a24a114e21f1fc77551/duckdb-0.8.1-cp311-cp311-win32.whl", hash = "sha256:2d8f9cc301e8455a4f89aa1088b8a2d628f0c1f158d4cf9bc78971ed88d82eea", size = 8753987, upload-time = "2023-06-13T16:52:25.976Z" },
- { url = "https://files.pythonhosted.org/packages/ee/af/e3fdfa4776124e797d8303f85f3bd97e17f7a1aa1a90189b545369d267d1/duckdb-0.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:07457a43605223f62d93d2a5a66b3f97731f79bbbe81fdd5b79954306122f612", size = 9816167, upload-time = "2023-06-13T16:52:29.185Z" },
+version = "1.5.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9c/c2/b62ec24d57bb8df4e24b0b58f7f8facb32f5fdb9f1895aed9e9fcdded168/duckdb-1.5.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b543841b0ae18a9c982345cfa3987e9c065d3a4b0f067daa473d92d1e65f528", size = 32708371, upload-time = "2026-07-22T10:53:41.642Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/ce/769171ba45f0b73632dc3bc3108d891e81dd6c6bbfba630a34a75b4dcc0f/duckdb-1.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a925d06c2a4c3b64553d6cc1aced5028d376d4479bed689a7d47e9b1dccd80a", size = 17343979, upload-time = "2026-07-22T10:53:44.951Z" },
+ { url = "https://files.pythonhosted.org/packages/46/59/a8e3384ee916e00d5dcf985194c1511d61978540778a1e96fa47f9fb3e0d/duckdb-1.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0c42757cb34722144bd4dfb94b6f336339e7b2468f6813fa7fa9a319ba07bab4", size = 15493704, upload-time = "2026-07-22T10:53:47.912Z" },
+ { url = "https://files.pythonhosted.org/packages/6f/1d/9840179c2607b90523a2884a129c4d4e6dbdc1178ba62a976c1043beba88/duckdb-1.5.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e72f9e1a4f90a5c8483ad4d540e495bf0834ba61c360b52499a573d7ed62a3f", size = 19366574, upload-time = "2026-07-22T10:53:51.876Z" },
+ { url = "https://files.pythonhosted.org/packages/b5/55/f9641a4eebcc2f4df631287d6c3b9ed2eea3b92644f93acbad825e3972b6/duckdb-1.5.5-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b6f86ed85d4ef5e0211eaebf75d057bd8bb520bba438a95dd0f4e42234bbfe", size = 21477952, upload-time = "2026-07-22T10:53:55.575Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/3a/07c3556e37a5c97b95917b029c8fdde4a25fbd76a660bacdac195cf20dcb/duckdb-1.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:9f4287f97ccf0c1f3d471e7115be2b067cbf99627e2d34bffd462dd64703cddc", size = 13156986, upload-time = "2026-07-22T10:53:58.823Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/ff/07b48eef2078ca033847e9caa46cc7633b714c5f91ad1ce091c8ca89d792/duckdb-1.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:179633a3fc6296c75d57c69c1e239fa9e5cdcb670fd1dbff88a02663f932905c", size = 14001317, upload-time = "2026-07-22T10:54:01.724Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/40/2e05d324400fdaa5656c9f48d6298da421cb034d85e509fa0e6e325cf04b/duckdb-1.5.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d4dd65f8941a604b947e0b9b4b4f7165988e29a23ec0b69b4038520956d9933e", size = 32753858, upload-time = "2026-07-22T10:54:05.514Z" },
+ { url = "https://files.pythonhosted.org/packages/79/15/5ceb58ffb5bb8a62b3fd7abb39c41467cdf94850ece02e6d88664dfc75ce/duckdb-1.5.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33db46679b071f108d57139493dee2d37e1f5efcf5c5c039c2969eed11a6c8a7", size = 17368293, upload-time = "2026-07-22T10:54:09.139Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/5c/bf02da0b354fe83cca4f95a4fbf762181af466f7d551ab2a093f7698882a/duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae", size = 15509131, upload-time = "2026-07-22T10:54:12.228Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/a9/5f1f09da421d8e930e0b063d11c1b3f90363f40ede74438cd188afdd13a2/duckdb-1.5.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f316eae2323d9a851883fdf2dee91c1f9efe251ab33e14a2272f82a913422ed6", size = 19391959, upload-time = "2026-07-22T10:54:15.551Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/98/6549769f158126fa64fd6c1ac2eb59a18282146c939867a3eb31b7c1db07/duckdb-1.5.5-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a6d2d11859d82a936ebdcb30ce3d8a1cbb3e990bff05c12abb9b54c44fa7bd1", size = 21510909, upload-time = "2026-07-22T10:54:19.681Z" },
+ { url = "https://files.pythonhosted.org/packages/af/b7/5753b41d3124838f868f9f523362812d9fc45409e9e4dd70dcbb0a25826e/duckdb-1.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:ddfbdb096c11d51ee22492397d342c90a82e62c5d09961477895934d0a25372f", size = 13168544, upload-time = "2026-07-22T10:54:22.789Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/28/44b679c7d46245f8398feae7edac959d1b83d4eb143e25b3fce0630b78bd/duckdb-1.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:2725d2b9ace3a4e75d72fc5a239f6a44b502c580edadb8fb2676db772c5f9282", size = 13988684, upload-time = "2026-07-22T10:54:26.003Z" },
+ { url = "https://files.pythonhosted.org/packages/47/37/4a38116e7700720fd152c666292214fd3abdf916496991296d8d1f66efbf/duckdb-1.5.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd98829b67788609017e65c761bd42a5dd0f9129441bed8bda4d6881ccf819f0", size = 32754294, upload-time = "2026-07-22T10:54:29.822Z" },
+ { url = "https://files.pythonhosted.org/packages/66/42/7d392f1ba1eee0eaf4ab4c8c7a604bfe3536cd63f979cf5c98798664f807/duckdb-1.5.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feead93c56679b79592d437c62975d39cb67adedffa7592c763baf8160ac7366", size = 17368211, upload-time = "2026-07-22T10:54:33.359Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/a5/0a6f4fa60562faa615e55e15bd1953a2f2b17a8edd8105e5cda215e43457/duckdb-1.5.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49c963d9469373d7aba8d750d9ea565ab823e94166efed953f184dd9b169b98c", size = 15509136, upload-time = "2026-07-22T10:54:36.369Z" },
+ { url = "https://files.pythonhosted.org/packages/e4/cb/023c89f51978545b9fab318581bba0c457a58e7530d2d933e54ae7d8647c/duckdb-1.5.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a736217825461732b5442d05a220f3da2e23a0dae114efbf08c9bf171b53098a", size = 19392147, upload-time = "2026-07-22T10:54:39.551Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/c5/41bef391fb8b23dbc133c9f2ba016e7a7a8124513d2cc1b430f1897d87e4/duckdb-1.5.5-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:078e6a60dd8eedde5832f45422ca5c4a6b8c837aeabd8a56ca0b7d933f588053", size = 21511060, upload-time = "2026-07-22T10:54:42.788Z" },
+ { url = "https://files.pythonhosted.org/packages/07/9f/c44dfc1f924ac29b3252dc1b91393c01d009dbfe9f8ed33f10b986151bd1/duckdb-1.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:6826504277dba513c0c5d71d828456c94d729c9d2482f94b2e289f90a9167e28", size = 13168028, upload-time = "2026-07-22T10:54:46.127Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/88/591384b2cd59abddd6f5dc175e60374f9abae6064429f0c4402854c10f44/duckdb-1.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:baa9c5702002fabb559ded2a39008f9f421fcbc7237d388b8213eff1e08858de", size = 13989955, upload-time = "2026-07-22T10:54:49.262Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" },
+ { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" },
+ { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" },
]
[[package]]
@@ -709,18 +661,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/a2/e90242f53f7ae41554419b1695b4820b364df87c8350aa420b60b20cab92/duckdb_engine-0.17.0-py3-none-any.whl", hash = "sha256:3aa72085e536b43faab635f487baf77ddc5750069c16a2f8d9c6c3cb6083e979", size = 49676, upload-time = "2025-03-29T09:49:15.564Z" },
]
-[[package]]
-name = "exceptiongroup"
-version = "1.3.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "typing-extensions", marker = "python_full_version < '3.11'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" },
-]
-
[[package]]
name = "fastapi"
version = "0.128.0"
@@ -757,19 +697,137 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/5e/2e/b41d8a1a917d6581fc27a35d05561037b048e47df50f27f8ac9c7e27a710/freezegun-1.5.5-py3-none-any.whl", hash = "sha256:cd557f4a75cf074e84bc374249b9dd491eaeacd61376b9eb3c423282211619d2", size = 19266, upload-time = "2025-08-09T10:39:06.636Z" },
]
+[[package]]
+name = "google-api-core"
+version = "2.30.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "google-auth" },
+ { name = "googleapis-common-protos" },
+ { name = "proto-plus" },
+ { name = "protobuf" },
+ { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" },
+]
+
+[package.optional-dependencies]
+grpc = [
+ { name = "grpcio" },
+ { name = "grpcio-status" },
+]
+
+[[package]]
+name = "google-auth"
+version = "2.57.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cryptography" },
+ { name = "pyasn1-modules" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/41/64/55f316b729f92a552d26e00aa3b1542b2e149d0a5efe2842afff0cac7af7/google_auth-2.57.0.tar.gz", hash = "sha256:9b4f96d6a1feb5f7201231f47cfb3de08d8f176f8a61f9e461555116e95a8789", size = 370794, upload-time = "2026-08-25T19:18:26.419Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/00/f3/8508a702c094af5f6e89773f4dfdeee74913df0f41a02c21b5e7dc3d75cd/google_auth-2.57.0-py3-none-any.whl", hash = "sha256:180dafe015cfb62193bea26b677500fab5b9fd51a1e825ebf3ad9b182047ae59", size = 259728, upload-time = "2026-08-24T21:55:08.449Z" },
+]
+
+[package.optional-dependencies]
+pyopenssl = [
+ { name = "cryptography" },
+]
+
+[[package]]
+name = "google-cloud-bigquery"
+version = "3.44.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "google-api-core", extra = ["grpc"] },
+ { name = "google-auth", extra = ["pyopenssl"] },
+ { name = "google-cloud-core" },
+ { name = "google-resumable-media" },
+ { name = "packaging" },
+ { name = "python-dateutil" },
+ { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2f/fb/86d5bfdfbd9d810f6a49eb3e9cc591a2b46d33262338acc244cc79d032c1/google_cloud_bigquery-3.44.0.tar.gz", hash = "sha256:30651ae469b419f450b9c96581fd4942e2e060490df1ac0314bf379f16883215", size = 527575, upload-time = "2026-08-25T19:18:36.317Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2d/84/af193ca97ce72b56fd05f8ad15776bf7446d99cb0ba1cd2f4880dbc52543/google_cloud_bigquery-3.44.0-py3-none-any.whl", hash = "sha256:ac2f0a6ab61a3c742ba4674dc220fb98461c13e1def96fe7a980ef7d5e0c0285", size = 267691, upload-time = "2026-08-24T21:55:19.249Z" },
+]
+
+[[package]]
+name = "google-cloud-core"
+version = "2.7.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "google-api-core" },
+ { name = "google-auth" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f9/54/790548b190cff9cf07225c811d2668eacd8f2cbf04114475cbe3e5738752/google_cloud_core-2.7.0.tar.gz", hash = "sha256:874aaf89765db87a9b911b7a2ca7c5068554868eed9e75c7766affe342a2913d", size = 36603, upload-time = "2026-08-25T19:18:43.076Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/cb/64/904dbc9bee128e7b87eeb60da67c8fa4d1a8acdc9a45dd2fedca67ef184d/google_cloud_core-2.7.0-py3-none-any.whl", hash = "sha256:c18a250904cfdda021eb3ae8b8238c9f9ca272a4cbbfb5cba946b3fe3022eed1", size = 31046, upload-time = "2026-08-24T21:55:28.36Z" },
+]
+
+[[package]]
+name = "google-crc32c"
+version = "1.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5d/ef/21ccfaab3d5078d41efe8612e0ed0bfc9ce22475de074162a91a25f7980d/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8", size = 31298, upload-time = "2025-12-16T00:20:32.241Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/b8/f8413d3f4b676136e965e764ceedec904fe38ae8de0cdc52a12d8eb1096e/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7", size = 30872, upload-time = "2025-12-16T00:33:58.785Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/fd/33aa4ec62b290477181c55bb1c9302c9698c58c0ce9a6ab4874abc8b0d60/google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15", size = 33243, upload-time = "2025-12-16T00:40:21.46Z" },
+ { url = "https://files.pythonhosted.org/packages/71/03/4820b3bd99c9653d1a5210cb32f9ba4da9681619b4d35b6a052432df4773/google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a", size = 33608, upload-time = "2025-12-16T00:40:22.204Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/43/acf61476a11437bf9733fb2f70599b1ced11ec7ed9ea760fdd9a77d0c619/google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2", size = 34439, upload-time = "2025-12-16T00:35:20.458Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" },
+ { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" },
+ { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" },
+ { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" },
+ { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" },
+ { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" },
+ { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" },
+ { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" },
+ { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" },
+ { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" },
+ { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" },
+ { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" },
+ { url = "https://files.pythonhosted.org/packages/52/c5/c171e4d8c44fec1422d801a6d2e5d7ddabd733eeda505c79730ee9607f07/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93", size = 28615, upload-time = "2025-12-16T00:40:29.298Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" },
+]
+
+[[package]]
+name = "google-resumable-media"
+version = "2.10.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "google-crc32c" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/17/4b/44e128cd7b0e72cd26da4a3052cd8a9ef1b5a890be7c611558067ea0b354/google_resumable_media-2.10.2.tar.gz", hash = "sha256:1de441703cd298d75a419bfdc0066e9fc7b0a1de630df96eea8ce8f5c759358c", size = 2164918, upload-time = "2026-08-25T19:19:11.814Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/55/30/3e976681b5319715ab022363f58f6f8e10f546a1777756fe3ebd918857e4/google_resumable_media-2.10.2-py3-none-any.whl", hash = "sha256:e3cedc827a4ea41e216582d74346f1fb9fceb625a8c3c53912f2ca1d663334d7", size = 81531, upload-time = "2026-08-25T19:18:07.193Z" },
+]
+
+[[package]]
+name = "googleapis-common-protos"
+version = "1.75.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c0/90/fb8f1c84537fbf210c1f53a53ae473a805f6599c5a40b93c1bbadd211f7a/googleapis_common_protos-1.75.2.tar.gz", hash = "sha256:8829a3d1e4508c5b7b9a6b9525f7fccff611f8531644579a76466c29295d4bb2", size = 154083, upload-time = "2026-08-25T19:19:13.028Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/47/5b/1c9e55363c3b1890a98cae813de5b4ea327845756cd8fb7ee690140c7eac/googleapis_common_protos-1.75.2-py3-none-any.whl", hash = "sha256:6b83302f554ea93a0f48409c7fc2050f954bcbcddb7e3a9c76d4a823cb22920e", size = 307002, upload-time = "2026-08-25T19:18:08.927Z" },
+]
+
[[package]]
name = "greenlet"
version = "3.3.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/8a/99/1cd3411c56a410994669062bd73dd58270c00cc074cac15f385a1fd91f8a/greenlet-3.3.1.tar.gz", hash = "sha256:41848f3230b58c08bb43dee542e74a2a2e34d3c59dc3076cec9151aeeedcae98", size = 184690, upload-time = "2026-01-23T15:31:02.076Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/fe/65/5b235b40581ad75ab97dcd8b4218022ae8e3ab77c13c919f1a1dfe9171fd/greenlet-3.3.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:04bee4775f40ecefcdaa9d115ab44736cd4b9c5fba733575bfe9379419582e13", size = 273723, upload-time = "2026-01-23T15:30:37.521Z" },
- { url = "https://files.pythonhosted.org/packages/ce/ad/eb4729b85cba2d29499e0a04ca6fbdd8f540afd7be142fd571eea43d712f/greenlet-3.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e1457f4fed12a50e427988a07f0f9df53cf0ee8da23fab16e6732c2ec909d4", size = 574874, upload-time = "2026-01-23T16:00:54.551Z" },
- { url = "https://files.pythonhosted.org/packages/87/32/57cad7fe4c8b82fdaa098c89498ef85ad92dfbb09d5eb713adedfc2ae1f5/greenlet-3.3.1-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:070472cd156f0656f86f92e954591644e158fd65aa415ffbe2d44ca77656a8f5", size = 586309, upload-time = "2026-01-23T16:05:25.18Z" },
- { url = "https://files.pythonhosted.org/packages/87/eb/8a1ec2da4d55824f160594a75a9d8354a5fe0a300fb1c48e7944265217e1/greenlet-3.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a300354f27dd86bae5fbf7002e6dd2b3255cd372e9242c933faf5e859b703fe", size = 586985, upload-time = "2026-01-23T15:32:47.968Z" },
- { url = "https://files.pythonhosted.org/packages/15/1c/0621dd4321dd8c351372ee8f9308136acb628600658a49be1b7504208738/greenlet-3.3.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e84b51cbebf9ae573b5fbd15df88887815e3253fc000a7d0ff95170e8f7e9729", size = 1547271, upload-time = "2026-01-23T16:04:18.977Z" },
- { url = "https://files.pythonhosted.org/packages/9d/53/24047f8924c83bea7a59c8678d9571209c6bfe5f4c17c94a78c06024e9f2/greenlet-3.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0093bd1a06d899892427217f0ff2a3c8f306182b8c754336d32e2d587c131b4", size = 1613427, upload-time = "2026-01-23T15:33:44.428Z" },
- { url = "https://files.pythonhosted.org/packages/ff/07/ac9bf1ec008916d1a3373cae212884c1dcff4a4ba0d41127ce81a8deb4e9/greenlet-3.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:7932f5f57609b6a3b82cc11877709aa7a98e3308983ed93552a1c377069b20c8", size = 226100, upload-time = "2026-01-23T15:30:56.957Z" },
{ url = "https://files.pythonhosted.org/packages/ec/e8/2e1462c8fdbe0f210feb5ac7ad2d9029af8be3bf45bd9fa39765f821642f/greenlet-3.3.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5fd23b9bc6d37b563211c6abbb1b3cab27db385a4449af5c32e932f93017080c", size = 274974, upload-time = "2026-01-23T15:31:02.891Z" },
{ url = "https://files.pythonhosted.org/packages/7e/a8/530a401419a6b302af59f67aaf0b9ba1015855ea7e56c036b5928793c5bd/greenlet-3.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f51496a0bfbaa9d74d36a52d2580d1ef5ed4fdfcff0a73730abfbbbe1403dd", size = 577175, upload-time = "2026-01-23T16:00:56.213Z" },
{ url = "https://files.pythonhosted.org/packages/8e/89/7e812bb9c05e1aaef9b597ac1d0962b9021d2c6269354966451e885c4e6b/greenlet-3.3.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb0feb07fe6e6a74615ee62a880007d976cf739b6669cce95daa7373d4fc69c5", size = 590401, upload-time = "2026-01-23T16:05:26.365Z" },
@@ -811,6 +869,71 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e1/2b/98c7f93e6db9977aaee07eb1e51ca63bd5f779b900d362791d3252e60558/greenlet-3.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:301860987846c24cb8964bdec0e31a96ad4a2a801b41b4ef40963c1b44f33451", size = 233181, upload-time = "2026-01-23T15:33:00.29Z" },
]
+[[package]]
+name = "grpcio"
+version = "1.83.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d3/f6/3b781cd07a715ea5f5125ae264226e7fc4d87603d6d3955022cabfdc5da2/grpcio-1.83.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8ff0b8767ddd62704e0d9571c1890af08d84a3a689ebba1807e62519d0b3277f", size = 6338720, upload-time = "2026-07-23T15:19:13.177Z" },
+ { url = "https://files.pythonhosted.org/packages/21/cc/d14833d15d5984e366f1b027fa78bd038c9b028c66880bffb0f5a4d25ee2/grpcio-1.83.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:4772402f43517b4824980be4b3b2274a81eec0004a70009473c31b340d43e223", size = 12178773, upload-time = "2026-07-23T15:19:15.401Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/98/8acbb416544e7871132d8e42a07ed70c802d70e6a16c6009e505a34d32a4/grpcio-1.83.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f4cee5fc86e84a0cf7ad1574b454c3320e087c07f55b7df5dc0ac6a873fb90c0", size = 6921203, upload-time = "2026-07-23T15:19:17.824Z" },
+ { url = "https://files.pythonhosted.org/packages/45/9c/0fdbfaf4fc54e5c88f6bce4008a065092fe7fbc4460eb5617ae8b20fd505/grpcio-1.83.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f5e822a7e7d03282f6ad225e710493c48b9057a353358344a5f7c42b2b37618d", size = 7648508, upload-time = "2026-07-23T15:19:19.685Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/ea/107b9dbb2ed3ad14dd774fd3dde7d29ff9938a6c198654becb2c3a0e9a6a/grpcio-1.83.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f5f410d7c2903eabb34789dfd6342eef04af1ad459943936b7e09a9f5bd417b9", size = 7079466, upload-time = "2026-07-23T15:19:21.478Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/06/9fa9941089e6fae83b060b6ce61c1e81053e52decae43197245f45e07d36/grpcio-1.83.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee94a4016fdf8699fb1fd8a38652475ff677f1c72074cee44deeeb9a7e95e745", size = 7605583, upload-time = "2026-07-23T15:19:23.74Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/2f/f10fb56062dc2771c630827a82d9ad0ecd05cad572ea3b08d49f6631680a/grpcio-1.83.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c6444666317338e903093c7c756e6cc88eee59f798cb8dd41e87725bf54e1617", size = 8637810, upload-time = "2026-07-23T15:19:25.536Z" },
+ { url = "https://files.pythonhosted.org/packages/99/55/f84927258f6a1b6ea6dea661fdc6de859b35e560c96f3012d15ccd39f85e/grpcio-1.83.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa074041231f03959cb097dd5517b0677b8ea49215bae01d5710a7b69dd59969", size = 8008021, upload-time = "2026-07-23T15:19:27.863Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/6b/cdf72161397ccd29d4ca2192f641524536c9cf54ad948c9dd0e0e01138fa/grpcio-1.83.0-cp311-cp311-win32.whl", hash = "sha256:cb056f6e171c42639a50460b2929c82241fda51f71cf3dcdd68090fe45095a45", size = 4404376, upload-time = "2026-07-23T15:19:30.137Z" },
+ { url = "https://files.pythonhosted.org/packages/df/ed/e0ffeb4c848699c194dc9fb6a29ab29bcb2b6aac8c416bf18c51bfe8242c/grpcio-1.83.0-cp311-cp311-win_amd64.whl", hash = "sha256:7416952ca770477990257206276999056f8316d79196f2f25942393e58a20b49", size = 5164469, upload-time = "2026-07-23T15:19:31.941Z" },
+ { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" },
+ { url = "https://files.pythonhosted.org/packages/39/33/b5b50fc2c6fbe350e04814047bb2d409feec7b36ef8b170254c050e06bc0/grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da", size = 12160538, upload-time = "2026-07-23T15:19:35.958Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" },
+ { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" },
+ { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" },
+ { url = "https://files.pythonhosted.org/packages/17/5b/77af31228f55f55a2a5112bb0077ad0a1c4d23dbb0c2853a62475bbdcc14/grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc", size = 4394004, upload-time = "2026-07-23T15:19:50.618Z" },
+ { url = "https://files.pythonhosted.org/packages/c0/da/f706e39550e7a3732ce2b9c5926107a93d74a802775b19b642a6df27dc96/grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df", size = 5158525, upload-time = "2026-07-23T15:19:52.246Z" },
+ { url = "https://files.pythonhosted.org/packages/56/eb/135daaa713f32d33b8f99b4153b3f8dc3b2a124996ac15581bf9ebdad3c3/grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0", size = 6304480, upload-time = "2026-07-23T15:19:53.962Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/a1/121806ce69f23138dabe06aa595b0e5f1ae051a37e4c1954eed7d692c800/grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1", size = 12154419, upload-time = "2026-07-23T15:19:56.3Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/e8/d0389e09cd6b4c4d3089b92967ae4e3ffd64795bd349bf2f85cd6656d3da/grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867", size = 6873200, upload-time = "2026-07-23T15:19:58.701Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/51/f464c1d211fa50d5adbabe1b2e519948d99c13757052bfc9ea7afa28e284/grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881", size = 7618811, upload-time = "2026-07-23T15:20:00.733Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/c0/539fe0832f2dd6500a28f5263071623fb34e8d4867aec632ccf81bd21156/grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f", size = 7042310, upload-time = "2026-07-23T15:20:02.675Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/ca/ccf617d37ffa72567fa8e005ec7090c99da922799be2fb9847c8b21ca18c/grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61", size = 7575412, upload-time = "2026-07-23T15:20:04.712Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/b9/fd8d5245f823a8e0fd35d90e20ea3aa4acd47f8d5318fa8df307df52dec6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45", size = 8604248, upload-time = "2026-07-23T15:20:06.77Z" },
+ { url = "https://files.pythonhosted.org/packages/14/1e/f37632fc11db72dfa4bba86c3a43e54358e53030df111ecae5e91a733ad6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c", size = 7977458, upload-time = "2026-07-23T15:20:09.109Z" },
+ { url = "https://files.pythonhosted.org/packages/93/b6/d70b69ae5c0cfc341b9ba474980e4ed99cbf05c0e4a14e9eee8cb73db0a5/grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf", size = 4393993, upload-time = "2026-07-23T15:20:11.017Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/13/45d4cccb555cf4c476226979bf3d2fd0b0254216f7564c3a053e35117efc/grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735", size = 5159650, upload-time = "2026-07-23T15:20:12.979Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/ab/d3874931d123a95e83a3ebf8aa04537988fb62425cedb8bf3cefc5ad41b2/grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c", size = 12166617, upload-time = "2026-07-23T15:20:17.435Z" },
+ { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" },
+ { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" },
+ { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" },
+ { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" },
+ { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" },
+ { url = "https://files.pythonhosted.org/packages/83/fa/c3fda157287f64bc65acee6c5aa90c41acf9e0d3a8e69a265eecff6d00a1/grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33", size = 4471765, upload-time = "2026-07-23T15:20:33.195Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" },
+]
+
+[[package]]
+name = "grpcio-status"
+version = "1.83.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "googleapis-common-protos" },
+ { name = "grpcio" },
+ { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/52/fd/848dd7e009de85f8ca59999d1cc618ff8ebf7ea5636d083a47455d212d24/grpcio_status-1.83.0.tar.gz", hash = "sha256:837219c6de9afdccb6f6f72b34bc71e151a2011ef04040e3faaca746a57e54ae", size = 13965, upload-time = "2026-07-23T15:24:26.98Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d6/00/73204406228cf989bea6b0fd9fe4702fab49a8a152a0c6f90856dadb6ac7/grpcio_status-1.83.0-py3-none-any.whl", hash = "sha256:f6a838a7c5fb84ae98833ec0ef81ed438c26e11e54b2ddb8e92ad328c861de69", size = 14636, upload-time = "2026-07-23T15:23:49.044Z" },
+]
+
[[package]]
name = "h11"
version = "0.16.0"
@@ -839,13 +962,6 @@ version = "0.7.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/b5/46/120a669232c7bdedb9d52d4aeae7e6c7dfe151e99dc70802e2fc7a5e1993/httptools-0.7.1.tar.gz", hash = "sha256:abd72556974f8e7c74a259655924a717a2365b236c882c3f6f8a45fe94703ac9", size = 258961, upload-time = "2025-10-10T03:55:08.559Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/c7/e5/c07e0bcf4ec8db8164e9f6738c048b2e66aabf30e7506f440c4cc6953f60/httptools-0.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:11d01b0ff1fe02c4c32d60af61a4d613b74fad069e47e06e9067758c01e9ac78", size = 204531, upload-time = "2025-10-10T03:54:20.887Z" },
- { url = "https://files.pythonhosted.org/packages/7e/4f/35e3a63f863a659f92ffd92bef131f3e81cf849af26e6435b49bd9f6f751/httptools-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:84d86c1e5afdc479a6fdabf570be0d3eb791df0ae727e8dbc0259ed1249998d4", size = 109408, upload-time = "2025-10-10T03:54:22.455Z" },
- { url = "https://files.pythonhosted.org/packages/f5/71/b0a9193641d9e2471ac541d3b1b869538a5fb6419d52fd2669fa9c79e4b8/httptools-0.7.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c8c751014e13d88d2be5f5f14fc8b89612fcfa92a9cc480f2bc1598357a23a05", size = 440889, upload-time = "2025-10-10T03:54:23.753Z" },
- { url = "https://files.pythonhosted.org/packages/eb/d9/2e34811397b76718750fea44658cb0205b84566e895192115252e008b152/httptools-0.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:654968cb6b6c77e37b832a9be3d3ecabb243bbe7a0b8f65fbc5b6b04c8fcabed", size = 440460, upload-time = "2025-10-10T03:54:25.313Z" },
- { url = "https://files.pythonhosted.org/packages/01/3f/a04626ebeacc489866bb4d82362c0657b2262bef381d68310134be7f40bb/httptools-0.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b580968316348b474b020edf3988eecd5d6eec4634ee6561e72ae3a2a0e00a8a", size = 425267, upload-time = "2025-10-10T03:54:26.81Z" },
- { url = "https://files.pythonhosted.org/packages/a5/99/adcd4f66614db627b587627c8ad6f4c55f18881549bab10ecf180562e7b9/httptools-0.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d496e2f5245319da9d764296e86c5bb6fcf0cf7a8806d3d000717a889c8c0b7b", size = 424429, upload-time = "2025-10-10T03:54:28.174Z" },
- { url = "https://files.pythonhosted.org/packages/d5/72/ec8fc904a8fd30ba022dfa85f3bbc64c3c7cd75b669e24242c0658e22f3c/httptools-0.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cbf8317bfccf0fed3b5680c559d3459cccf1abe9039bfa159e62e391c7270568", size = 86173, upload-time = "2025-10-10T03:54:29.5Z" },
{ url = "https://files.pythonhosted.org/packages/9c/08/17e07e8d89ab8f343c134616d72eebfe03798835058e2ab579dcc8353c06/httptools-0.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:474d3b7ab469fefcca3697a10d11a32ee2b9573250206ba1e50d5980910da657", size = 206521, upload-time = "2025-10-10T03:54:31.002Z" },
{ url = "https://files.pythonhosted.org/packages/aa/06/c9c1b41ff52f16aee526fd10fbda99fa4787938aa776858ddc4a1ea825ec/httptools-0.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3c3b7366bb6c7b96bd72d0dbe7f7d5eead261361f013be5f6d9590465ea1c70", size = 110375, upload-time = "2025-10-10T03:54:31.941Z" },
{ url = "https://files.pythonhosted.org/packages/cc/cc/10935db22fda0ee34c76f047590ca0a8bd9de531406a3ccb10a90e12ea21/httptools-0.7.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:379b479408b8747f47f3b253326183d7c009a3936518cdb70db58cffd369d9df", size = 456621, upload-time = "2025-10-10T03:54:33.176Z" },
@@ -978,17 +1094,6 @@ version = "3.0.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631, upload-time = "2025-09-27T18:36:05.558Z" },
- { url = "https://files.pythonhosted.org/packages/98/1b/fbd8eed11021cabd9226c37342fa6ca4e8a98d8188a8d9b66740494960e4/markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", size = 12057, upload-time = "2025-09-27T18:36:07.165Z" },
- { url = "https://files.pythonhosted.org/packages/40/01/e560d658dc0bb8ab762670ece35281dec7b6c1b33f5fbc09ebb57a185519/markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", size = 22050, upload-time = "2025-09-27T18:36:08.005Z" },
- { url = "https://files.pythonhosted.org/packages/af/cd/ce6e848bbf2c32314c9b237839119c5a564a59725b53157c856e90937b7a/markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", size = 20681, upload-time = "2025-09-27T18:36:08.881Z" },
- { url = "https://files.pythonhosted.org/packages/c9/2a/b5c12c809f1c3045c4d580b035a743d12fcde53cf685dbc44660826308da/markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", size = 20705, upload-time = "2025-09-27T18:36:10.131Z" },
- { url = "https://files.pythonhosted.org/packages/cf/e3/9427a68c82728d0a88c50f890d0fc072a1484de2f3ac1ad0bfc1a7214fd5/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", size = 21524, upload-time = "2025-09-27T18:36:11.324Z" },
- { url = "https://files.pythonhosted.org/packages/bc/36/23578f29e9e582a4d0278e009b38081dbe363c5e7165113fad546918a232/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", size = 20282, upload-time = "2025-09-27T18:36:12.573Z" },
- { url = "https://files.pythonhosted.org/packages/56/21/dca11354e756ebd03e036bd8ad58d6d7168c80ce1fe5e75218e4945cbab7/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", size = 20745, upload-time = "2025-09-27T18:36:13.504Z" },
- { url = "https://files.pythonhosted.org/packages/87/99/faba9369a7ad6e4d10b6a5fbf71fa2a188fe4a593b15f0963b73859a1bbd/markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", size = 14571, upload-time = "2025-09-27T18:36:14.779Z" },
- { url = "https://files.pythonhosted.org/packages/d6/25/55dc3ab959917602c96985cb1253efaa4ff42f71194bddeb61eb7278b8be/markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", size = 15056, upload-time = "2025-09-27T18:36:16.125Z" },
- { url = "https://files.pythonhosted.org/packages/d0/9e/0a02226640c255d1da0b8d12e24ac2aa6734da68bff14c05dd53b94a0fc3/markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", size = 13932, upload-time = "2025-09-27T18:36:17.311Z" },
{ url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" },
{ url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" },
{ url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" },
@@ -1081,14 +1186,6 @@ version = "1.1.2"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/4d/f2/bfb55a6236ed8725a96b0aa3acbd0ec17588e6a2c3b62a93eb513ed8783f/msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e", size = 173581, upload-time = "2025-10-08T09:15:56.596Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/f5/a2/3b68a9e769db68668b25c6108444a35f9bd163bb848c0650d516761a59c0/msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2", size = 81318, upload-time = "2025-10-08T09:14:38.722Z" },
- { url = "https://files.pythonhosted.org/packages/5b/e1/2b720cc341325c00be44e1ed59e7cfeae2678329fbf5aa68f5bda57fe728/msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87", size = 83786, upload-time = "2025-10-08T09:14:40.082Z" },
- { url = "https://files.pythonhosted.org/packages/71/e5/c2241de64bfceac456b140737812a2ab310b10538a7b34a1d393b748e095/msgpack-1.1.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b696e83c9f1532b4af884045ba7f3aa741a63b2bc22617293a2c6a7c645f251", size = 398240, upload-time = "2025-10-08T09:14:41.151Z" },
- { url = "https://files.pythonhosted.org/packages/b7/09/2a06956383c0fdebaef5aa9246e2356776f12ea6f2a44bd1368abf0e46c4/msgpack-1.1.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:365c0bbe981a27d8932da71af63ef86acc59ed5c01ad929e09a0b88c6294e28a", size = 406070, upload-time = "2025-10-08T09:14:42.821Z" },
- { url = "https://files.pythonhosted.org/packages/0e/74/2957703f0e1ef20637d6aead4fbb314330c26f39aa046b348c7edcf6ca6b/msgpack-1.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41d1a5d875680166d3ac5c38573896453bbbea7092936d2e107214daf43b1d4f", size = 393403, upload-time = "2025-10-08T09:14:44.38Z" },
- { url = "https://files.pythonhosted.org/packages/a5/09/3bfc12aa90f77b37322fc33e7a8a7c29ba7c8edeadfa27664451801b9860/msgpack-1.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:354e81bcdebaab427c3df4281187edc765d5d76bfb3a7c125af9da7a27e8458f", size = 398947, upload-time = "2025-10-08T09:14:45.56Z" },
- { url = "https://files.pythonhosted.org/packages/4b/4f/05fcebd3b4977cb3d840f7ef6b77c51f8582086de5e642f3fefee35c86fc/msgpack-1.1.2-cp310-cp310-win32.whl", hash = "sha256:e64c8d2f5e5d5fda7b842f55dec6133260ea8f53c4257d64494c534f306bf7a9", size = 64769, upload-time = "2025-10-08T09:14:47.334Z" },
- { url = "https://files.pythonhosted.org/packages/d0/3e/b4547e3a34210956382eed1c85935fff7e0f9b98be3106b3745d7dec9c5e/msgpack-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa", size = 71293, upload-time = "2025-10-08T09:14:48.665Z" },
{ url = "https://files.pythonhosted.org/packages/2c/97/560d11202bcd537abca693fd85d81cebe2107ba17301de42b01ac1677b69/msgpack-1.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2e86a607e558d22985d856948c12a3fa7b42efad264dca8a3ebbcfa2735d786c", size = 82271, upload-time = "2025-10-08T09:14:49.967Z" },
{ url = "https://files.pythonhosted.org/packages/83/04/28a41024ccbd67467380b6fb440ae916c1e4f25e2cd4c63abe6835ac566e/msgpack-1.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:283ae72fc89da59aa004ba147e8fc2f766647b1251500182fac0350d8af299c0", size = 84914, upload-time = "2025-10-08T09:14:50.958Z" },
{ url = "https://files.pythonhosted.org/packages/71/46/b817349db6886d79e57a966346cf0902a426375aadc1e8e7a86a75e22f19/msgpack-1.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61c8aa3bd513d87c72ed0b37b53dd5c5a0f58f2ff9f26e1555d3bd7948fb7296", size = 416962, upload-time = "2025-10-08T09:14:51.997Z" },
@@ -1173,7 +1270,6 @@ dependencies = [
{ name = "pip" },
{ name = "pyproject-hooks" },
{ name = "setuptools" },
- { name = "tomli", marker = "python_full_version < '3.11'" },
{ name = "wheel" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c4/79/d149fb40bc425ad9defcb8ff73c65088bbc36a84b1825e035397d1c40624/pip_tools-7.5.2.tar.gz", hash = "sha256:2d64d72da6a044da1110257d333960563d7a4743637e8617dd2610ae7b82d60f", size = 164815, upload-time = "2025-11-12T22:46:12.627Z" }
@@ -1215,6 +1311,33 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/5d/19/fd3ef348460c80af7bb4669ea7926651d1f95c23ff2df18b9d24bab4f3fa/pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77", size = 226437, upload-time = "2025-12-16T21:14:32.409Z" },
]
+[[package]]
+name = "proto-plus"
+version = "1.28.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/40/a6/4fbadcc2044034449b3f8f0ce82dcf3005d53f37c136642103fd4836a31c/proto_plus-1.28.4.tar.gz", hash = "sha256:5ff7ecad828e032a491fcb86947801768e32237f99dd049b649965b892ae9a63", size = 58679, upload-time = "2026-08-25T19:19:15.102Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/41/5d/0f04b85dafdc3250ced7f2592efc17dce7f40712e941e9632202481e600d/proto_plus-1.28.4-py3-none-any.whl", hash = "sha256:4b01341272f8a348db3f003b6143109f83ab43091019d5181b3fcdf500ab32aa", size = 50797, upload-time = "2026-08-25T19:18:12.338Z" },
+]
+
+[[package]]
+name = "protobuf"
+version = "7.36.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a7/e7/0553e21d25ca4d9f573135775348a372c3ec34a93a71d5f297c3bac38341/protobuf-7.36.0.tar.gz", hash = "sha256:e8e09cb0d794c6687926fa558a8a6e72aa10edb997d5ca61da0765f12a3e00ea", size = 510034, upload-time = "2026-08-20T16:34:01.071Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/8f/ae/58e3ca96cb2e118cc546b677359b3c6659f79a140935c08dec94c7998585/protobuf-7.36.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:9103532dffd80c6fab7e50c65a31007680a06eb57537d437bb1b35812c138a37", size = 453256, upload-time = "2026-08-20T16:33:53.945Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/15/5162230af4912697f0fe406f6800f80760945babcff0e2c2fe6c84ef2d5d/protobuf-7.36.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:bf94a5917c71058262de683669bc0a797a7669d3de71f0b36d058e3194f47b44", size = 341436, upload-time = "2026-08-20T16:33:55.134Z" },
+ { url = "https://files.pythonhosted.org/packages/d7/09/1670b2bfc9a45e807e520c3e9be36524db9ccc7dc05ea17af7681cabdc61/protobuf-7.36.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:3297e60abdff301e5f74393d87f6cc59dacab5f024a89548a6e8de1d26576b16", size = 354440, upload-time = "2026-08-20T16:33:56.077Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/f8/bd5804695ba400e423c33fd4d9f58c28d86633d5ba1945c36ff3967d98cb/protobuf-7.36.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:70f5ec8eb0da81a44360c0dc0beac99a0d78071d21956a7076bae8bd2051841b", size = 340439, upload-time = "2026-08-20T16:33:56.992Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/9f/acd02338235a3e7d03168c4303478347b7624fc8189ff4e7f0d2654bbe86/protobuf-7.36.0-cp310-abi3-win32.whl", hash = "sha256:7326fd717bdc419162a735938d89d4032332bcc3408804012b24ff3a37086071", size = 440216, upload-time = "2026-08-20T16:33:57.99Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/4e/12cb93270967a2affff5b3f720694700d4d87712a67afd05c8cb3f6fa52c/protobuf-7.36.0-cp310-abi3-win_amd64.whl", hash = "sha256:1781cc1de61249b750848029bca452c0a8b7e990080316b9bbc2518b2117b488", size = 453731, upload-time = "2026-08-20T16:33:58.951Z" },
+ { url = "https://files.pythonhosted.org/packages/01/c3/629999e78d46c1115c11886d51c6bd68c17ce4a944f1ea3e153a91316a33/protobuf-7.36.0-py3-none-any.whl", hash = "sha256:53374d53fc29a67f7dbbf0ade47d7526a0f0137bf0f9c90e48d8a60790ef748c", size = 177024, upload-time = "2026-08-20T16:34:00.053Z" },
+]
+
[[package]]
name = "psycopg"
version = "3.3.2"
@@ -1245,6 +1368,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e7/c3/26b8a0908a9db249de3b4169692e1c7c19048a9bc41a4d3209cee7dbb758/psycopg_pool-3.3.0-py3-none-any.whl", hash = "sha256:2e44329155c410b5e8666372db44276a8b1ebd8c90f1c3026ebba40d4bc81063", size = 39995, upload-time = "2025-12-01T11:34:29.761Z" },
]
+[[package]]
+name = "pyasn1"
+version = "0.6.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" },
+]
+
+[[package]]
+name = "pyasn1-modules"
+version = "0.4.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pyasn1" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" },
+]
+
[[package]]
name = "pycparser"
version = "3.0"
@@ -1278,19 +1422,6 @@ dependencies = [
]
sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" },
- { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" },
- { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" },
- { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" },
- { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" },
- { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" },
- { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" },
- { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" },
- { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" },
- { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" },
- { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" },
- { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" },
- { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" },
{ url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" },
{ url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" },
{ url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" },
@@ -1369,14 +1500,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" },
{ url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" },
{ url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" },
- { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" },
- { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" },
- { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" },
- { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" },
- { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" },
- { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" },
- { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" },
- { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" },
{ url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" },
{ url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" },
{ url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" },
@@ -1434,7 +1557,6 @@ dependencies = [
{ name = "isort" },
{ name = "mccabe" },
{ name = "platformdirs" },
- { name = "tomli", marker = "python_full_version < '3.11'" },
{ name = "tomlkit" },
]
sdist = { url = "https://files.pythonhosted.org/packages/5a/d2/b081da1a8930d00e3fc06352a1d449aaf815d4982319fab5d8cdb2e9ab35/pylint-4.0.4.tar.gz", hash = "sha256:d9b71674e19b1c36d79265b5887bf8e55278cbe236c9e95d22dc82cf044fdbd2", size = 1571735, upload-time = "2025-11-30T13:29:04.315Z" }
@@ -1470,12 +1592,10 @@ version = "9.0.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "colorama", marker = "sys_platform == 'win32'" },
- { name = "exceptiongroup", marker = "python_full_version < '3.11'" },
{ name = "iniconfig" },
{ name = "packaging" },
{ name = "pluggy" },
{ name = "pygments" },
- { name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" }
wheels = [
@@ -1487,7 +1607,6 @@ name = "pytest-asyncio"
version = "1.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" },
{ name = "pytest" },
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
@@ -1566,9 +1685,6 @@ name = "pywin32"
version = "311"
source = { registry = "https://pypi.org/simple" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/7b/40/44efbb0dfbd33aca6a6483191dae0716070ed99e2ecb0c53683f400a0b4f/pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3", size = 8760432, upload-time = "2025-07-14T20:13:05.9Z" },
- { url = "https://files.pythonhosted.org/packages/5e/bf/360243b1e953bd254a82f12653974be395ba880e7ec23e3731d9f73921cc/pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b", size = 9590103, upload-time = "2025-07-14T20:13:07.698Z" },
- { url = "https://files.pythonhosted.org/packages/57/38/d290720e6f138086fb3d5ffe0b6caa019a791dd57866940c82e4eeaf2012/pywin32-311-cp310-cp310-win_arm64.whl", hash = "sha256:0502d1facf1fed4839a9a51ccbcc63d952cf318f78ffc00a7e78528ac27d7a2b", size = 8778557, upload-time = "2025-07-14T20:13:11.11Z" },
{ url = "https://files.pythonhosted.org/packages/7c/af/449a6a91e5d6db51420875c54f6aff7c97a86a3b13a0b4f1a5c13b988de3/pywin32-311-cp311-cp311-win32.whl", hash = "sha256:184eb5e436dea364dcd3d2316d577d625c0351bf237c4e9a5fabbcfa5a58b151", size = 8697031, upload-time = "2025-07-14T20:13:13.266Z" },
{ url = "https://files.pythonhosted.org/packages/51/8f/9bb81dd5bb77d22243d33c8397f09377056d5c687aa6d4042bea7fbf8364/pywin32-311-cp311-cp311-win_amd64.whl", hash = "sha256:3ce80b34b22b17ccbd937a6e78e7225d80c52f5ab9940fe0506a1a16f3dab503", size = 9508308, upload-time = "2025-07-14T20:13:15.147Z" },
{ url = "https://files.pythonhosted.org/packages/44/7b/9c2ab54f74a138c491aba1b1cd0795ba61f144c711daea84a88b63dc0f6c/pywin32-311-cp311-cp311-win_arm64.whl", hash = "sha256:a733f1388e1a842abb67ffa8e7aad0e70ac519e09b0f6a784e65a136ec7cefd2", size = 8703930, upload-time = "2025-07-14T20:13:16.945Z" },
@@ -1589,15 +1705,6 @@ version = "6.0.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" },
- { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" },
- { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" },
- { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" },
- { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" },
- { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" },
- { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" },
- { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" },
- { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" },
{ url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" },
{ url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" },
{ url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" },
@@ -1742,11 +1849,6 @@ dependencies = [
]
sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/4aff125021a9c5e0183f2f55dd7d04b7256a0e1e10db50d537a7415d9c55/snowflake_connector_python-4.0.0.tar.gz", hash = "sha256:4b10a865c4a5e1fa60c365c7fe41e0433605e6e5edc824e8730a9038f330b3a6", size = 813937, upload-time = "2025-10-09T10:11:34.631Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/e4/75/f845ca5079a6b911023fa945dbf1bac0ed1c2f5967108b14440c740cb410/snowflake_connector_python-4.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2c3e0f6d103fe67c975550ed424f579d3e7ae503d56467e5549f3a1a1e0e8f24", size = 1030251, upload-time = "2025-10-09T10:11:36.37Z" },
- { url = "https://files.pythonhosted.org/packages/fd/80/3a7e36a9e53beeb27c0599d2703f33bb812be931b469b154b08df0eeeaf5/snowflake_connector_python-4.0.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:e8d5b66f283967c700fff2303ac5e52d1a3cf41990a634f121ac8b1f1cd9af10", size = 1043041, upload-time = "2025-10-09T10:11:37.719Z" },
- { url = "https://files.pythonhosted.org/packages/6e/3b/bda95c4de593743c021a9968d70087674189c60a8317185de1b0f32d17c8/snowflake_connector_python-4.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ad5d0f1ebcb2c6b7a7859ee3d4e02203087e40faae539a336bbcb45a3660777", size = 2666209, upload-time = "2025-10-09T10:11:17.54Z" },
- { url = "https://files.pythonhosted.org/packages/60/e6/30c4015e2712bf8bf83b54ddadeee0494b68ae6d0f6d49d9373f463305d4/snowflake_connector_python-4.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4106a66e770e564b3037457b7b01b15ca28aee61afb88560b664aa8af439b533", size = 2693962, upload-time = "2025-10-09T10:11:20.735Z" },
- { url = "https://files.pythonhosted.org/packages/1b/e0/5f494b3353216e629f8b9d7269024518e0db9f2992df471de6ef43b60f7b/snowflake_connector_python-4.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:7789df78f7c7abfb351f2709258d05a94652cfe3c2c617fb15f15a11fc1b7b25", size = 1177353, upload-time = "2025-10-09T10:11:51.07Z" },
{ url = "https://files.pythonhosted.org/packages/0c/86/0dceb37f50cd28ee61af1f0396eccd17a563d56d66067e0842ff8dfafe6d/snowflake_connector_python-4.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ca2503f705627f7e045da6254d97c37210a3b0a18b43d0f1b29616d0c7aaa01", size = 1030446, upload-time = "2025-10-09T10:11:39.321Z" },
{ url = "https://files.pythonhosted.org/packages/7b/80/1ac8ae9494b2ee5bcc0e4cefe5aada86d0e61d21208a0107e99c5bec92ec/snowflake_connector_python-4.0.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:fd0d2d2c5cfd15f041e8522f5f8bdad0be4de7d805dd1646377fccd6bd404fa8", size = 1043031, upload-time = "2025-10-09T10:11:40.638Z" },
{ url = "https://files.pythonhosted.org/packages/fe/db/fbd1dbe2d6ca2b8f99337e39d596a800e6c54451a93a9167927a15f60534/snowflake_connector_python-4.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebbdeec0d65c2e3f648c8b05839001c062984959417902717f7fc6eed983211d", size = 2677427, upload-time = "2025-10-09T10:11:22.537Z" },
@@ -1783,13 +1885,6 @@ dependencies = [
]
sdist = { url = "https://files.pythonhosted.org/packages/06/aa/9ce0f3e7a9829ead5c8ce549392f33a12c4555a6c0609bb27d882e9c7ddf/sqlalchemy-2.0.46.tar.gz", hash = "sha256:cf36851ee7219c170bb0793dbc3da3e80c582e04a5437bc601bfe8c85c9216d7", size = 9865393, upload-time = "2026-01-21T18:03:45.119Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/40/26/66ba59328dc25e523bfcb0f8db48bdebe2035e0159d600e1f01c0fc93967/sqlalchemy-2.0.46-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:895296687ad06dc9b11a024cf68e8d9d3943aa0b4964278d2553b86f1b267735", size = 2155051, upload-time = "2026-01-21T18:27:28.965Z" },
- { url = "https://files.pythonhosted.org/packages/21/cd/9336732941df972fbbfa394db9caa8bb0cf9fe03656ec728d12e9cbd6edc/sqlalchemy-2.0.46-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab65cb2885a9f80f979b85aa4e9c9165a31381ca322cbde7c638fe6eefd1ec39", size = 3234666, upload-time = "2026-01-21T18:32:28.72Z" },
- { url = "https://files.pythonhosted.org/packages/38/62/865ae8b739930ec433cd4123760bee7f8dafdc10abefd725a025604fb0de/sqlalchemy-2.0.46-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:52fe29b3817bd191cc20bad564237c808967972c97fa683c04b28ec8979ae36f", size = 3232917, upload-time = "2026-01-21T18:44:54.064Z" },
- { url = "https://files.pythonhosted.org/packages/24/38/805904b911857f2b5e00fdea44e9570df62110f834378706939825579296/sqlalchemy-2.0.46-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:09168817d6c19954d3b7655da6ba87fcb3a62bb575fb396a81a8b6a9fadfe8b5", size = 3185790, upload-time = "2026-01-21T18:32:30.581Z" },
- { url = "https://files.pythonhosted.org/packages/69/4f/3260bb53aabd2d274856337456ea52f6a7eccf6cce208e558f870cec766b/sqlalchemy-2.0.46-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:be6c0466b4c25b44c5d82b0426b5501de3c424d7a3220e86cd32f319ba56798e", size = 3207206, upload-time = "2026-01-21T18:44:55.93Z" },
- { url = "https://files.pythonhosted.org/packages/ce/b3/67c432d7f9d88bb1a61909b67e29f6354d59186c168fb5d381cf438d3b73/sqlalchemy-2.0.46-cp310-cp310-win32.whl", hash = "sha256:1bc3f601f0a818d27bfe139f6766487d9c88502062a2cd3a7ee6c342e81d5047", size = 2115296, upload-time = "2026-01-21T18:33:12.498Z" },
- { url = "https://files.pythonhosted.org/packages/4a/8c/25fb284f570f9d48e6c240f0269a50cec9cf009a7e08be4c0aaaf0654972/sqlalchemy-2.0.46-cp310-cp310-win_amd64.whl", hash = "sha256:e0c05aff5c6b1bb5fb46a87e0f9d2f733f83ef6cbbbcd5c642b6c01678268061", size = 2138540, upload-time = "2026-01-21T18:33:14.22Z" },
{ url = "https://files.pythonhosted.org/packages/69/ac/b42ad16800d0885105b59380ad69aad0cce5a65276e269ce2729a2343b6a/sqlalchemy-2.0.46-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:261c4b1f101b4a411154f1da2b76497d73abbfc42740029205d4d01fa1052684", size = 2154851, upload-time = "2026-01-21T18:27:30.54Z" },
{ url = "https://files.pythonhosted.org/packages/a0/60/d8710068cb79f64d002ebed62a7263c00c8fd95f4ebd4b5be8f7ca93f2bc/sqlalchemy-2.0.46-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:181903fe8c1b9082995325f1b2e84ac078b1189e2819380c2303a5f90e114a62", size = 3311241, upload-time = "2026-01-21T18:32:33.45Z" },
{ url = "https://files.pythonhosted.org/packages/2b/0f/20c71487c7219ab3aa7421c7c62d93824c97c1460f2e8bb72404b0192d13/sqlalchemy-2.0.46-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:590be24e20e2424a4c3c1b0835e9405fa3d0af5823a1a9fc02e5dff56471515f", size = 3310741, upload-time = "2026-01-21T18:44:57.887Z" },
@@ -2021,7 +2116,6 @@ source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
{ name = "h11" },
- { name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c3/d1/8f3c683c9561a4e6689dd3b1d345c815f10f86acd044ee1fb9a4dcd0b8c5/uvicorn-0.40.0.tar.gz", hash = "sha256:839676675e87e73694518b5574fd0f24c9d97b46bea16df7b8c05ea1a51071ea", size = 81761, upload-time = "2025-12-21T14:16:22.45Z" }
wheels = [
@@ -2045,12 +2139,6 @@ version = "0.22.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/eb/14/ecceb239b65adaaf7fde510aa8bd534075695d1e5f8dadfa32b5723d9cfb/uvloop-0.22.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c", size = 1343335, upload-time = "2025-10-16T22:16:11.43Z" },
- { url = "https://files.pythonhosted.org/packages/ba/ae/6f6f9af7f590b319c94532b9567409ba11f4fa71af1148cab1bf48a07048/uvloop-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792", size = 742903, upload-time = "2025-10-16T22:16:12.979Z" },
- { url = "https://files.pythonhosted.org/packages/09/bd/3667151ad0702282a1f4d5d29288fce8a13c8b6858bf0978c219cd52b231/uvloop-0.22.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac33ed96229b7790eb729702751c0e93ac5bc3bcf52ae9eccbff30da09194b86", size = 3648499, upload-time = "2025-10-16T22:16:14.451Z" },
- { url = "https://files.pythonhosted.org/packages/b3/f6/21657bb3beb5f8c57ce8be3b83f653dd7933c2fd00545ed1b092d464799a/uvloop-0.22.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:481c990a7abe2c6f4fc3d98781cc9426ebd7f03a9aaa7eb03d3bfc68ac2a46bd", size = 3700133, upload-time = "2025-10-16T22:16:16.272Z" },
- { url = "https://files.pythonhosted.org/packages/09/e0/604f61d004ded805f24974c87ddd8374ef675644f476f01f1df90e4cdf72/uvloop-0.22.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a592b043a47ad17911add5fbd087c76716d7c9ccc1d64ec9249ceafd735f03c2", size = 3512681, upload-time = "2025-10-16T22:16:18.07Z" },
- { url = "https://files.pythonhosted.org/packages/bb/ce/8491fd370b0230deb5eac69c7aae35b3be527e25a911c0acdffb922dc1cd/uvloop-0.22.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1489cf791aa7b6e8c8be1c5a080bae3a672791fcb4e9e12249b05862a2ca9cec", size = 3615261, upload-time = "2025-10-16T22:16:19.596Z" },
{ url = "https://files.pythonhosted.org/packages/c7/d5/69900f7883235562f1f50d8184bb7dd84a2fb61e9ec63f3782546fdbd057/uvloop-0.22.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c60ebcd36f7b240b30788554b6f0782454826a0ed765d8430652621b5de674b9", size = 1352420, upload-time = "2025-10-16T22:16:21.187Z" },
{ url = "https://files.pythonhosted.org/packages/a8/73/c4e271b3bce59724e291465cc936c37758886a4868787da0278b3b56b905/uvloop-0.22.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b7f102bf3cb1995cfeaee9321105e8f5da76fdb104cdad8986f85461a1b7b77", size = 748677, upload-time = "2025-10-16T22:16:22.558Z" },
{ url = "https://files.pythonhosted.org/packages/86/94/9fb7fad2f824d25f8ecac0d70b94d0d48107ad5ece03769a9c543444f78a/uvloop-0.22.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21", size = 3753819, upload-time = "2025-10-16T22:16:23.903Z" },
@@ -2091,7 +2179,6 @@ dependencies = [
{ name = "distlib" },
{ name = "filelock" },
{ name = "platformdirs" },
- { name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/aa/a3/4d310fa5f00863544e1d0f4de93bddec248499ccf97d4791bc3122c9d4f3/virtualenv-20.36.1.tar.gz", hash = "sha256:8befb5c81842c641f8ee658481e42641c68b5eab3521d8e092d18320902466ba", size = 6032239, upload-time = "2026-01-09T18:21:01.296Z" }
wheels = [
@@ -2107,18 +2194,6 @@ dependencies = [
]
sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/a7/1a/206e8cf2dd86fddf939165a57b4df61607a1e0add2785f170a3f616b7d9f/watchfiles-1.1.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:eef58232d32daf2ac67f42dea51a2c80f0d03379075d44a587051e63cc2e368c", size = 407318, upload-time = "2025-10-14T15:04:18.753Z" },
- { url = "https://files.pythonhosted.org/packages/b3/0f/abaf5262b9c496b5dad4ed3c0e799cbecb1f8ea512ecb6ddd46646a9fca3/watchfiles-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:03fa0f5237118a0c5e496185cafa92878568b652a2e9a9382a5151b1a0380a43", size = 394478, upload-time = "2025-10-14T15:04:20.297Z" },
- { url = "https://files.pythonhosted.org/packages/b1/04/9cc0ba88697b34b755371f5ace8d3a4d9a15719c07bdc7bd13d7d8c6a341/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ca65483439f9c791897f7db49202301deb6e15fe9f8fe2fed555bf986d10c31", size = 449894, upload-time = "2025-10-14T15:04:21.527Z" },
- { url = "https://files.pythonhosted.org/packages/d2/9c/eda4615863cd8621e89aed4df680d8c3ec3da6a4cf1da113c17decd87c7f/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f0ab1c1af0cb38e3f598244c17919fb1a84d1629cc08355b0074b6d7f53138ac", size = 459065, upload-time = "2025-10-14T15:04:22.795Z" },
- { url = "https://files.pythonhosted.org/packages/84/13/f28b3f340157d03cbc8197629bc109d1098764abe1e60874622a0be5c112/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bc570d6c01c206c46deb6e935a260be44f186a2f05179f52f7fcd2be086a94d", size = 488377, upload-time = "2025-10-14T15:04:24.138Z" },
- { url = "https://files.pythonhosted.org/packages/86/93/cfa597fa9389e122488f7ffdbd6db505b3b915ca7435ecd7542e855898c2/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e84087b432b6ac94778de547e08611266f1f8ffad28c0ee4c82e028b0fc5966d", size = 595837, upload-time = "2025-10-14T15:04:25.057Z" },
- { url = "https://files.pythonhosted.org/packages/57/1e/68c1ed5652b48d89fc24d6af905d88ee4f82fa8bc491e2666004e307ded1/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:620bae625f4cb18427b1bb1a2d9426dc0dd5a5ba74c7c2cdb9de405f7b129863", size = 473456, upload-time = "2025-10-14T15:04:26.497Z" },
- { url = "https://files.pythonhosted.org/packages/d5/dc/1a680b7458ffa3b14bb64878112aefc8f2e4f73c5af763cbf0bd43100658/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:544364b2b51a9b0c7000a4b4b02f90e9423d97fbbf7e06689236443ebcad81ab", size = 455614, upload-time = "2025-10-14T15:04:27.539Z" },
- { url = "https://files.pythonhosted.org/packages/61/a5/3d782a666512e01eaa6541a72ebac1d3aae191ff4a31274a66b8dd85760c/watchfiles-1.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bbe1ef33d45bc71cf21364df962af171f96ecaeca06bd9e3d0b583efb12aec82", size = 630690, upload-time = "2025-10-14T15:04:28.495Z" },
- { url = "https://files.pythonhosted.org/packages/9b/73/bb5f38590e34687b2a9c47a244aa4dd50c56a825969c92c9c5fc7387cea1/watchfiles-1.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1a0bb430adb19ef49389e1ad368450193a90038b5b752f4ac089ec6942c4dff4", size = 622459, upload-time = "2025-10-14T15:04:29.491Z" },
- { url = "https://files.pythonhosted.org/packages/f1/ac/c9bb0ec696e07a20bd58af5399aeadaef195fb2c73d26baf55180fe4a942/watchfiles-1.1.1-cp310-cp310-win32.whl", hash = "sha256:3f6d37644155fb5beca5378feb8c1708d5783145f2a0f1c4d5a061a210254844", size = 272663, upload-time = "2025-10-14T15:04:30.435Z" },
- { url = "https://files.pythonhosted.org/packages/11/a0/a60c5a7c2ec59fa062d9a9c61d02e3b6abd94d32aac2d8344c4bdd033326/watchfiles-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:a36d8efe0f290835fd0f33da35042a1bb5dc0e83cbc092dcf69bce442579e88e", size = 287453, upload-time = "2025-10-14T15:04:31.53Z" },
{ url = "https://files.pythonhosted.org/packages/1f/f8/2c5f479fb531ce2f0564eda479faecf253d886b1ab3630a39b7bf7362d46/watchfiles-1.1.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f57b396167a2565a4e8b5e56a5a1c537571733992b226f4f1197d79e94cf0ae5", size = 406529, upload-time = "2025-10-14T15:04:32.899Z" },
{ url = "https://files.pythonhosted.org/packages/fe/cd/f515660b1f32f65df671ddf6f85bfaca621aee177712874dc30a97397977/watchfiles-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:421e29339983e1bebc281fab40d812742268ad057db4aee8c4d2bce0af43b741", size = 394384, upload-time = "2025-10-14T15:04:33.761Z" },
{ url = "https://files.pythonhosted.org/packages/7b/c3/28b7dc99733eab43fca2d10f55c86e03bd6ab11ca31b802abac26b23d161/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e43d39a741e972bab5d8100b5cdacf69db64e34eb19b6e9af162bccf63c5cc6", size = 448789, upload-time = "2025-10-14T15:04:34.679Z" },
@@ -2191,10 +2266,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4f/55/2af26693fd15165c4ff7857e38330e1b61ab8c37d15dc79118cdba115b7a/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c91ed27800188c2ae96d16e3149f199d62f86c7af5f5f4d2c61a3ed8cd3666c", size = 455072, upload-time = "2025-10-14T15:05:48.928Z" },
{ url = "https://files.pythonhosted.org/packages/66/1d/d0d200b10c9311ec25d2273f8aad8c3ef7cc7ea11808022501811208a750/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:311ff15a0bae3714ffb603e6ba6dbfba4065ab60865d15a6ec544133bdb21099", size = 629104, upload-time = "2025-10-14T15:05:49.908Z" },
{ url = "https://files.pythonhosted.org/packages/e3/bd/fa9bb053192491b3867ba07d2343d9f2252e00811567d30ae8d0f78136fe/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a916a2932da8f8ab582f242c065f5c81bed3462849ca79ee357dd9551b0e9b01", size = 622112, upload-time = "2025-10-14T15:05:50.941Z" },
- { url = "https://files.pythonhosted.org/packages/ba/4c/a888c91e2e326872fa4705095d64acd8aa2fb9c1f7b9bd0588f33850516c/watchfiles-1.1.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:17ef139237dfced9da49fb7f2232c86ca9421f666d78c264c7ffca6601d154c3", size = 409611, upload-time = "2025-10-14T15:06:05.809Z" },
- { url = "https://files.pythonhosted.org/packages/1e/c7/5420d1943c8e3ce1a21c0a9330bcf7edafb6aa65d26b21dbb3267c9e8112/watchfiles-1.1.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:672b8adf25b1a0d35c96b5888b7b18699d27d4194bac8beeae75be4b7a3fc9b2", size = 396889, upload-time = "2025-10-14T15:06:07.035Z" },
- { url = "https://files.pythonhosted.org/packages/0c/e5/0072cef3804ce8d3aaddbfe7788aadff6b3d3f98a286fdbee9fd74ca59a7/watchfiles-1.1.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77a13aea58bc2b90173bc69f2a90de8e282648939a00a602e1dc4ee23e26b66d", size = 451616, upload-time = "2025-10-14T15:06:08.072Z" },
- { url = "https://files.pythonhosted.org/packages/83/4e/b87b71cbdfad81ad7e83358b3e447fedd281b880a03d64a760fe0a11fc2e/watchfiles-1.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b495de0bb386df6a12b18335a0285dda90260f51bdb505503c02bcd1ce27a8b", size = 458413, upload-time = "2025-10-14T15:06:09.209Z" },
{ url = "https://files.pythonhosted.org/packages/d3/8e/e500f8b0b77be4ff753ac94dc06b33d8f0d839377fee1b78e8c8d8f031bf/watchfiles-1.1.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:db476ab59b6765134de1d4fe96a1a9c96ddf091683599be0f26147ea1b2e4b88", size = 408250, upload-time = "2025-10-14T15:06:10.264Z" },
{ url = "https://files.pythonhosted.org/packages/bd/95/615e72cd27b85b61eec764a5ca51bd94d40b5adea5ff47567d9ebc4d275a/watchfiles-1.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:89eef07eee5e9d1fda06e38822ad167a044153457e6fd997f8a858ab7564a336", size = 396117, upload-time = "2025-10-14T15:06:11.28Z" },
{ url = "https://files.pythonhosted.org/packages/c9/81/e7fe958ce8a7fb5c73cc9fb07f5aeaf755e6aa72498c57d760af760c91f8/watchfiles-1.1.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce19e06cbda693e9e7686358af9cd6f5d61312ab8b00488bc36f5aabbaf77e24", size = 450493, upload-time = "2025-10-14T15:06:12.321Z" },
@@ -2207,15 +2278,6 @@ version = "16.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/20/74/221f58decd852f4b59cc3354cccaf87e8ef695fede361d03dc9a7396573b/websockets-16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a", size = 177343, upload-time = "2026-01-10T09:22:21.28Z" },
- { url = "https://files.pythonhosted.org/packages/19/0f/22ef6107ee52ab7f0b710d55d36f5a5d3ef19e8a205541a6d7ffa7994e5a/websockets-16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0", size = 175021, upload-time = "2026-01-10T09:22:22.696Z" },
- { url = "https://files.pythonhosted.org/packages/10/40/904a4cb30d9b61c0e278899bf36342e9b0208eb3c470324a9ecbaac2a30f/websockets-16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957", size = 175320, upload-time = "2026-01-10T09:22:23.94Z" },
- { url = "https://files.pythonhosted.org/packages/9d/2f/4b3ca7e106bc608744b1cdae041e005e446124bebb037b18799c2d356864/websockets-16.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72", size = 183815, upload-time = "2026-01-10T09:22:25.469Z" },
- { url = "https://files.pythonhosted.org/packages/86/26/d40eaa2a46d4302becec8d15b0fc5e45bdde05191e7628405a19cf491ccd/websockets-16.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde", size = 185054, upload-time = "2026-01-10T09:22:27.101Z" },
- { url = "https://files.pythonhosted.org/packages/b0/ba/6500a0efc94f7373ee8fefa8c271acdfd4dca8bd49a90d4be7ccabfc397e/websockets-16.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3", size = 184565, upload-time = "2026-01-10T09:22:28.293Z" },
- { url = "https://files.pythonhosted.org/packages/04/b4/96bf2cee7c8d8102389374a2616200574f5f01128d1082f44102140344cc/websockets-16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3", size = 183848, upload-time = "2026-01-10T09:22:30.394Z" },
- { url = "https://files.pythonhosted.org/packages/02/8e/81f40fb00fd125357814e8c3025738fc4ffc3da4b6b4a4472a82ba304b41/websockets-16.0-cp310-cp310-win32.whl", hash = "sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9", size = 178249, upload-time = "2026-01-10T09:22:32.083Z" },
- { url = "https://files.pythonhosted.org/packages/b4/5f/7e40efe8df57db9b91c88a43690ac66f7b7aa73a11aa6a66b927e44f26fa/websockets-16.0-cp310-cp310-win_amd64.whl", hash = "sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35", size = 178685, upload-time = "2026-01-10T09:22:33.345Z" },
{ url = "https://files.pythonhosted.org/packages/f2/db/de907251b4ff46ae804ad0409809504153b3f30984daf82a1d84a9875830/websockets-16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8", size = 177340, upload-time = "2026-01-10T09:22:34.539Z" },
{ url = "https://files.pythonhosted.org/packages/f3/fa/abe89019d8d8815c8781e90d697dec52523fb8ebe308bf11664e8de1877e/websockets-16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad", size = 175022, upload-time = "2026-01-10T09:22:36.332Z" },
{ url = "https://files.pythonhosted.org/packages/58/5d/88ea17ed1ded2079358b40d31d48abe90a73c9e5819dbcde1606e991e2ad/websockets-16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d", size = 175319, upload-time = "2026-01-10T09:22:37.602Z" },
@@ -2287,18 +2349,6 @@ version = "2.0.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/49/2a/6de8a50cb435b7f42c46126cf1a54b2aab81784e74c8595c8e025e8f36d3/wrapt-2.0.1.tar.gz", hash = "sha256:9c9c635e78497cacb81e84f8b11b23e0aacac7a136e73b8e5b2109a1d9fc468f", size = 82040, upload-time = "2025-11-07T00:45:33.312Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/61/0d/12d8c803ed2ce4e5e7d5b9f5f602721f9dfef82c95959f3ce97fa584bb5c/wrapt-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:64b103acdaa53b7caf409e8d45d39a8442fe6dcfec6ba3f3d141e0cc2b5b4dbd", size = 77481, upload-time = "2025-11-07T00:43:11.103Z" },
- { url = "https://files.pythonhosted.org/packages/05/3e/4364ebe221ebf2a44d9fc8695a19324692f7dd2795e64bd59090856ebf12/wrapt-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:91bcc576260a274b169c3098e9a3519fb01f2989f6d3d386ef9cbf8653de1374", size = 60692, upload-time = "2025-11-07T00:43:13.697Z" },
- { url = "https://files.pythonhosted.org/packages/1f/ff/ae2a210022b521f86a8ddcdd6058d137c051003812b0388a5e9a03d3fe10/wrapt-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ab594f346517010050126fcd822697b25a7031d815bb4fbc238ccbe568216489", size = 61574, upload-time = "2025-11-07T00:43:14.967Z" },
- { url = "https://files.pythonhosted.org/packages/c6/93/5cf92edd99617095592af919cb81d4bff61c5dbbb70d3c92099425a8ec34/wrapt-2.0.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:36982b26f190f4d737f04a492a68accbfc6fa042c3f42326fdfbb6c5b7a20a31", size = 113688, upload-time = "2025-11-07T00:43:18.275Z" },
- { url = "https://files.pythonhosted.org/packages/a0/0a/e38fc0cee1f146c9fb266d8ef96ca39fb14a9eef165383004019aa53f88a/wrapt-2.0.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23097ed8bc4c93b7bf36fa2113c6c733c976316ce0ee2c816f64ca06102034ef", size = 115698, upload-time = "2025-11-07T00:43:19.407Z" },
- { url = "https://files.pythonhosted.org/packages/b0/85/bef44ea018b3925fb0bcbe9112715f665e4d5309bd945191da814c314fd1/wrapt-2.0.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8bacfe6e001749a3b64db47bcf0341da757c95959f592823a93931a422395013", size = 112096, upload-time = "2025-11-07T00:43:16.5Z" },
- { url = "https://files.pythonhosted.org/packages/7c/0b/733a2376e413117e497aa1a5b1b78e8f3a28c0e9537d26569f67d724c7c5/wrapt-2.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8ec3303e8a81932171f455f792f8df500fc1a09f20069e5c16bd7049ab4e8e38", size = 114878, upload-time = "2025-11-07T00:43:20.81Z" },
- { url = "https://files.pythonhosted.org/packages/da/03/d81dcb21bbf678fcda656495792b059f9d56677d119ca022169a12542bd0/wrapt-2.0.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:3f373a4ab5dbc528a94334f9fe444395b23c2f5332adab9ff4ea82f5a9e33bc1", size = 111298, upload-time = "2025-11-07T00:43:22.229Z" },
- { url = "https://files.pythonhosted.org/packages/c9/d5/5e623040e8056e1108b787020d56b9be93dbbf083bf2324d42cde80f3a19/wrapt-2.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f49027b0b9503bf6c8cdc297ca55006b80c2f5dd36cecc72c6835ab6e10e8a25", size = 113361, upload-time = "2025-11-07T00:43:24.301Z" },
- { url = "https://files.pythonhosted.org/packages/a1/f3/de535ccecede6960e28c7b722e5744846258111d6c9f071aa7578ea37ad3/wrapt-2.0.1-cp310-cp310-win32.whl", hash = "sha256:8330b42d769965e96e01fa14034b28a2a7600fbf7e8f0cc90ebb36d492c993e4", size = 58035, upload-time = "2025-11-07T00:43:28.96Z" },
- { url = "https://files.pythonhosted.org/packages/21/15/39d3ca5428a70032c2ec8b1f1c9d24c32e497e7ed81aed887a4998905fcc/wrapt-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:1218573502a8235bb8a7ecaed12736213b22dcde9feab115fa2989d42b5ded45", size = 60383, upload-time = "2025-11-07T00:43:25.804Z" },
- { url = "https://files.pythonhosted.org/packages/43/c2/dfd23754b7f7a4dce07e08f4309c4e10a40046a83e9ae1800f2e6b18d7c1/wrapt-2.0.1-cp310-cp310-win_arm64.whl", hash = "sha256:eda8e4ecd662d48c28bb86be9e837c13e45c58b8300e43ba3c9b4fa9900302f7", size = 58894, upload-time = "2025-11-07T00:43:27.074Z" },
{ url = "https://files.pythonhosted.org/packages/98/60/553997acf3939079dab022e37b67b1904b5b0cc235503226898ba573b10c/wrapt-2.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e17283f533a0d24d6e5429a7d11f250a58d28b4ae5186f8f47853e3e70d2590", size = 77480, upload-time = "2025-11-07T00:43:30.573Z" },
{ url = "https://files.pythonhosted.org/packages/2d/50/e5b3d30895d77c52105c6d5cbf94d5b38e2a3dd4a53d22d246670da98f7c/wrapt-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:85df8d92158cb8f3965aecc27cf821461bb5f40b450b03facc5d9f0d4d6ddec6", size = 60690, upload-time = "2025-11-07T00:43:31.594Z" },
{ url = "https://files.pythonhosted.org/packages/f0/40/660b2898703e5cbbb43db10cdefcc294274458c3ca4c68637c2b99371507/wrapt-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c1be685ac7700c966b8610ccc63c3187a72e33cab53526a27b2a285a662cd4f7", size = 61578, upload-time = "2025-11-07T00:43:32.918Z" },
diff --git a/datajunction-server/datajunction_server/alembic/versions/2026_08_24_0000-rg0001reaggregate_add_reaggregate_to_noderevision.py b/datajunction-server/datajunction_server/alembic/versions/2026_08_24_0000-rg0001reaggregate_add_reaggregate_to_noderevision.py
new file mode 100644
index 000000000..8a04b3fd2
--- /dev/null
+++ b/datajunction-server/datajunction_server/alembic/versions/2026_08_24_0000-rg0001reaggregate_add_reaggregate_to_noderevision.py
@@ -0,0 +1,27 @@
+"""
+Add reaggregate column to noderevision
+
+Revision ID: rg0001reaggregate
+Revises: cm0003dropowner
+Create Date: 2026-08-24 00:00:00.000000+00:00
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "rg0001reaggregate"
+down_revision = "cm0003dropowner"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column(
+ "noderevision",
+ sa.Column("reaggregate", sa.JSON(), nullable=True),
+ )
+
+
+def downgrade():
+ op.drop_column("noderevision", "reaggregate")
diff --git a/datajunction-server/datajunction_server/alembic/versions/2026_09_15_0000-fm0001params_add_params_to_frozen_measures.py b/datajunction-server/datajunction_server/alembic/versions/2026_09_15_0000-fm0001params_add_params_to_frozen_measures.py
new file mode 100644
index 000000000..c00473a55
--- /dev/null
+++ b/datajunction-server/datajunction_server/alembic/versions/2026_09_15_0000-fm0001params_add_params_to_frozen_measures.py
@@ -0,0 +1,29 @@
+"""
+Add params column to frozen_measures
+
+Adds a JSON column to persist tuning parameters (e.g., accuracy) for sketch-backed measures.
+
+Revision ID: fm0001params
+Revises: rg0001reaggregate
+Create Date: 2026-09-15 00:00:00.000000+00:00
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "fm0001params"
+down_revision = "rg0001reaggregate"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column(
+ "frozen_measures",
+ sa.Column("params", sa.JSON(), nullable=True),
+ )
+
+
+def downgrade():
+ op.drop_column("frozen_measures", "params")
diff --git a/datajunction-server/datajunction_server/api/cubes.py b/datajunction-server/datajunction_server/api/cubes.py
index 58e92f6a2..55616b86f 100644
--- a/datajunction-server/datajunction_server/api/cubes.py
+++ b/datajunction-server/datajunction_server/api/cubes.py
@@ -15,6 +15,9 @@
build_combiner_sql_from_preaggs,
)
from datajunction_server.construction.build_v3.cte import strip_role_suffix
+from datajunction_server.construction.build_v3.cube_matcher import (
+ validate_cube_reaggregate_materialization,
+)
from datajunction_server.construction.dimensions import build_dimensions_from_cube_query
from datajunction_server.database.materialization import Materialization
from datajunction_server.database.node import Node
@@ -30,7 +33,6 @@
AccessDenialMode,
get_access_checker,
)
-from datajunction_server.models.access import ResourceAction
from datajunction_server.internal.materializations import (
build_cube_materialization,
stop_cube_materialization_workflows,
@@ -40,6 +42,7 @@
get_single_cube_revision_metadata,
)
from datajunction_server.internal.views import CubeViewNames, _build_view_body
+from datajunction_server.models.access import ResourceAction
from datajunction_server.models.cube import (
CubeRevisionMetadata,
DimensionValue,
@@ -59,11 +62,10 @@
)
from datajunction_server.models.dialect import Dialect
from datajunction_server.models.materialization import (
- DRUID_AGG_MAPPING,
- DRUID_SKETCH_TYPES,
Granularity,
MaterializationJobTypeEnum,
MaterializationStrategy,
+ get_druid_aggregator_spec,
)
from datajunction_server.models.metric import TranslatedSQL
from datajunction_server.models.node_type import NodeNameVersion
@@ -175,32 +177,56 @@ def _build_metrics_spec(
if internal_name:
component = component_by_name.get(internal_name)
- druid_type = "longSum" # Default fallback
-
- if component:
- # Use merge function for pre-aggregated data, fall back to aggregation
- agg_func = component.merge or component.aggregation
- if agg_func:
- key = (col.type, agg_func.lower())
- if key in DRUID_AGG_MAPPING:
- druid_type = DRUID_AGG_MAPPING[key]
-
- metric_spec = {
+ metric_spec = (
+ get_druid_aggregator_spec(
+ column_name=col.name,
+ column_type=col.type,
+ aggregation=component.aggregation,
+ merge=component.merge,
+ params=component.params,
+ )
+ if component
+ else None
+ )
+ # Unmappable measures fall back to longSum here, since we're loading
+ # pre-aggregated data; the materialization config omits them instead.
+ metric_spec = metric_spec or {
"fieldName": col.name,
"name": col.name,
- "type": druid_type,
+ "type": "longSum",
}
- # HLL sketches need additional configuration
- if druid_type in DRUID_SKETCH_TYPES:
- metric_spec["lgK"] = 12 # Log2 of K, controls precision (4-21)
- metric_spec["tgtHllType"] = "HLL_4" # HLL_4, HLL_6, or HLL_8
-
metrics.append(metric_spec)
return metrics
+async def _validate_cube_reaggregate_materialization(
+ session: AsyncSession,
+ cube: Node,
+) -> None:
+ """
+ Validate materialization safety using full metric decomposition.
+ """
+ if not cube.current: # pragma: no cover
+ return
+
+ from datajunction_server.construction.build_v3.builder import setup_build_context
+
+ ctx = await setup_build_context(
+ session=session,
+ metrics=cube.current.cube_node_metrics,
+ dimensions=cube.current.cube_node_dimensions,
+ filters=cube.current.cube_filters or None,
+ dialect=Dialect.SPARK,
+ use_materialized=False,
+ )
+ validate_cube_reaggregate_materialization(
+ cube.current,
+ decomposed_metrics=ctx.decomposed_metrics,
+ )
+
+
@router.get("/cubes", name="Get all Cubes")
async def get_all_cubes(
*,
@@ -288,6 +314,7 @@ async def cube_materialization_info(
message=f"Cube node `{name}` does not exist.",
http_status_code=404,
)
+ await _validate_cube_reaggregate_materialization(session, node)
temporal_partitions = node.current.temporal_partition_columns() # type: ignore
if len(temporal_partitions) != 1:
raise DJInvalidInputException(
@@ -520,6 +547,7 @@ async def materialize_cube(
message=f"Cube '{name}' has no current revision",
http_status_code=HTTPStatus.NOT_FOUND,
)
+ await _validate_cube_reaggregate_materialization(session, node)
cube_tps = cube_revision.temporal_partition_columns()
diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py b/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py
index 536f54abd..802cb6ce1 100644
--- a/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py
+++ b/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py
@@ -1,6 +1,7 @@
"""Metric metadata scalars"""
import strawberry
+from strawberry.scalars import JSON
from datajunction_server.models.cube_materialization import (
Aggregability as Aggregability_,
@@ -15,9 +16,15 @@
MetricComponent as MetricComponent_,
)
from datajunction_server.models.node import MetricDirection as MetricDirection_
+from datajunction_server.models.reaggregate import (
+ DimensionReaggregateRule as DimensionReaggregateRule_,
+ ReaggregateSpec as ReaggregateSpec_,
+ ReaggregationFunction as ReaggregationFunction_,
+)
MetricDirection = strawberry.enum(MetricDirection_)
Aggregability = strawberry.enum(Aggregability_)
+ReaggregationFunction = strawberry.enum(ReaggregationFunction_)
@strawberry.type
@@ -32,12 +39,48 @@ class Unit:
abbreviation: str | None
+@strawberry.experimental.pydantic.type(
+ model=DimensionReaggregateRule_,
+ all_fields=True,
+)
+class DimensionReaggregateRule: ...
+
+
+@strawberry.experimental.pydantic.type(model=ReaggregateSpec_)
+class ReaggregateSpec:
+ """
+ Metric reaggregation declaration.
+
+ Fields are listed explicitly rather than via `all_fields` because `params`
+ is an open dict, which has no automatic GraphQL mapping.
+ """
+
+ fn: strawberry.auto
+ weight: strawberry.auto
+ rules: strawberry.auto
+ params: JSON | None = None
+
+
@strawberry.experimental.pydantic.type(model=AggregationRule_, all_fields=True)
class AggregationRule: ...
-@strawberry.experimental.pydantic.type(model=MetricComponent_, all_fields=True)
-class MetricComponent: ...
+@strawberry.experimental.pydantic.type(model=MetricComponent_)
+class MetricComponent:
+ """
+ A single measure with accumulate/merge phases.
+
+ Fields are listed explicitly rather than via `all_fields` because `params`
+ is an open dict, which has no automatic GraphQL mapping.
+ """
+
+ name: strawberry.auto
+ expression: strawberry.auto
+ aggregation: strawberry.auto
+ merge: strawberry.auto
+ rule: strawberry.auto
+ grain_alias: strawberry.auto
+ params: JSON | None = None
@strawberry.experimental.pydantic.type(model=DecomposedMetric_, all_fields=True)
diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/node.py b/datajunction-server/datajunction_server/api/graphql/scalars/node.py
index e92b7ffd3..24be9d652 100644
--- a/datajunction-server/datajunction_server/api/graphql/scalars/node.py
+++ b/datajunction-server/datajunction_server/api/graphql/scalars/node.py
@@ -31,7 +31,9 @@
)
from datajunction_server.api.graphql.scalars.metricmetadata import (
DecomposedMetric,
+ DimensionReaggregateRule,
MetricMetadata,
+ ReaggregateSpec,
)
from datajunction_server.api.graphql.scalars.user import User
from datajunction_server.api.graphql.utils import extract_fields
@@ -45,6 +47,7 @@
from datajunction_server.models.node import NodeMode as NodeMode_
from datajunction_server.models.node import NodeStatus as NodeStatus_
from datajunction_server.models.node import NodeType as NodeType_
+from datajunction_server.models.reaggregate import parse_reaggregate_spec
from datajunction_server.sql.parsing.backends.antlr4 import ast, parse
NodeType = strawberry.enum(NodeType_)
@@ -410,6 +413,29 @@ def materializations(
# Only metrics will have these fields
required_dimensions: list[Column] | None = None
+ @strawberry.field
+ def reaggregate(self, root: DBNodeRevision) -> ReaggregateSpec | None:
+ """
+ Metric reaggregation declaration.
+ """
+ if root.type != NodeType.METRIC:
+ return None
+ spec = parse_reaggregate_spec(root.reaggregate)
+ if not spec:
+ return None
+ return ReaggregateSpec(
+ fn=spec.fn, # type: ignore
+ weight=spec.weight,
+ params=spec.params,
+ rules=[
+ DimensionReaggregateRule(
+ dimension=rule.dimension,
+ fn=rule.fn, # type: ignore
+ )
+ for rule in spec.rules
+ ],
+ )
+
@strawberry.field
def primary_key(self, root: DBNodeRevision) -> list[str]:
"""
diff --git a/datajunction-server/datajunction_server/api/graphql/schema.graphql b/datajunction-server/datajunction_server/api/graphql/schema.graphql
index 6d442f339..e4f8b97e0 100644
--- a/datajunction-server/datajunction_server/api/graphql/schema.graphql
+++ b/datajunction-server/datajunction_server/api/graphql/schema.graphql
@@ -7,6 +7,7 @@ enum Aggregability {
type AggregationRule {
type: Aggregability!
level: [String!]
+ reaggregate: DimensionReaggregateRule
}
type Attribute {
@@ -149,6 +150,11 @@ type DimensionLink {
defaultValue: String
}
+type DimensionReaggregateRule {
+ dimension: String!
+ fn: ReaggregationFunction!
+}
+
type Engine {
name: String!
version: String!
@@ -296,6 +302,7 @@ type MetricComponent {
merge: String
rule: AggregationRule!
grainAlias: String
+ params: JSON
}
enum MetricDirection {
@@ -397,6 +404,7 @@ type NodeRevision {
dimensionLinks: [DimensionLink!]!
availability: AvailabilityState
materializations: [MaterializationConfig!]
+ reaggregate: ReaggregateSpec
primaryKey: [String!]!
metricMetadata: MetricMetadata
isDerivedMetric: Boolean!
@@ -716,6 +724,25 @@ type Query {
listNamespaces: [Namespace!]!
}
+type ReaggregateSpec {
+ fn: ReaggregationFunction
+ weight: String
+ rules: [DimensionReaggregateRule!]!
+ params: JSON
+}
+
+enum ReaggregationFunction {
+ AUTO
+ NONE
+ SUM
+ AVG
+ WEIGHTED_AVG
+ LAST_VALUE
+ FIRST_VALUE
+ MIN
+ MAX
+}
+
type SemanticEntity {
name: String!
diff --git a/datajunction-server/datajunction_server/api/sql.py b/datajunction-server/datajunction_server/api/sql.py
index 3e1fca938..a5561f7cf 100644
--- a/datajunction-server/datajunction_server/api/sql.py
+++ b/datajunction-server/datajunction_server/api/sql.py
@@ -218,36 +218,7 @@ async def get_measures_sql_v3(
session: AsyncSession = Depends(get_session),
current_user: User = Depends(get_current_user),
) -> MeasuresSQLResponse:
- """
- Generate pre-aggregated measures SQL for the requested metrics.
-
- Measures SQL represents the first stage of metric computation - it decomposes
- each metric into its atomic aggregation components (e.g., SUM(amount), COUNT(*))
- and produces SQL that computes these components at the requested dimensional grain.
-
- Metrics are separated into grain groups, which represent sets of metrics that can be
- computed together at a common grain. Each grain group produces its own SQL query, which
- can be materialized independently to produce intermediate tables that are then queried
- to compute final metric values.
-
- Returns:
- One or more `GrainGroupSQL` objects, each containing:
- - SQL query computing metric components at the specified grain
- - Column metadata with semantic types
- - Component details for downstream re-aggregation
-
- Args:
- cube: Optional cube node name. When provided, the cube's stored filters are
- automatically prepended to the query filters.
- use_materialized: If True (default), use materialized tables when available.
- Set to False when generating SQL for materialization refresh to avoid
- circular references.
- include_temporal_filters: If True, checks if metrics+dimensions resolve to
- a cube with temporal partitions, and applies partition filters if so.
- lookback_window: Lookback window for temporal filters when applicable.
-
- See also: `/sql/metrics/v3/` for the final combined query with metric expressions.
- """
+ """Generate pre-aggregated measures SQL for the requested metrics."""
merged_filters = list(filters)
cube_node = None
if cube:
@@ -440,38 +411,7 @@ async def get_combined_measures_sql_v3(
session: AsyncSession = Depends(get_session),
current_user: User = Depends(get_current_user),
) -> CombinedMeasuresSQLResponse:
- """
- Generate combined pre-aggregated measures SQL for the requested metrics.
-
- This endpoint combines multiple grain groups into a single SQL query using
- FULL OUTER JOIN on shared dimensions. Dimension columns are wrapped with
- COALESCE to handle NULLs from non-matching rows.
-
- This is useful for:
- - Druid cube materialization where a single combined table is needed
- - Simplifying downstream queries that need data from multiple fact tables
- - Pre-computing joined aggregations for dashboards
-
- The combined SQL contains:
- - CTEs for each grain group's pre-aggregated data
- - FULL OUTER JOIN between grain groups on shared dimensions
- - COALESCE on dimension columns to handle NULL values
- - All measure columns from all grain groups
-
- Args:
- metrics: List of metric names to include
- dimensions: List of dimensions to group by (the grain)
- filters: Optional filters to apply
- use_preagg_tables: If False (default), compute from scratch using source tables.
- If True, read from pre-aggregation tables.
-
- Returns:
- Combined SQL query with column metadata and grain information.
-
- See also:
- - `/sql/measures/v3/` for individual grain group queries
- - `/sql/metrics/v3/` for final metric computations with combiner expressions
- """
+ """Generate combined pre-aggregated measures SQL for the requested metrics."""
_t0 = time.monotonic()
if use_preagg_tables:
# Generate SQL that reads from pre-agg tables (deterministic names)
@@ -610,40 +550,7 @@ async def get_metrics_sql_v3(
session: AsyncSession = Depends(get_session),
current_user: User = Depends(get_current_user),
) -> V3TranslatedSQL:
- """
- Generate final metrics SQL with fully computed metric expressions for the
- requested metrics, dimensions, and filters using the specified dialect.
-
- Metrics SQL is the second (and final) stage of metric computation - it takes
- the pre-aggregated components from Measures SQL and applies combiner expressions
- to produce the actual metric values requested.
-
- - Metric components are re-aggregated as needed to match the requested
- dimensional grain.
-
- - Derived metrics (defined as expressions over other metrics)
- (e.g., `conversion_rate = order_count / visitor_count`) are computed by
- substituting component references with their re-aggregated expressions.
-
- - When metrics come from different fact tables, their
- grain groups are FULL OUTER JOINed on the common dimensions, with COALESCE
- for dimension columns to handle NULLs from non-matching rows.
-
- - Dimension references in metric expressions are resolved to their
- final column aliases.
-
- Args:
- metrics: List of metric names to include
- dimensions: List of dimensions to group by (the grain)
- filters: Optional filters to apply
- cube: Optional cube node name. When provided, the cube's stored filters are
- prepended and the cube is passed directly to avoid a redundant lookup.
- dialect: SQL dialect for the generated query. If not specified, auto-resolves
- based on cube availability (uses Druid if cube exists, else metric's catalog).
- use_materialized: If True (default), use materialized tables when available.
- Set to False when generating SQL for materialization refresh to avoid
- circular references.
- """
+ """Generate final metrics SQL with fully computed metric expressions."""
# Shared metrics-SQL core (cube pinning, cube_filters prepend, dialect
# auto-resolve, build_metrics_sql, and the build-latency metrics + [SQL] log).
# Also used by the semantic-layer endpoint.
diff --git a/datajunction-server/datajunction_server/construction/build_v3/builder.py b/datajunction-server/datajunction_server/construction/build_v3/builder.py
index 9b1b5fa7a..6abff4a47 100644
--- a/datajunction-server/datajunction_server/construction/build_v3/builder.py
+++ b/datajunction-server/datajunction_server/construction/build_v3/builder.py
@@ -21,6 +21,7 @@
)
from datajunction_server.construction.build_v3.decomposition import (
decompose_and_group_metrics,
+ missing_reaggregate_dimensions,
)
from datajunction_server.construction.build_v3.dimensions import parse_dimension_ref
from datajunction_server.construction.build_v3.filters import (
@@ -57,6 +58,7 @@
from datajunction_server.errors import DJError, DJInvalidInputException, ErrorCode
from datajunction_server.instrumentation import events
from datajunction_server.models.dialect import Dialect
+from datajunction_server.models.materialization import MaterializationTarget
from datajunction_server.models.partition import PartitionType
from datajunction_server.sql.parsing import ast
from datajunction_server.sql.parsing.backends.antlr4 import parse
@@ -250,6 +252,7 @@ async def setup_build_context(
include_temporal_filters: bool = False,
lookback_window: str | None = None,
matched_cube: NodeRevision | None = None,
+ materialization_target: MaterializationTarget | None = None,
) -> BuildContext:
"""
Create and initialize a BuildContext with all setup done.
@@ -320,6 +323,14 @@ async def setup_build_context(
# Add dimensions referenced in metric expressions (e.g., LAG ORDER BY)
add_dimensions_from_metric_expressions(ctx, ctx.decomposed_metrics)
+ output_dimensions_after_expression_scan = list(ctx.dimensions)
+ internal_reaggregate_dimensions = missing_reaggregate_dimensions(
+ ctx.decomposed_metrics.values(),
+ output_dimensions_after_expression_scan,
+ )
+ for dimension in internal_reaggregate_dimensions:
+ if dimension not in ctx.dimensions:
+ ctx.dimensions.append(dimension)
# A second load_nodes pass is needed when either:
# 1. metric expressions introduced dimension nodes not yet in ctx.nodes, OR
@@ -333,8 +344,15 @@ async def setup_build_context(
}
missing_dim_nodes = dim_roots_after - ctx.nodes.keys()
internally_added_roots = dim_roots_after - dim_roots_before_load
- if missing_dim_nodes or internally_added_roots:
- await load_nodes(ctx)
+ try:
+ if (
+ missing_dim_nodes
+ or internally_added_roots
+ or internal_reaggregate_dimensions
+ ):
+ await load_nodes(ctx)
+ finally:
+ ctx.dimensions = output_dimensions_after_expression_scan
# Classify filters into dimension filters (WHERE) and metric filters (HAVING)
# This MUST happen AFTER all nodes are loaded so we can correctly identify
@@ -355,6 +373,7 @@ async def build_measures_sql(
lookback_window: str | None = None,
query_parameters: dict[str, Any] | None = None,
matched_cube: NodeRevision | None = None,
+ materialization_target: MaterializationTarget | None = None,
) -> GeneratedMeasuresSQL:
"""
Build measures SQL for a set of metrics, dimensions, and filters.
@@ -395,6 +414,7 @@ async def build_measures_sql(
include_temporal_filters=include_temporal_filters,
lookback_window=lookback_window,
matched_cube=matched_cube,
+ materialization_target=materialization_target,
)
# Build grain groups from context
diff --git a/datajunction-server/datajunction_server/construction/build_v3/combiners.py b/datajunction-server/datajunction_server/construction/build_v3/combiners.py
index c39857745..7c5d710cf 100644
--- a/datajunction-server/datajunction_server/construction/build_v3/combiners.py
+++ b/datajunction-server/datajunction_server/construction/build_v3/combiners.py
@@ -23,6 +23,9 @@
from datajunction_server.construction.build_v3.cte import (
process_metric_combiner_expression,
)
+from datajunction_server.construction.build_v3.decomposition import (
+ build_merge_call,
+)
from datajunction_server.construction.build_v3.preagg_matcher import (
get_temporal_partitions,
)
@@ -600,6 +603,7 @@ async def build_combiner_sql_from_preaggs(
measure_identity_token(
compute_expression_hash(m.expression),
m.aggregation,
+ m.params,
)
for m in gg.components
if m.expression
@@ -861,20 +865,19 @@ def _build_grain_group_from_preagg_table(
# Find the component to get the merge function
merge_func = None
+ merge_args: list[str] = []
for comp in original_gg.components:
if ( # pragma: no branch
comp.name == col.name
or original_gg.component_aliases.get(comp.name) == col.name
):
merge_func = comp.merge
+ merge_args = comp.merge_args
break
if merge_func:
# Apply re-aggregation
- agg_expr = ast.Function(
- name=ast.Name(merge_func),
- args=[col_ref],
- )
+ agg_expr = build_merge_call(merge_func, merge_args, col_ref)
aliased = ast.Alias(child=agg_expr, alias=ast.Name(col.name))
select_items.append(aliased)
else:
diff --git a/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py b/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py
index 07fead205..f0955bd04 100644
--- a/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py
+++ b/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py
@@ -12,9 +12,13 @@
from sqlalchemy import and_, select
from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload, load_only, noload, selectinload
+from sqlalchemy.orm import aliased, joinedload, load_only, noload, selectinload
-from datajunction_server.construction.build_v3.decomposition import is_derived_metric
+from datajunction_server.construction.build_v3.decomposition import (
+ _reaggregate_dimension_requested,
+ is_derived_metric,
+ missing_reaggregate_dimensions,
+)
from datajunction_server.construction.build_v3.dimensions import parse_dimension_ref
from datajunction_server.construction.build_v3.filters import (
parse_and_resolve_filters,
@@ -36,14 +40,19 @@
)
from datajunction_server.database.catalog import Catalog
from datajunction_server.database.column import Column
-from datajunction_server.database.node import Node, NodeRevision
+from datajunction_server.database.node import Node, NodeRelationship, NodeRevision
from datajunction_server.database.partition import Partition
from datajunction_server.errors import DJInvalidInputException
from datajunction_server.instrumentation.provider import timed
from datajunction_server.models.decompose import Aggregability
from datajunction_server.models.dialect import Dialect
from datajunction_server.models.node_type import NodeType
+from datajunction_server.models.reaggregate import (
+ ReaggregationFunction,
+ dimension_reaggregate_rules,
+)
from datajunction_server.naming import amenable_name
+from datajunction_server.sql.decompose import MetricComponentExtractor
from datajunction_server.sql.parsing import ast
logger = logging.getLogger(__name__)
@@ -53,6 +62,327 @@
# parsed: coverage can't be proven, so cube matching must fail SAFE (reject the
# cube and fall back) rather than fail open and emit invalid SQL.
_FILTER_COVERAGE_UNKNOWN = object()
+ReaggregateRequirement = tuple[str, str, ReaggregationFunction]
+
+
+def _split_dimension_ref(ref: str) -> tuple[str, str | None]:
+ """
+ Return a dimension ref without role suffix and the role suffix, if present.
+ """
+ if "[" not in ref:
+ return ref, None
+ base, role = ref.rsplit("[", 1)
+ return base, role.rstrip("]")
+
+
+def _cube_dimension_covers_reaggregate_dimension(
+ protected_dimension: str,
+ cube_dimension: str,
+) -> bool:
+ """
+ Return whether a cube dimension materializes a protected dimension.
+ """
+ if _reaggregate_dimension_requested(protected_dimension, [cube_dimension]):
+ return True
+
+ protected_base, protected_role = _split_dimension_ref(protected_dimension)
+ cube_base, cube_role = _split_dimension_ref(cube_dimension)
+ if protected_role != cube_role:
+ return False
+
+ # Deployment validation allows a metric to author the protected dimension as
+ # a bare parent column (the same form required_dimensions accepts). Cubes
+ # expose dimensions as fully-qualified element names, so compare the short
+ # column name for that narrow compatibility case.
+ if "." not in protected_base:
+ return protected_base == cube_base.rsplit(".", 1)[-1]
+ return False
+
+
+def _missing_reaggregate_requirements(
+ requirements: list[ReaggregateRequirement],
+ cube_dims: set[str],
+) -> list[ReaggregateRequirement]:
+ """
+ Reaggregate requirements not covered by a cube's materialized dimensions.
+ """
+ return [
+ requirement
+ for requirement in requirements
+ if not any(
+ _cube_dimension_covers_reaggregate_dimension(
+ requirement[1],
+ cube_dim,
+ )
+ for cube_dim in cube_dims
+ )
+ ]
+
+
+async def _metric_graph_has_reaggregate(
+ session: AsyncSession,
+ metrics: list[str],
+) -> bool:
+ """
+ Cheaply test whether requested metrics or metric ancestors use reaggregate.
+ """
+ seen: set[str] = set()
+ pending = set(metrics)
+ ParentNode = aliased(Node)
+
+ while pending:
+ batch = sorted(pending - seen)
+ if not batch:
+ break
+ seen.update(batch)
+ result = await session.execute(
+ select(
+ Node.name,
+ NodeRevision.reaggregate,
+ ParentNode.name.label("parent_name"),
+ )
+ .select_from(Node)
+ .join(
+ NodeRevision,
+ and_(
+ Node.id == NodeRevision.node_id,
+ Node.current_version == NodeRevision.version,
+ ),
+ )
+ .outerjoin(NodeRelationship, NodeRelationship.child_id == NodeRevision.id)
+ .outerjoin(
+ ParentNode,
+ and_(
+ NodeRelationship.parent_id == ParentNode.id,
+ ParentNode.type == NodeType.METRIC,
+ ),
+ )
+ .where(Node.name.in_(batch))
+ .where(Node.type == NodeType.METRIC),
+ )
+ for _, reaggregate, parent_name in result.all():
+ if reaggregate:
+ return True
+ if parent_name and parent_name not in seen:
+ pending.add(parent_name)
+
+ return False
+
+
+async def _reaggregate_requirements_for_metrics_if_needed(
+ session: AsyncSession,
+ metrics: list[str],
+ dimensions: list[str],
+) -> list[ReaggregateRequirement]:
+ """
+ Run full metric decomposition only when reaggregate is possible.
+ """
+ if not await _metric_graph_has_reaggregate(session, metrics):
+ return []
+ return await _reaggregate_requirements_for_metrics(session, metrics, dimensions)
+
+
+def _reaggregate_requirements_for_cube_metrics(
+ cube: NodeRevision,
+ metrics: list[str],
+ requested_dimensions: list[str],
+) -> list[ReaggregateRequirement]:
+ """
+ Reaggregation requirements that are absent from the output grain.
+ """
+ requirements: list[ReaggregateRequirement] = []
+ metric_names = set(metrics)
+ for metric_revision in cube.metric_node_revisions():
+ if (
+ not metric_revision
+ or metric_revision.name not in metric_names
+ or not metric_revision.reaggregate
+ ):
+ continue
+ for rule in dimension_reaggregate_rules(metric_revision.reaggregate):
+ if not _reaggregate_dimension_requested(
+ rule.dimension,
+ requested_dimensions,
+ ):
+ requirements.append(
+ (
+ metric_revision.name,
+ rule.dimension,
+ rule.fn,
+ ),
+ )
+ return requirements
+
+
+async def _reaggregate_requirements_for_metrics(
+ session: AsyncSession,
+ metrics: list[str],
+ requested_dimensions: list[str],
+) -> list[ReaggregateRequirement]:
+ """
+ Reaggregation requirements from full metric decomposition.
+
+ Direct cube metadata only sees the metrics declared on the cube. Decomposition
+ also catches derived metrics whose base components carry reaggregate rules.
+ """
+ requirements: list[ReaggregateRequirement] = []
+ for metric_name in metrics:
+ extractor = await MetricComponentExtractor.from_node_name(
+ metric_name,
+ session,
+ )
+ components, _ = await extractor.extract(session)
+ for component in components:
+ reaggregate = component.rule.reaggregate
+ if (
+ reaggregate
+ and not _reaggregate_dimension_requested(
+ reaggregate.dimension,
+ requested_dimensions,
+ )
+ and (
+ metric_name,
+ reaggregate.dimension,
+ reaggregate.fn,
+ )
+ not in requirements
+ ):
+ requirements.append(
+ (
+ metric_name,
+ reaggregate.dimension,
+ reaggregate.fn,
+ ),
+ )
+ return requirements
+
+
+def _reaggregate_requirements_for_decomposed_metrics(
+ decomposed_metrics: dict[str, DecomposedMetricInfo],
+ metrics: list[str],
+ requested_dimensions: list[str],
+) -> list[ReaggregateRequirement]:
+ """
+ Reaggregation requirements from already-decomposed planner context.
+ """
+ requirements: list[ReaggregateRequirement] = []
+ metric_names = set(metrics)
+ for metric_name, decomposed in decomposed_metrics.items():
+ if metric_name not in metric_names:
+ continue
+ for component in decomposed.components:
+ reaggregate = component.rule.reaggregate
+ if reaggregate and not _reaggregate_dimension_requested(
+ reaggregate.dimension,
+ requested_dimensions,
+ ):
+ requirement = (
+ metric_name,
+ reaggregate.dimension,
+ reaggregate.fn,
+ )
+ if requirement not in requirements:
+ requirements.append(requirement)
+ return requirements
+
+
+def _reaggregate_dimensions_for_cube_metrics(
+ cube: NodeRevision,
+ metrics: list[str],
+ requested_dimensions: list[str],
+) -> list[str]:
+ """
+ Protected dimensions a cube must materialize for semi-additive rollups.
+ """
+ required: list[str] = []
+ for _, dimension, _ in _reaggregate_requirements_for_cube_metrics(
+ cube,
+ metrics,
+ requested_dimensions,
+ ):
+ if dimension not in required:
+ required.append(dimension)
+ return required
+
+
+def _format_reaggregate_requirements(
+ requirements: list[ReaggregateRequirement],
+) -> list[str]:
+ """Human-readable semi-additive requirements for error messages."""
+ return [
+ f"{metric} -> {dimension} ({function.value})"
+ for metric, dimension, function in requirements
+ ]
+
+
+def validate_cube_covers_reaggregate_dimensions(
+ cube: NodeRevision,
+ metrics: list[str],
+ dimensions: list[str],
+ *,
+ decomposed_metrics: dict[str, DecomposedMetricInfo] | None = None,
+ additional_requirements: list[ReaggregateRequirement] | None = None,
+ usage: str = "Cube",
+ http_status_code: int = 422,
+) -> None:
+ """
+ Fail loud when a materialized cube cannot safely serve semi-additive metrics.
+
+ V1 materialized cube usage is only safe for semi-additive metrics when the
+ cube itself retains the protected dimension. Otherwise querying the cube can
+ silently merge across that dimension with the wrong Druid aggregator.
+ """
+ requirements = _reaggregate_requirements_for_cube_metrics(
+ cube,
+ metrics,
+ dimensions,
+ )
+ if decomposed_metrics is not None:
+ for requirement in _reaggregate_requirements_for_decomposed_metrics(
+ decomposed_metrics,
+ metrics,
+ dimensions,
+ ):
+ if requirement not in requirements:
+ requirements.append(requirement)
+ for requirement in additional_requirements or []:
+ if requirement not in requirements:
+ requirements.append(requirement)
+
+ cube_dims = set(cube.cube_dimensions())
+ missing_requirements = _missing_reaggregate_requirements(requirements, cube_dims)
+ if not missing_requirements:
+ return
+
+ raise DJInvalidInputException(
+ f"{usage} `{cube.name}` cannot safely use materialized semi-additive "
+ "metric(s) because it does not materialize protected dimension(s) "
+ f"{_format_reaggregate_requirements(missing_requirements)}. "
+ "Add the protected dimension to the cube, or run the query with "
+ "use_materialized=false.",
+ http_status_code=http_status_code,
+ )
+
+
+def validate_cube_reaggregate_materialization(
+ cube: NodeRevision,
+ decomposed_metrics: dict[str, DecomposedMetricInfo] | None = None,
+) -> None:
+ """
+ Guard Druid cube materialization from baking in semi-additive misaggregation.
+
+ V1 does not materialize collapsed semi-additive metrics unless the cube grain
+ includes the protected dimension. This avoids generating a Druid metricsSpec
+ that sums first/last/min/max collapse inputs across the protected dimension.
+ """
+ validate_cube_covers_reaggregate_dimensions(
+ cube,
+ cube.cube_node_metrics,
+ cube.cube_node_dimensions,
+ decomposed_metrics=decomposed_metrics,
+ usage="Cube",
+ http_status_code=400,
+ )
async def _required_filter_dimensions(
@@ -190,7 +520,15 @@ async def find_matching_cube(
result = await session.execute(statement)
candidate_cubes = result.unique().scalars().all()
-
+ reaggregate_requirements = (
+ await _reaggregate_requirements_for_metrics_if_needed(
+ session,
+ metrics,
+ dimensions,
+ )
+ if candidate_cubes
+ else []
+ )
# Find the best matching cube (smallest grain that covers all dimensions)
best_match: NodeRevision | None = None
best_grain_size = float("inf")
@@ -214,12 +552,32 @@ async def find_matching_cube(
# Druid SQL referencing a missing column).
cube_dims = set(cube_rev.cube_dimensions())
- if not required_dims.issubset(cube_dims):
+ missing_required_dims = required_dims - cube_dims
+ cube_reaggregate_requirements = _reaggregate_requirements_for_cube_metrics(
+ cube_rev,
+ metrics,
+ dimensions,
+ )
+ for requirement in reaggregate_requirements:
+ if requirement not in cube_reaggregate_requirements:
+ cube_reaggregate_requirements.append(requirement)
+ missing_reaggregate_requirements = _missing_reaggregate_requirements(
+ cube_reaggregate_requirements,
+ cube_dims,
+ )
+ if missing_required_dims:
logger.debug(
f"[BuildV3] Cube {cube_rev.name} dims {cube_dims} "
f"don't cover required {required_dims}",
)
continue
+ if missing_reaggregate_requirements:
+ logger.debug(
+ f"[BuildV3] Cube {cube_rev.name} dims {cube_dims} "
+ f"don't cover reaggregate requirements "
+ f"{_format_reaggregate_requirements(missing_reaggregate_requirements)}",
+ )
+ continue
# Found a match - prefer smallest grain (less roll-up work)
if len(cube_dims) < best_grain_size:
@@ -244,6 +602,7 @@ async def validate_pinned_cube_covers_filters(
cube: NodeRevision,
dimensions: list[str],
filters: list[str] | None,
+ metrics: list[str] | None = None,
) -> None:
"""
Ensure an explicitly pinned cube covers every filtered dimension.
@@ -278,6 +637,18 @@ async def validate_pinned_cube_covers_filters(
http_status_code=422,
)
+ validate_cube_covers_reaggregate_dimensions(
+ cube,
+ metrics or cube.cube_node_metrics,
+ dimensions,
+ additional_requirements=await _reaggregate_requirements_for_metrics_if_needed(
+ session,
+ metrics or cube.cube_node_metrics,
+ dimensions,
+ ),
+ usage="Pinned cube",
+ )
+
async def resolve_dialect_and_engine_for_metrics(
session: AsyncSession,
@@ -465,6 +836,13 @@ def build_sql_from_cube_impl(
Returns:
GeneratedSQL with the query and column metadata.
"""
+ validate_cube_covers_reaggregate_dimensions(
+ cube,
+ ctx.metrics,
+ ctx.dimensions,
+ decomposed_metrics=ctx.decomposed_metrics,
+ )
+
# Build synthetic GrainGroupSQL for cube table
# This applies all filters in the cube CTE's WHERE clause
synthetic_grain_group = build_synthetic_grain_group(
@@ -640,13 +1018,27 @@ def build_synthetic_grain_group(
# (filter-only dimensions were added by add_dimensions_from_filters() in setup_build_context)
dimension_aliases: dict[str, str] = {}
- # Add all dimensions (requested + filter-only). We need all dimensions
+ output_dimensions = [
+ dim for dim in ctx.dimensions if dim not in ctx.filter_dimensions
+ ]
+ internal_reaggregate_dimensions = missing_reaggregate_dimensions(
+ decomposed_metrics.values(),
+ output_dimensions,
+ )
+
+ # Add all dimensions (requested + filter-only + internal semi-additive
+ # protected dimensions). We need all dimensions
# in the cube SELECT for proper filter resolution.
# dim_short_names holds the alias (short name) used everywhere outside the CTE.
# dim_physical_names holds the actual column name in the Druid table (may differ).
dim_short_names = []
dim_physical_names = []
- for dim_ref in ctx.dimensions:
+ dimension_to_alias: dict[str, str] = {}
+ dimension_refs = list(ctx.dimensions)
+ for dim_ref in internal_reaggregate_dimensions:
+ if dim_ref not in dimension_refs:
+ dimension_refs.append(dim_ref)
+ for dim_ref in dimension_refs:
parsed_dim = parse_dimension_ref(dim_ref)
short_name = parsed_dim.column_name
if parsed_dim.role:
@@ -662,6 +1054,7 @@ def build_synthetic_grain_group(
# the WHERE is applied directly on the cube table, so we must reference
# the physical column (e.g. common_DOT_..._DOT_dateint) not the alias.
dimension_aliases[dim_ref] = physical_name
+ dimension_to_alias[dim_ref] = short_name
grain_group_columns.append(
ColumnMetadata(
name=short_name,
@@ -735,6 +1128,17 @@ def build_synthetic_grain_group(
if metric_node and not is_derived_metric(ctx, metric_node):
base_metrics.append(metric_name)
+ reaggregate_dimension_aliases: dict[str, str] = {}
+ internal_reaggregate_dimension_set = set(internal_reaggregate_dimensions)
+ for comp in all_components:
+ if (
+ comp.rule.reaggregate
+ and comp.rule.reaggregate.dimension in internal_reaggregate_dimension_set
+ ):
+ reaggregate_dimension_aliases[comp.name] = dimension_to_alias[
+ comp.rule.reaggregate.dimension
+ ]
+
# Create the synthetic GrainGroupSQL
# Note: We use a placeholder parent_name since the cube combines multiple parents
return GrainGroupSQL(
@@ -745,6 +1149,7 @@ def build_synthetic_grain_group(
metrics=base_metrics, # Only base metrics, not derived
parent_name=cube.name, # Use cube name as parent
component_aliases=component_aliases,
+ reaggregate_dimension_aliases=reaggregate_dimension_aliases,
is_merged=False,
components=all_components,
dialect=ctx.dialect,
diff --git a/datajunction-server/datajunction_server/construction/build_v3/decomposition.py b/datajunction-server/datajunction_server/construction/build_v3/decomposition.py
index 8a3c1b56f..e5eb36ffe 100644
--- a/datajunction-server/datajunction_server/construction/build_v3/decomposition.py
+++ b/datajunction-server/datajunction_server/construction/build_v3/decomposition.py
@@ -9,6 +9,8 @@
from __future__ import annotations
+from collections import defaultdict
+from collections.abc import Iterable
from typing import cast
from sqlalchemy.ext.asyncio import AsyncSession
@@ -22,10 +24,13 @@
from datajunction_server.database.node import Node
from datajunction_server.errors import DJInvalidInputException
from datajunction_server.models.decompose import Aggregability, MetricComponent
+from datajunction_server.models.dialect import Dialect
+from datajunction_server.models.materialization import MaterializationTarget
from datajunction_server.models.node_type import NodeType
from datajunction_server.sql.decompose import MetricComponentExtractor
from datajunction_server.sql.parsing import ast
from datajunction_server.sql.parsing.backends.antlr4 import parse
+from datajunction_server.utils import SEPARATOR
async def decompose_and_group_metrics(
@@ -73,6 +78,7 @@ async def decompose_and_group_metrics(
base_metric,
nodes_cache=ctx.nodes,
parent_map=ctx.parent_map,
+ dialect=ctx.dialect,
)
all_decomposed[base_metric.name] = decomposed
@@ -92,6 +98,7 @@ async def decompose_and_group_metrics(
metric_node,
nodes_cache=ctx.nodes,
parent_map=ctx.parent_map,
+ dialect=ctx.dialect,
)
all_decomposed[metric_name] = derived_decomposed
else:
@@ -115,6 +122,7 @@ async def decompose_and_group_metrics(
metric_node,
nodes_cache=ctx.nodes,
parent_map=ctx.parent_map,
+ dialect=ctx.dialect,
)
all_decomposed[metric_node.name] = decomposed
@@ -140,6 +148,7 @@ async def decompose_metric(
*,
nodes_cache: dict[str, Node] | None = None,
parent_map: dict[str, list[str]] | None = None,
+ dialect: Dialect = Dialect.SPARK,
) -> DecomposedMetricInfo:
"""
Decompose a metric into its constituent components.
@@ -156,6 +165,10 @@ async def decompose_metric(
parent_map, avoids database queries by using cached data.
parent_map: Optional dict of child_name -> list of parent_names.
Required if nodes_cache is provided.
+ dialect: Dialect the combiner is rendered for. ``decompose_and_group_metrics``
+ passes the build's resolved dialect, which for a materialized cube is
+ the one taken from its availability catalog -- Druid, in practice.
+ Defaults to Spark for direct callers.
Returns:
DecomposedMetricInfo with components, combiner expression, and aggregability
@@ -166,7 +179,7 @@ async def decompose_metric(
)
# Use the MetricComponentExtractor with optional cache
- extractor = MetricComponentExtractor(metric_node.current.id)
+ extractor = MetricComponentExtractor(metric_node.current.id, dialect=dialect)
components, derived_ast = await extractor.extract(
session,
nodes_cache=nodes_cache,
@@ -200,7 +213,10 @@ async def decompose_metric(
)
-def build_component_expression(component: MetricComponent) -> ast.Expression:
+def build_component_expression(
+ component: MetricComponent,
+ materialization_target: MaterializationTarget | None = None,
+) -> ast.Expression:
"""
Build the accumulate expression AST for a metric component.
@@ -210,11 +226,18 @@ def build_component_expression(component: MetricComponent) -> ast.Expression:
Note: Templates may be pre-expanded (e.g., "SUM(POWER(match_score, 2))")
by the decomposition phase, so we detect this by checking for parentheses
without template placeholders.
+
+ When ``materialization_target`` is one the component declares a ``serialize``
+ conversion for, the accumulated expression is wrapped in it. That happens
+ only while writing a measures table: a query-time build passes no target, so
+ the unwrapped expression stands.
"""
if not component.aggregation: # pragma: no cover
# No aggregation - just return the expression as a column
return ast.Column(name=ast.Name(component.expression))
+ accumulated: ast.Expression
+
# Check if it's an unexpanded template with {}
if "{" in component.aggregation: # pragma: no cover
# Template like "SUM(POWER({}, 2))" - expand it
@@ -224,25 +247,67 @@ def build_component_expression(component: MetricComponent) -> ast.Expression:
if isinstance(expr_ast, ast.Alias):
expr_ast = expr_ast.child
expr_ast.clear_parent()
- return cast(ast.Expression, expr_ast)
+ accumulated = cast(ast.Expression, expr_ast)
# Check if it's a pre-expanded template (contains parentheses, like "SUM(POWER(x, 2))")
# vs a simple function name (like "SUM")
- if "(" in component.aggregation:
+ elif "(" in component.aggregation:
# Pre-expanded template - parse it directly as a complete expression
expr_ast = parse(f"SELECT {component.aggregation}").select.projection[0]
if isinstance(expr_ast, ast.Alias):
expr_ast = expr_ast.child # pragma: no cover
expr_ast.clear_parent()
- return cast(ast.Expression, expr_ast)
+ accumulated = cast(ast.Expression, expr_ast)
else:
# Simple function name like "SUM" - build SUM(expression)
arg_expr = parse(f"SELECT {component.expression}").select.projection[0]
- func = ast.Function(
+ accumulated = ast.Function(
name=ast.Name(component.aggregation),
args=[cast(ast.Expression, arg_expr)],
)
- return func
+
+ # All accumulate shapes must receive the target conversion.
+ return _apply_serialize(accumulated, component, materialization_target)
+
+
+def _apply_serialize(
+ expr: ast.Expression,
+ component: MetricComponent,
+ materialization_target: MaterializationTarget | None,
+) -> ast.Expression:
+ """
+ Wrap an accumulated expression in the component's serialize conversion.
+
+ Returns the expression untouched unless the component declares a conversion
+ and names this target -- so every existing component, and every query-time
+ build, is unaffected.
+ """
+ if not component.serializes_for(materialization_target):
+ return expr
+ wrapped = parse(
+ f"SELECT {component.serialize.replace('{}', str(expr))}",
+ ).select.projection[0]
+ wrapped.clear_parent()
+ return cast(ast.Expression, wrapped)
+
+
+def build_merge_call(
+ merge: str,
+ merge_args: list[str],
+ arg: ast.Expression,
+) -> ast.Function:
+ """
+ Build a component's Phase 2 merge call, including any fixed tuning arguments.
+
+ Components without `merge_args` -- every non-sketch aggregation -- produce
+ the same single-argument call as before.
+ """
+ extra: list[ast.Expression] = []
+ for literal in merge_args:
+ parsed = parse(f"SELECT {literal}").select.projection[0]
+ parsed.clear_parent()
+ extra.append(cast(ast.Expression, parsed))
+ return ast.Function(name=ast.Name(merge), args=[arg, *extra])
def get_base_metrics_for_derived(ctx: BuildContext, metric_node: Node) -> list[Node]:
@@ -260,12 +325,9 @@ def collect_bases(node: Node):
return
visited.add(node.name)
- # Recurse through metric parents. A node with any metric parent is itself
- # derived; a node with no metric parent is a base metric -- regardless of
- # whether its data source is a fact/transform or a dimension node (a base
- # metric can be defined directly on a dimension node). Classifying by
- # "has a metric parent" avoids mistaking a dimension-sourced base metric
- # for a bare required-dimension reference, which would drop its grain group.
+ # Recurse through metric parents. A node with any metric parent is derived;
+ # a node with no metric parent is a base metric. Checking for metric parents
+ # avoids dropping base metrics that are defined directly on dimension nodes.
has_metric_parent = False
for parent_name in ctx.parent_map.get(node.name, []):
parent = ctx.nodes.get(parent_name)
@@ -322,6 +384,76 @@ def get_native_grain(node: Node) -> list[str]:
return pk_columns
+def _dimension_ref_base(ref: str) -> str:
+ """Return a dimension ref without its role suffix."""
+ return ref.split("[", 1)[0]
+
+
+def _dimension_ref_role(ref: str) -> str | None:
+ """Return the role suffix for a dimension ref, if any."""
+ if "[" not in ref:
+ return None
+ return ref.rsplit("[", 1)[1].rstrip("]")
+
+
+def _dimension_ref_column(ref: str) -> str:
+ """Return the column part of a fully qualified or local dimension ref."""
+ return _dimension_ref_base(ref).rsplit(SEPARATOR, 1)[-1]
+
+
+def _reaggregate_dimension_requested(
+ protected_dimension: str,
+ requested_dimensions: list[str],
+) -> bool:
+ """
+ Whether the protected dimension is already in the requested output grain.
+
+ Role-qualified dimensions only count when the role matches exactly. Coarser
+ dimensions on the same node do not count.
+ """
+ protected_base = _dimension_ref_base(protected_dimension)
+ protected_role = _dimension_ref_role(protected_dimension)
+ protected_col = _dimension_ref_column(protected_dimension)
+ for requested in requested_dimensions:
+ requested_base = _dimension_ref_base(requested)
+ requested_role = _dimension_ref_role(requested)
+ if requested == protected_dimension:
+ return True
+ if requested_base == protected_base and requested_role == protected_role:
+ return True
+ if (
+ protected_role is None
+ and requested_role is None
+ and requested == protected_col
+ ):
+ return True
+ return False
+
+
+def missing_reaggregate_dimensions(
+ decomposed_metrics: Iterable[DecomposedMetricInfo],
+ requested_dimensions: list[str],
+) -> list[str]:
+ """
+ Protected dimensions needed as internal grain but absent from output grain.
+ """
+ missing: list[str] = []
+ for decomposed in decomposed_metrics:
+ for component in decomposed.components:
+ if not component.rule.reaggregate:
+ continue
+ dimension = component.rule.reaggregate.dimension
+ if (
+ not _reaggregate_dimension_requested(
+ dimension,
+ requested_dimensions,
+ )
+ and dimension not in missing
+ ):
+ missing.append(dimension)
+ return missing
+
+
def analyze_grain_groups(
metric_group: MetricGroup,
requested_dimensions: list[str],
@@ -341,7 +473,7 @@ def analyze_grain_groups(
Args:
metric_group: MetricGroup with decomposed metrics
- requested_dimensions: Dimensions requested by user (column names only)
+ requested_dimensions: Dimension refs requested by the user
Returns:
List of GrainGroups, one per unique grain
@@ -349,11 +481,15 @@ def analyze_grain_groups(
parent_node = metric_group.parent_node
# Group components by their effective grain
- # Key: (aggregability, tuple of additional grain columns)
+ # Key: (aggregability, tuple of additional grain columns, tuple of internal dimensions)
grain_buckets: dict[
- tuple[Aggregability, tuple[str, ...]],
+ tuple[Aggregability, tuple[str, ...], tuple[str, ...]],
list[tuple[Node, MetricComponent]],
] = {}
+ reaggregate_component_dimensions: dict[
+ tuple[Aggregability, tuple[str, ...], tuple[str, ...]],
+ dict[str, str],
+ ] = {}
# Track non-decomposable metrics (those with no components)
non_decomposable: list[DecomposedMetricInfo] = []
@@ -369,29 +505,50 @@ def analyze_grain_groups(
agg_type = component.rule.type
# Explicitly type the key to satisfy mypy
- key: tuple[Aggregability, tuple[str, ...]]
+ key: tuple[Aggregability, tuple[str, ...], tuple[str, ...]]
+ reaggregate_dimension = (
+ component.rule.reaggregate.dimension
+ if (
+ component.rule.reaggregate
+ and not _reaggregate_dimension_requested(
+ component.rule.reaggregate.dimension,
+ requested_dimensions,
+ )
+ )
+ else None
+ )
+ internal_dimensions = (
+ (reaggregate_dimension,) if reaggregate_dimension else ()
+ )
if agg_type == Aggregability.FULL:
- # FULL: no additional grain columns needed
- key = (Aggregability.FULL, ())
+ # FULL: no additional grain columns needed, unless a semi-additive
+ # component must keep its protected dimension as internal grain.
+ key = (Aggregability.FULL, (), internal_dimensions)
elif agg_type == Aggregability.LIMITED:
# LIMITED: add level columns to grain
level_cols = tuple(sorted(component.rule.level or []))
- key = (Aggregability.LIMITED, level_cols)
+ key = (Aggregability.LIMITED, level_cols, internal_dimensions)
else: # NONE
# NONE: use native grain (PK columns)
native_grain = get_native_grain(parent_node)
key = (
Aggregability.NONE,
tuple(sorted(native_grain)),
+ internal_dimensions,
) # pragma: no cover
if key not in grain_buckets:
grain_buckets[key] = []
grain_buckets[key].append((decomposed.metric_node, component))
+ if reaggregate_dimension:
+ reaggregate_component_dimensions.setdefault(key, {})[component.name] = (
+ reaggregate_dimension
+ )
# Convert buckets to GrainGroup objects
grain_groups = []
- for (agg_type, grain_cols), components in grain_buckets.items():
+ for bucket_key, components in grain_buckets.items():
+ agg_type, grain_cols, _internal_dimensions = bucket_key
# Map each grain expression to its SQL alias.
# LIMITED: alias comes from component.grain_alias (_make_component decides:
# plain column maps to the bare name, complex expr maps to component.name).
@@ -412,6 +569,10 @@ def analyze_grain_groups(
grain_columns=list(grain_cols),
components=components,
grain_col_aliases=grain_col_aliases,
+ reaggregate_component_dimensions=reaggregate_component_dimensions.get(
+ bucket_key,
+ {},
+ ),
),
)
@@ -470,9 +631,9 @@ def merge_grain_groups(grain_groups: list[GrainGroup]) -> list[GrainGroup]:
Returns:
List of grain groups with compatible groups merged
"""
- from collections import defaultdict
-
- # Group by parent node name
+ # Group by parent node name first, then by the internal grain needed for
+ # semi-additive collapse. LIMITED/NONE groups must not be merged into a
+ # semi-additive FULL group, as extra grain columns would corrupt MAX_BY/MIN_BY.
by_parent: dict[str, list[GrainGroup]] = defaultdict(list)
for gg in grain_groups:
by_parent[gg.parent_node.name].append(gg)
@@ -483,6 +644,11 @@ def merge_grain_groups(grain_groups: list[GrainGroup]) -> list[GrainGroup]:
if len(parent_groups) == 1:
# Only one group for this parent - no merge needed
merged_groups.append(parent_groups[0])
+ elif any(group.reaggregate_component_dimensions for group in parent_groups):
+ # Keep semi-additive groups isolated. Merging a protected-dimension
+ # group with another grain can add rows inside the protected bucket,
+ # corrupting the already-aggregated value for MAX_BY/MIN_BY.
+ merged_groups.extend(parent_groups)
else:
# Multiple groups for same parent - merge them
merged = _merge_parent_grain_groups(parent_groups)
@@ -520,17 +686,19 @@ def _merge_parent_grain_groups(groups: list[GrainGroup]) -> GrainGroup:
# Collect all components and track their original aggregabilities
all_components: list[tuple[Node, MetricComponent]] = []
component_aggregabilities: dict[str, Aggregability] = {}
+ reaggregate_component_dimensions: dict[str, str] = {}
for gg in groups:
for metric_node, component in gg.components:
all_components.append((metric_node, component))
# Track original aggregability for each component
component_aggregabilities[component.name] = gg.aggregability
+ reaggregate_component_dimensions.update(
+ gg.reaggregate_component_dimensions,
+ )
- # Carry over non-decomposable metrics from every contributing group.
- # Without this, merging a NONE group into a FULL/LIMITED neighbor
- # silently drops the non-decomposable metric expressions and the
- # response loses those metrics entirely.
+ # Carry over non-decomposable metrics from every contributing group to prevent them
+ # from being silently dropped when merging a NONE group into a FULL/LIMITED neighbor.
all_non_decomposable: list[DecomposedMetricInfo] = []
for gg in groups:
all_non_decomposable.extend(gg.non_decomposable_metrics)
@@ -559,5 +727,6 @@ def _merge_parent_grain_groups(groups: list[GrainGroup]) -> GrainGroup:
is_merged=True,
component_aggregabilities=component_aggregabilities,
grain_col_aliases=merged_grain_col_aliases,
+ reaggregate_component_dimensions=reaggregate_component_dimensions,
non_decomposable_metrics=all_non_decomposable,
)
diff --git a/datajunction-server/datajunction_server/construction/build_v3/loaders.py b/datajunction-server/datajunction_server/construction/build_v3/loaders.py
index 934bf069b..9c744c900 100644
--- a/datajunction-server/datajunction_server/construction/build_v3/loaders.py
+++ b/datajunction-server/datajunction_server/construction/build_v3/loaders.py
@@ -87,6 +87,7 @@ async def batch_load_nodes_with_dependencies(
NodeRevision.name,
NodeRevision.query,
NodeRevision.schema_,
+ NodeRevision.reaggregate,
NodeRevision.table,
),
# NOTE: don't noload Column.attributes — Columns are identity-
diff --git a/datajunction-server/datajunction_server/construction/build_v3/measures.py b/datajunction-server/datajunction_server/construction/build_v3/measures.py
index f6912654c..50ae7695c 100644
--- a/datajunction-server/datajunction_server/construction/build_v3/measures.py
+++ b/datajunction-server/datajunction_server/construction/build_v3/measures.py
@@ -28,6 +28,7 @@
from datajunction_server.construction.build_v3.decomposition import (
analyze_grain_groups,
build_component_expression,
+ build_merge_call,
merge_grain_groups,
)
from datajunction_server.construction.build_v3.dimensions import (
@@ -76,6 +77,7 @@
from datajunction_server.internal.scan_estimation import calculate_scan_estimate
from datajunction_server.models.decompose import Aggregability, MetricComponent
from datajunction_server.models.node_type import NodeType
+from datajunction_server.models.materialization import MaterializationTarget
from datajunction_server.sql.functions import function_registry
from datajunction_server.sql.parsing import ast
from datajunction_server.sql.parsing import types as ct
@@ -155,10 +157,51 @@ def _parse_type_string(type_str: str | None) -> ct.ColumnType | None:
return _TYPE_STRING_MAP.get(normalized)
+def _multi_argument_accumulate_types(
+ aggregation: str,
+ parent_node: Node | None,
+) -> list[ct.ColumnType] | None:
+ """
+ Argument types of an accumulate whose outermost call takes more than one.
+
+ Type inference feeds a component exactly one input type, which is right for
+ every aggregation that takes one argument -- including the templated ones,
+ whose outermost call is still a single-argument ``SUM``. A sketch breaks
+ that assumption: ``nflx_tdigest(latency_ms, CAST(200.0 AS DOUBLE))`` needs
+ two, so calling ``infer_type`` with one raises ``TypeError``, and the caller
+ quietly falls back to the metric's own type. That is how a struct-valued
+ sketch column comes to be recorded as ``double``.
+
+ Returns None whenever this does not apply or an argument cannot be typed,
+ so the single-argument path is left exactly as it was.
+ """
+ if parent_node is None:
+ return None
+ call = parse(f"SELECT {aggregation}").select.projection[0]
+ if not isinstance(call, ast.Function) or len(call.args) < 2:
+ return None
+
+ arg_types: list[ct.ColumnType] = []
+ for arg in call.args:
+ if isinstance(arg, ast.Column):
+ # Only the columns need the parent to resolve; literals and casts
+ # carry their own type without being bound to a table.
+ resolved = _parse_type_string(
+ get_column_type(parent_node, str(arg.alias_or_name.name)),
+ )
+ else:
+ resolved = arg.type
+ if resolved is None:
+ return None
+ arg_types.append(resolved)
+ return arg_types
+
+
def infer_component_type(
component: MetricComponent,
metric_type: str,
parent_node: Node | None = None,
+ materialization_target: MaterializationTarget | None = None,
) -> str:
"""
Infer the SQL type of a metric component based on its aggregation function.
@@ -175,6 +218,14 @@ def infer_component_type(
Returns:
The inferred SQL type string
"""
+ # A serialized column stores the converted type, and the family declares it.
+ # Checked first: inference below cannot see the conversion, and for a
+ # multi-argument sketch accumulate it falls back to `metric_type` entirely,
+ # so leaving this to inference records a type that makes the Druid
+ # aggregator lookup miss and drop the measure with no error.
+ if component.serializes_for(materialization_target) and component.serialize_type:
+ return component.serialize_type
+
if not component.aggregation:
return metric_type # pragma: no cover
@@ -199,17 +250,22 @@ def infer_component_type(
col_type_str = get_column_type(parent_node, component.expression)
input_type = _parse_type_string(col_type_str)
+ multi_arg_types = _multi_argument_accumulate_types(agg_str, parent_node)
+
try:
- if input_type:
+ if multi_arg_types is not None:
+ result_type = func_class.infer_type(*multi_arg_types)
+ elif input_type:
result_type = func_class.infer_type(input_type)
else: # pragma: no cover
# Fallback: try with a generic ColumnType
result_type = func_class.infer_type(ct.ColumnType("unknown", "unknown"))
- return str(result_type)
except (TypeError, NotImplementedError, AttributeError):
# Function may require more specific types - fall back to metric type
return metric_type
+ return str(result_type)
+
def _get_filter_column_name_for_dimension(
resolved_dim: ResolvedDimension,
@@ -1078,6 +1134,8 @@ def build_select_ast(
grain_columns: list[str] | None = None,
grain_col_aliases: dict[str, str] | None = None,
grain_col_specs: list[tuple[ast.Expression, str]] | None = None,
+ output_dimension_refs: set[str] | None = None,
+ internal_dimension_aliases: dict[str, str] | None = None,
filters: list[str] | None = None,
skip_aggregation: bool = False,
) -> tuple[ast.Query, list[str]]:
@@ -1096,6 +1154,9 @@ def build_select_ast(
the SQL alias to use in the generated SELECT. When provided,
the alias is taken from this dict (keyed by the raw expression
string from ``rule.level``).
+ output_dimension_refs: Optional set of dimension refs to expose as user-facing
+ dimensions. Other resolved dimensions are join/group-only.
+ internal_dimension_aliases: Dimension refs to project as private grain columns.
filters: Optional list of filter strings to apply as WHERE clause.
Filter strings can reference dimensions (e.g., "v3.product.category = 'Electronics'")
or local columns (e.g., "status = 'active'").
@@ -1119,16 +1180,29 @@ def build_select_ast(
spark_hints = _collect_spark_hints(resolved_dimensions, dim_aliases)
# Add dimension columns to projection
- # Filter-only dimensions are excluded from projection but included in GROUP BY
+ # Filter-only dimensions are excluded from projection but included in GROUP BY.
+ # Internal dimensions are projected under private aliases so the metrics layer
+ # can collapse semi-additive measures without exposing those dimensions.
+ internal_dimension_aliases = internal_dimension_aliases or {}
for resolved_dim in resolved_dimensions:
clean_alias = ctx.alias_registry.register(resolved_dim.original_ref)
if resolved_dim.original_ref in ctx.filter_dimensions:
continue
+ if (
+ output_dimension_refs is not None
+ and resolved_dim.original_ref not in output_dimension_refs
+ and resolved_dim.original_ref not in internal_dimension_aliases
+ ):
+ continue
+ output_alias = internal_dimension_aliases.get(
+ resolved_dim.original_ref,
+ clean_alias,
+ )
col_expr = build_dimension_col_expr(
resolved_dim,
main_alias,
dim_aliases,
- clean_alias,
+ output_alias,
ctx=ctx,
resolved_dimensions=resolved_dimensions,
parent_node_name=parent_node.name,
@@ -1690,6 +1764,7 @@ def build_grain_group_from_preagg(
select_items: list[ast.Aliasable | ast.Expression | ast.Column] = []
columns: list[ColumnMetadata] = []
component_aliases: dict[str, str] = {}
+ reaggregate_dimension_aliases: dict[str, str] = {}
metrics_covered: set[str] = set()
unique_components: list[MetricComponent] = []
seen_components: set[str] = set()
@@ -1787,6 +1862,14 @@ def build_grain_group_from_preagg(
),
)
+ for (
+ component_name,
+ dimension_ref,
+ ) in grain_group.reaggregate_component_dimensions.items():
+ reaggregate_dimension_aliases[component_name] = ctx.alias_registry.register(
+ dimension_ref,
+ )
+
# Add measure columns with re-aggregation (or grain columns if no merge func)
for metric_node, component in grain_group.components:
metrics_covered.add(metric_node.name)
@@ -1819,9 +1902,10 @@ def build_grain_group_from_preagg(
# If no merge function, output column directly (e.g., grain column for LIMITED)
# Otherwise, apply the merge function for re-aggregation
if component.merge:
- agg_expr = ast.Function(
- name=ast.Name(component.merge),
- args=[_preagg_column(scan_name, scan_alias)],
+ agg_expr = build_merge_call(
+ component.merge,
+ component.merge_args,
+ _preagg_column(scan_name, scan_alias),
)
aliased = ast.Alias(child=agg_expr, alias=ast.Name(output_alias))
select_items.append(aliased)
@@ -1947,6 +2031,7 @@ def build_grain_group_from_preagg(
metrics=list(metrics_covered),
parent_name=parent_node.name,
component_aliases=component_aliases,
+ reaggregate_dimension_aliases=reaggregate_dimension_aliases,
is_merged=grain_group.is_merged,
component_aggregabilities=grain_group.component_aggregabilities,
components=unique_components,
@@ -1960,6 +2045,7 @@ def build_grain_group_sql(
grain_group: GrainGroup,
resolved_dimensions: list[ResolvedDimension],
components_per_metric: dict[str, int],
+ output_dimension_refs: set[str] | None = None,
) -> GrainGroupSQL:
"""
Build SQL for a single grain group.
@@ -2012,6 +2098,21 @@ def build_grain_group_sql(
# Track mapping from component name to actual SQL alias
# This is needed for metrics SQL to correctly reference component columns
component_aliases: dict[str, str] = {}
+ reaggregate_dimension_aliases: dict[str, str] = {}
+ internal_dimension_aliases: dict[str, str] = {}
+
+ if output_dimension_refs:
+ for resolved_dim in resolved_dimensions:
+ if resolved_dim.original_ref in output_dimension_refs:
+ ctx.alias_registry.register(resolved_dim.original_ref)
+
+ for (
+ component_name,
+ dimension_ref,
+ ) in grain_group.reaggregate_component_dimensions.items():
+ dimension_alias = ctx.alias_registry.register(dimension_ref)
+ reaggregate_dimension_aliases[component_name] = dimension_alias
+ internal_dimension_aliases[dimension_ref] = dimension_alias
for metric_node, component in grain_group.components:
metrics_covered.add(metric_node.name)
@@ -2057,7 +2158,10 @@ def build_grain_group_sql(
# FULL: apply aggregation at finest grain, will be re-aggregated in final SELECT
# Always use component.name for consistency - no special case for single-component
component_alias = component.name
- expr_ast = build_component_expression(component)
+ expr_ast = build_component_expression(
+ component,
+ ctx.materialization_target,
+ )
component_expressions.append((component_alias, expr_ast))
component_metadata.append(
(component_alias, component, metric_node),
@@ -2076,7 +2180,7 @@ def build_grain_group_sql(
# Always use component.name for consistency - no special case for single-component
component_alias = component.name
- expr_ast = build_component_expression(component)
+ expr_ast = build_component_expression(component, ctx.materialization_target)
component_expressions.append((component_alias, expr_ast))
component_metadata.append((component_alias, component, metric_node))
@@ -2180,6 +2284,8 @@ def build_grain_group_sql(
resolved_dimensions=effective_resolved_dimensions,
parent_node=parent_node,
grain_columns=pass_through_columns,
+ output_dimension_refs=output_dimension_refs,
+ internal_dimension_aliases=internal_dimension_aliases,
filters=ctx.dimension_filters, # Use dimension_filters only (not metric_filters)
skip_aggregation=True, # Don't add GROUP BY
)
@@ -2203,6 +2309,8 @@ def build_grain_group_sql(
grain_columns=effective_grain_columns,
grain_col_aliases=grain_group.grain_col_aliases or None,
grain_col_specs=grain_col_specs, # already parsed above
+ output_dimension_refs=output_dimension_refs,
+ internal_dimension_aliases=internal_dimension_aliases,
filters=ctx.dimension_filters, # Use dimension_filters only (not metric_filters)
skip_aggregation=skip_agg,
)
@@ -2211,7 +2319,12 @@ def build_grain_group_sql(
columns_metadata = []
# Add dimension columns (skip filter-only dimensions as they're not in projection)
+ effective_output_dimension_refs = output_dimension_refs or {
+ resolved_dim.original_ref for resolved_dim in resolved_dimensions
+ }
for resolved_dim in resolved_dimensions:
+ if resolved_dim.original_ref not in effective_output_dimension_refs:
+ continue
# Skip filter-only dimensions from column metadata
if resolved_dim.original_ref in ctx.filter_dimensions:
continue
@@ -2268,6 +2381,30 @@ def build_grain_group_sql(
),
)
+ for dimension_ref, dimension_alias in internal_dimension_aliases.items():
+ internal_resolved_dim = next(
+ (dim for dim in resolved_dimensions if dim.original_ref == dimension_ref),
+ None,
+ )
+ if not internal_resolved_dim:
+ continue # pragma: no cover
+ dim_node = ctx.nodes.get(internal_resolved_dim.node_name)
+ col_type = (
+ get_column_type(parent_node, internal_resolved_dim.column_name)
+ if internal_resolved_dim.is_local
+ else get_column_type(dim_node, internal_resolved_dim.column_name)
+ if dim_node
+ else "string"
+ )
+ columns_metadata.append(
+ ColumnMetadata(
+ name=dimension_alias,
+ semantic_name=dimension_ref,
+ type=col_type,
+ semantic_type="dimension",
+ ),
+ )
+
# Add metric component columns
# All decomposed metrics are now treated as components - no special case for single-component
for comp_alias, component, metric_node in component_metadata:
@@ -2288,7 +2425,12 @@ def build_grain_group_sql(
ColumnMetadata(
name=ctx.alias_registry.get_alias(comp_alias) or comp_alias,
semantic_name=f"{metric_node.name}:{component.name}",
- type=infer_component_type(component, metric_type, parent_node),
+ type=infer_component_type(
+ component,
+ metric_type,
+ parent_node,
+ ctx.materialization_target,
+ ),
semantic_type="metric_component",
),
)
@@ -2313,6 +2455,8 @@ def build_grain_group_sql(
if grain_group.aggregability != Aggregability.NONE:
# FULL/LIMITED: dimensions are part of the grain
for resolved_dim in resolved_dimensions:
+ if resolved_dim.original_ref not in effective_output_dimension_refs:
+ continue
# Skip filter-only dimensions from grain
if resolved_dim.original_ref in ctx.filter_dimensions:
continue
@@ -2327,6 +2471,9 @@ def build_grain_group_sql(
for gc_alias in effective_grain_aliases:
if gc_alias not in full_grain: # pragma: no branch
full_grain.append(gc_alias)
+ for dimension_alias in internal_dimension_aliases.values():
+ if dimension_alias not in full_grain:
+ full_grain.append(dimension_alias)
# Sort for deterministic output
full_grain.sort()
@@ -2339,6 +2486,7 @@ def build_grain_group_sql(
metrics=list(metrics_covered),
parent_name=grain_group.parent_node.name,
component_aliases=component_aliases,
+ reaggregate_dimension_aliases=reaggregate_dimension_aliases,
is_merged=grain_group.is_merged,
component_aggregabilities=grain_group.component_aggregabilities,
components=unique_components,
@@ -2375,18 +2523,15 @@ def process_metric_group(
components_per_metric[decomposed.metric_node.name] = len(decomposed.components)
# Analyze grain groups - split by aggregability
- # Extract just the column names from dimensions for grain analysis
- dim_column_names = [parse_dimension_ref(d).column_name for d in ctx.dimensions]
- grain_groups = analyze_grain_groups(metric_group, dim_column_names)
+ output_dimensions = list(ctx.dimensions)
+ output_dimension_refs = set(output_dimensions)
+ grain_groups = analyze_grain_groups(metric_group, output_dimensions)
# Merge compatible grain groups from same parent into single CTEs
# This optimization reduces duplicate JOINs by outputting raw values
# at finest grain, with aggregations applied in final SELECT
grain_groups = merge_grain_groups(grain_groups)
- # Resolve dimensions (find join paths) - shared across grain groups
- resolved_dimensions = resolve_dimensions(ctx, parent_node)
-
# Build SQL for each grain group. Fan-out risk is flagged inside
# build_grain_group_sql, where the full set of emitted join paths is known.
grain_group_sqls: list[GrainGroupSQL] = []
@@ -2394,13 +2539,26 @@ def process_metric_group(
# Reset alias registry for each grain group to avoid conflicts
ctx.alias_registry = AliasRegistry()
ctx._table_alias_counter = 0
-
- grain_group_sql = build_grain_group_sql(
- ctx,
- grain_group,
- resolved_dimensions,
- components_per_metric,
- )
+ original_dimensions = ctx.dimensions
+ internal_dimensions = [
+ dimension_ref
+ for dimension_ref in dict.fromkeys(
+ grain_group.reaggregate_component_dimensions.values(),
+ )
+ if dimension_ref not in output_dimension_refs
+ ]
+ ctx.dimensions = output_dimensions + internal_dimensions
+ try:
+ resolved_dimensions = resolve_dimensions(ctx, parent_node)
+ grain_group_sql = build_grain_group_sql(
+ ctx,
+ grain_group,
+ resolved_dimensions,
+ components_per_metric,
+ output_dimension_refs=output_dimension_refs,
+ )
+ finally:
+ ctx.dimensions = original_dimensions
grain_group_sqls.append(grain_group_sql)
return grain_group_sqls
@@ -2514,6 +2672,33 @@ def find_parent_for_window_metric(
else:
return None, base_metrics # pragma: no cover
+ def find_base_metrics_for_window_parent(
+ metric_name: str,
+ visited: set[str],
+ ) -> set[str]:
+ """
+ Recursively find grain-group metrics needed to compute a window parent.
+
+ Window expressions may reference a derived metric directly, but the
+ window grain group still needs the underlying base metric components.
+ """
+ if metric_name in visited:
+ return set() # pragma: no cover
+ visited.add(metric_name)
+
+ for gg in existing_grain_groups:
+ if metric_name in gg.metrics:
+ return {metric_name}
+
+ grain_group_metrics: set[str] = set()
+ for parent_name in ctx.parent_map.get(metric_name, []):
+ parent_node = ctx.nodes.get(parent_name)
+ if parent_node and parent_node.type.value == "metric": # pragma: no branch
+ grain_group_metrics.update(
+ find_base_metrics_for_window_parent(parent_name, visited),
+ )
+ return grain_group_metrics
+
# Group window metrics by (ORDER BY grain, parent fact)
# This ensures window metrics from different facts are processed separately
# Key: (frozenset of grain cols, parent_name or "cross_fact")
@@ -2525,13 +2710,18 @@ def find_parent_for_window_metric(
parent_name, base_metrics = find_parent_for_window_metric(metric_name)
# Use "cross_fact" as a marker for cross-fact window metrics
parent_key = parent_name if parent_name else "cross_fact"
+ component_metrics = set()
+ for base_metric in base_metrics:
+ component_metrics.update(
+ find_base_metrics_for_window_parent(base_metric, set()),
+ )
group_key = (grain_key, parent_key)
if group_key not in grain_parent_to_metrics:
grain_parent_to_metrics[group_key] = []
grain_parent_to_base_metrics[group_key] = set()
grain_parent_to_metrics[group_key].append(metric_name)
- grain_parent_to_base_metrics[group_key].update(base_metrics)
+ grain_parent_to_base_metrics[group_key].update(component_metrics)
additional_grain_groups: list[GrainGroupSQL] = []
diff --git a/datajunction-server/datajunction_server/construction/build_v3/metrics.py b/datajunction-server/datajunction_server/construction/build_v3/metrics.py
index fb8cb3b28..31bc8eb8c 100644
--- a/datajunction-server/datajunction_server/construction/build_v3/metrics.py
+++ b/datajunction-server/datajunction_server/construction/build_v3/metrics.py
@@ -50,13 +50,159 @@
)
from datajunction_server.errors import DJInvalidInputException
from datajunction_server.models.decompose import Aggregability
+from datajunction_server.models.dialect import Dialect
from datajunction_server.models.node_type import NodeType
+from datajunction_server.models.reaggregate import ReaggregationFunction
from datajunction_server.sql.decompose import wrap_divisions_in_nullif
from datajunction_server.sql.parsing import ast
+from datajunction_server.sql.parsing.backends.antlr4 import parse
logger = logging.getLogger(__name__)
+def _build_reaggregate_collapse_expression(
+ collapse_function: ReaggregationFunction,
+ dialect: Dialect,
+ value_ref: ast.Expression,
+ protected_dim_ref: ast.Expression,
+) -> ast.Function:
+ """
+ Build the final aggregation for a semi-additive metric collapse.
+ """
+ if collapse_function == ReaggregationFunction.LAST_VALUE:
+ function_name = "LATEST_BY" if dialect == Dialect.DRUID else "MAX_BY"
+ return ast.Function(
+ ast.Name(function_name),
+ args=[value_ref, protected_dim_ref],
+ )
+ if collapse_function == ReaggregationFunction.FIRST_VALUE:
+ function_name = "EARLIEST_BY" if dialect == Dialect.DRUID else "MIN_BY"
+ return ast.Function(
+ ast.Name(function_name),
+ args=[value_ref, protected_dim_ref],
+ )
+ if collapse_function == ReaggregationFunction.MIN:
+ return ast.Function(ast.Name("MIN"), args=[value_ref])
+ if collapse_function == ReaggregationFunction.MAX:
+ return ast.Function(ast.Name("MAX"), args=[value_ref])
+
+ raise DJInvalidInputException(
+ f"Unsupported semi-additive collapse function: {collapse_function}",
+ )
+
+
+def _references_component(expr: ast.Node, component_name: str) -> bool:
+ """
+ Return whether an expression references a decomposed metric component.
+ """
+ for col in expr.find_all(ast.Column):
+ if col.name and col.name.name == component_name:
+ return True
+ return False
+
+
+def _replace_reaggregate_merge_expression(
+ combiner_ast: ast.Expression,
+ component_name: str,
+ merge_function: str,
+ collapse_expr: ast.Expression,
+) -> ast.Expression:
+ """
+ Replace the component's merge aggregate in a combiner with a collapse expr.
+
+ Semi-additive metrics still need the full decomposed combiner expression.
+ For example, ``SUM(balance) / 100`` should become
+ ``MAX_BY(balance_sum, date_id) / 100``, not just ``MAX_BY(...)``.
+ """
+ expr_ast = (
+ cast(ast.Expression, combiner_ast.child)
+ if isinstance(combiner_ast, ast.Alias)
+ else combiner_ast
+ )
+ target_function = merge_function.upper()
+
+ for func in list(expr_ast.find_all(ast.Function)):
+ if func.name.name.upper() == target_function and _references_component(
+ func,
+ component_name,
+ ):
+ if func is expr_ast:
+ return collapse_expr
+ if func.parent:
+ func.parent.replace(from_=func, to=collapse_expr)
+ return expr_ast
+
+ raise DJInvalidInputException(
+ "Unsupported semi-additive metric shape: could not find the component "
+ "merge expression in the metric combiner.",
+ )
+
+
+def _dimension_ref_base(ref: str) -> str:
+ """Return a dimension ref without its role suffix."""
+ return ref.split("[", 1)[0]
+
+
+def _dimension_ref_role(ref: str) -> str | None:
+ """Return the role suffix for a dimension ref, if any."""
+ if "[" not in ref:
+ return None
+ return ref.rsplit("[", 1)[1].rstrip("]")
+
+
+def _source_dimension_alias(ctx: BuildContext, dimension_ref: str) -> str | None:
+ """
+ Return the registered source column alias for a dimension ref.
+ """
+ alias = ctx.alias_registry.get_alias(dimension_ref)
+ if alias or _dimension_ref_role(dimension_ref) is not None:
+ return alias
+ return ctx.alias_registry.get_alias(_dimension_ref_base(dimension_ref))
+
+
+def _metric_refs_from_query(ctx: BuildContext, metric_name: str) -> set[str]:
+ """
+ Return metric refs used by a metric's query expression.
+ """
+ metric_node = ctx.nodes.get(metric_name)
+ if not metric_node:
+ return set() # pragma: no cover
+
+ if not metric_node.current or not metric_node.current.query:
+ return set() # pragma: no cover
+
+ query = parse(metric_node.current.query)
+ refs: set[str] = set()
+ for col in query.select.projection[0].find_all(ast.Column):
+ full_name = get_column_full_name(col)
+ if not full_name and hasattr(col, "identifier"):
+ full_name = col.identifier()
+ node = ctx.nodes.get(full_name) if full_name else None
+ if node and node.type == NodeType.METRIC:
+ refs.add(full_name)
+ return refs
+
+
+def _metric_parent_refs(ctx: BuildContext, metric_name: str) -> list[str]:
+ """
+ Return metric parents for a metric, using the parsed query as a fallback.
+ """
+ refs: list[str] = []
+ seen: set[str] = set()
+ for parent_name in ctx.parent_map.get(metric_name, []):
+ parent_node = ctx.nodes.get(parent_name)
+ if parent_node and parent_node.type == NodeType.METRIC:
+ refs.append(parent_name)
+ seen.add(parent_name)
+
+ for parent_name in sorted(_metric_refs_from_query(ctx, metric_name)):
+ if parent_name not in seen:
+ refs.append(parent_name)
+ seen.add(parent_name)
+
+ return refs
+
+
def classify_filters(
filters: list[str],
ctx: BuildContext,
@@ -207,6 +353,31 @@ def get_comp_aggregability(comp_name: str) -> Aggregability:
comp = decomposed.components[0]
orig_agg = get_comp_aggregability(comp.name)
+ if comp.rule.reaggregate and comp.name in gg.reaggregate_dimension_aliases:
+ _, col_name = comp_mappings[comp.name]
+ value_ref = make_column_ref(col_name, cte_alias)
+ protected_dim_ref = make_column_ref(
+ gg.reaggregate_dimension_aliases[comp.name],
+ cte_alias,
+ )
+ if not comp.merge: # pragma: no cover
+ raise DJInvalidInputException(
+ "Unsupported semi-additive metric shape: missing merge function.",
+ )
+ collapse_expr = _build_reaggregate_collapse_expression(
+ comp.rule.reaggregate.fn,
+ gg.dialect,
+ value_ref,
+ protected_dim_ref,
+ )
+ combiner_ast = parse(f"SELECT {decomposed.combiner}").select.projection[0]
+ return _replace_reaggregate_merge_expression(
+ cast(ast.Expression, combiner_ast),
+ comp.name,
+ comp.merge,
+ collapse_expr,
+ )
+
if orig_agg == Aggregability.LIMITED:
_, col_name = comp_mappings[comp.name]
col_ref = make_column_ref(col_name, cte_alias)
@@ -413,6 +584,41 @@ def collect_and_build_ctes(
return all_cte_asts, cte_aliases
+def _validate_reaggregate_base_group_join_safety(
+ grain_groups: list[GrainGroupSQL],
+) -> None:
+ """
+ Fail closed for final joins that can duplicate non-idempotent aggregates.
+
+ Semi-additive groups retain the protected dimension as private grain, so a
+ final join by only the requested output dimensions can fan out other CTEs.
+ LIMITED groups that were pre-aggregated to one row per output dimension use
+ MAX passthrough in the final select and are safe. Reaggregate groups are
+ idempotent under duplicate protected-dimension rows for their collapse.
+ Other separate groups must be rejected unless they were merged into the
+ protected grain earlier.
+ """
+ if len(grain_groups) <= 1 or not any(
+ gg.reaggregate_dimension_aliases for gg in grain_groups
+ ):
+ return
+
+ unsafe_groups = [
+ gg.parent_name
+ for gg in grain_groups
+ if not gg.reaggregate_dimension_aliases
+ and not (gg.aggregability == Aggregability.LIMITED and gg.is_pre_aggregated)
+ ]
+ if not unsafe_groups:
+ return
+
+ raise DJInvalidInputException(
+ "Semi-additive live SQL with multiple base grain groups is not "
+ "supported when another group can be fanned out before final "
+ f"aggregation. Unsafe parents: {sorted(set(unsafe_groups))}.",
+ )
+
+
def get_dimension_types(
grain_groups: list[GrainGroupSQL],
) -> dict[str, str]:
@@ -1230,11 +1436,7 @@ def build_window_agg_cte_from_grain_group(
# Find the base metrics that the window metrics reference
base_metrics_needed: set[str] = set()
for window_metric_name in window_grain_group.window_metrics_served:
- parent_names = ctx.parent_map.get(window_metric_name, [])
- for parent_name in parent_names:
- parent_node = ctx.nodes.get(parent_name)
- if parent_node and parent_node.type == NodeType.METRIC: # pragma: no branch
- base_metrics_needed.add(parent_name)
+ base_metrics_needed.update(_metric_parent_refs(ctx, window_metric_name))
def is_derived_metric(metric_name: str) -> bool:
"""Check if a metric is derived (has metric parents, no direct components)."""
@@ -1242,11 +1444,8 @@ def is_derived_metric(metric_name: str) -> bool:
if not decomposed or not isinstance(decomposed, DecomposedMetricInfo):
return False # pragma: no cover
# Derived metrics have parent metrics but no direct components in grain groups
- parent_metrics = ctx.parent_map.get(metric_name, [])
- has_metric_parents = any(
- ctx.nodes.get(p) and ctx.nodes.get(p).type == NodeType.METRIC # type: ignore
- for p in parent_metrics
- )
+ parent_metrics = _metric_parent_refs(ctx, metric_name)
+ has_metric_parents = bool(parent_metrics)
return has_metric_parents and not decomposed.components
def get_metric_aggregation_expr(
@@ -1284,7 +1483,7 @@ def get_metric_aggregation_expr(
# Try to find matching metric
parent_metric_name = None
- for parent_name in ctx.parent_map.get(metric_name, []):
+ for parent_name in _metric_parent_refs(ctx, metric_name):
parent_short = get_short_name(parent_name)
if parent_short == short_col_name or col_name.endswith(parent_name):
parent_metric_name = parent_name
@@ -1300,21 +1499,11 @@ def get_metric_aggregation_expr(
# Swap the column node with the parent's expression
col_node.swap(parent_expr)
else:
- # Base metric: replace component references with CTE column refs
- # Use base_grain_group's component_aliases since that's the source CTE
- for col_node in combiner_ast.find_all(ast.Column):
- col_full_name = (
- col_node.identifier() if hasattr(col_node, "identifier") else ""
- )
- # Check if this matches a component alias
- for (
- comp_name,
- comp_alias,
- ) in base_grain_group.component_aliases.items(): # pragma: no branch
- if col_full_name == comp_name or col_full_name.endswith(comp_alias):
- col_node.name = ast.Name(comp_alias)
- col_node._table = ast.Table(ast.Name(source_cte_alias))
- break
+ combiner_ast, _ = build_base_metric_expression(
+ decomposed,
+ source_cte_alias,
+ base_grain_group,
+ )
return combiner_ast
@@ -1326,13 +1515,13 @@ def get_metric_aggregation_expr(
# Get aggregation expression (handles both base and derived metrics)
combiner_ast = get_metric_aggregation_expr(base_metric_name, set())
- if not combiner_ast:
+ if combiner_ast is None:
continue # pragma: no cover
short_name = get_short_name(base_metric_name)
- aliased = combiner_ast.set_alias(ast.Name(short_name)) # type: ignore
- aliased.set_as(True)
- projection.append(aliased)
+ metric_alias = ast.Alias(child=combiner_ast, alias=ast.Name(short_name))
+ metric_alias.set_as(True)
+ projection.append(metric_alias)
# Build FROM clause
from_clause = ast.From(
@@ -1394,62 +1583,107 @@ def build_window_agg_cte_from_base_metrics(
# Find the base metrics that the window metrics reference
base_metrics_needed: set[str] = set()
for window_metric_name in window_grain_group.window_metrics_served:
- parent_names = ctx.parent_map.get(window_metric_name, [])
- for parent_name in parent_names:
- parent_node = ctx.nodes.get(parent_name)
- if parent_node and parent_node.type == NodeType.METRIC: # pragma: no branch
- base_metrics_needed.add(parent_name)
-
- # Build reaggregation expressions for each base metric
- # Since we're selecting from base_metrics, we reference the metric columns directly
+ base_metrics_needed.update(_metric_parent_refs(ctx, window_metric_name))
+
+ window_dimension_aliases = {
+ col.name
+ for col in window_grain_group.columns
+ if col.semantic_type == "dimension"
+ }
+
+ def build_base_metric_reaggregation_expr(
+ metric_name: str,
+ decomposed: DecomposedMetricInfo,
+ ) -> ast.Expression:
+ """
+ Build a reaggregation expression for a base metric from base_metrics.
+ """
+ short_name = get_short_name(metric_name)
+ metric_ref = make_column_ref(short_name, base_metrics_cte_alias)
+
+ if len(decomposed.components) == 1:
+ comp = decomposed.components[0]
+ if comp.rule.reaggregate:
+ protected_dim_alias = _source_dimension_alias(
+ ctx,
+ comp.rule.reaggregate.dimension,
+ )
+ if (
+ protected_dim_alias
+ and protected_dim_alias not in window_dimension_aliases
+ ):
+ return _build_reaggregate_collapse_expression(
+ comp.rule.reaggregate.fn,
+ ctx.dialect,
+ metric_ref,
+ make_column_ref(protected_dim_alias, base_metrics_cte_alias),
+ )
+
+ if decomposed.aggregability == Aggregability.NONE: # pragma: no cover
+ # NONE: non-additive (like AVG), need to recompute.
+ # For AVG, we need the raw sum and count, but those are in components.
+ # For now, just use the column directly (window function will handle it).
+ return metric_ref
+
+ # base_metrics exposes metric columns, not raw component/grain columns.
+ # Reaggregate leaf metrics from those projected metric values.
+ return ast.Function(ast.Name("SUM"), args=[metric_ref])
+
+ def build_metric_reaggregation_expr(
+ metric_name: str,
+ visited: set[str],
+ ) -> ast.Expression | None:
+ """
+ Build a reaggregation expression for base or derived metrics.
+ """
+ if metric_name in visited:
+ return None # pragma: no cover
+ visited.add(metric_name)
+
+ parent_refs = _metric_parent_refs(ctx, metric_name)
+ if not parent_refs:
+ decomposed = decomposed_metrics.get(metric_name)
+ if not decomposed:
+ return None # pragma: no cover
+ return build_base_metric_reaggregation_expr(metric_name, decomposed)
+
+ metric_node = ctx.nodes.get(metric_name)
+ if not metric_node:
+ return None # pragma: no cover
+ if not metric_node.current or not metric_node.current.query:
+ return None # pragma: no cover
+ original_query = parse(metric_node.current.query)
+ expr_ast = deepcopy(original_query.select.projection[0])
+ if isinstance(expr_ast, ast.Alias):
+ expr_ast = expr_ast.child # pragma: no cover
+
+ for col_node in list(expr_ast.find_all(ast.Column)):
+ parent_name = get_column_full_name(col_node)
+ if not parent_name and hasattr(col_node, "identifier"):
+ parent_name = col_node.identifier()
+ if not parent_name or parent_name not in parent_refs:
+ continue
+
+ parent_expr = build_metric_reaggregation_expr(
+ parent_name,
+ visited.copy(),
+ )
+ if parent_expr is not None:
+ col_node.swap(parent_expr)
+
+ wrap_divisions_in_nullif(cast(ast.Expression, expr_ast))
+ return expr_ast # type: ignore
+
+ # Build reaggregation expressions for each parent metric.
for base_metric_name in sorted(base_metrics_needed):
- decomposed = decomposed_metrics.get(base_metric_name)
- if not decomposed:
+ agg_expr = build_metric_reaggregation_expr(base_metric_name, set())
+ if agg_expr is None:
continue # pragma: no cover
short_name = get_short_name(base_metric_name)
-
- # For non-additive metrics (COUNT DISTINCT), we need COUNT(DISTINCT grain_col)
- # For additive metrics, we can SUM/AVG the pre-computed column
- if decomposed.aggregability == Aggregability.LIMITED:
- # LIMITED: needs COUNT DISTINCT at this grain
- # Find the grain column for COUNT DISTINCT
- if decomposed.components and decomposed.components[0].rule.level:
- grain_col = decomposed.components[0].rule.level[0]
- # The grain column is in base_metrics as a dimension column
- # Actually, for COUNT DISTINCT, the raw grain column should be in
- # the base grain group, not base_metrics. We need to reference
- # the original grain column from base_metrics.
- # For now, use the metric column - this works because base_metrics
- # computes COUNT DISTINCT at the fine grain, and we re-compute at coarser grain
- col_ref = make_column_ref(grain_col, base_metrics_cte_alias)
- agg_expr = ast.Function(
- ast.Name("COUNT"),
- args=[col_ref],
- quantifier=ast.SetQuantifier.Distinct,
- )
- else: # pragma: no cover
- # Fallback: SUM the metric column
- col_ref = make_column_ref(short_name, base_metrics_cte_alias)
- agg_expr = ast.Function(ast.Name("SUM"), args=[col_ref])
- elif decomposed.aggregability == Aggregability.FULL: # pragma: no cover
- # FULL: additive metric, use SUM
- col_ref = make_column_ref(short_name, base_metrics_cte_alias)
- agg_expr = ast.Function(ast.Name("SUM"), args=[col_ref])
- elif decomposed.aggregability == Aggregability.NONE: # pragma: no cover
- # NONE: non-additive (like AVG), need to recompute
- # For AVG, we need the raw sum and count, but those are in components
- # For now, just use the column directly (window function will handle it)
- col_ref = make_column_ref(short_name, base_metrics_cte_alias)
- agg_expr = col_ref # type: ignore
- else: # pragma: no cover
- # Default: SUM
- col_ref = make_column_ref(short_name, base_metrics_cte_alias)
- agg_expr = ast.Function(ast.Name("SUM"), args=[col_ref])
-
- aliased = agg_expr.set_alias(ast.Name(short_name)) # type: ignore
- aliased.set_as(True)
- projection.append(aliased)
+ metric_alias = ast.Alias(child=agg_expr, alias=ast.Name(short_name))
+ metric_alias.set_as(True)
+ projection.append(metric_alias)
# Build FROM clause - just base_metrics
from_clause = ast.From(
@@ -1760,6 +1994,7 @@ def generate_metrics_sql(
base_grain_groups,
skip_pre_agg=will_have_base_metrics_cte,
)
+ _validate_reaggregate_base_group_join_safety(base_grain_groups)
# Build dimension info and projection
# Filter out filter-only dimensions (they're needed for WHERE but not output)
diff --git a/datajunction-server/datajunction_server/construction/build_v3/preagg_matcher.py b/datajunction-server/datajunction_server/construction/build_v3/preagg_matcher.py
index 575a22901..4cdcdd311 100644
--- a/datajunction-server/datajunction_server/construction/build_v3/preagg_matcher.py
+++ b/datajunction-server/datajunction_server/construction/build_v3/preagg_matcher.py
@@ -19,6 +19,8 @@
from datajunction_server.database.preaggregation import (
PreAggregation,
compute_expression_hash,
+ get_measure_identities,
+ measure_identity_token,
)
from datajunction_server.errors import DJInvalidInputException
from datajunction_server.models.decompose import Aggregability, MetricComponent
@@ -42,10 +44,9 @@
def get_required_measure_identities(
grain_group: GrainGroup,
-) -> set[tuple[str, str]]:
+) -> set[str]:
"""
- Identity of each measure a grain group needs: (expression hash, Phase-1
- aggregation).
+ Identity token of each measure a grain group needs.
Matching is name-independent, so ``expr_hash`` stands in for identity -- but
it covers the expression alone, making ``SUM(x)`` and ``MAX(x)`` hash alike
@@ -55,16 +56,40 @@ def get_required_measure_identities(
Phase-1 ``aggregation`` rather than ``merge``, because merge is not unique --
COUNT accumulates with COUNT but merges with SUM.
+
+ Built through ``measure_identity_token`` rather than assembled here, so the
+ definition of identity lives in one place -- it also folds in sketch tuning
+ parameters, which a locally-built tuple would silently omit.
"""
return {
- (
+ measure_identity_token(
compute_expression_hash(component.expression),
- component.normalized_aggregation,
+ component.aggregation,
+ component.params,
)
for _, component in grain_group.components
}
+def required_reaggregate_grain(
+ ctx: BuildContext,
+ node_rev_id: int,
+ grain_group: GrainGroup,
+) -> set[str]:
+ """
+ Protected dimensions a pre-agg must retain for final semi-additive collapse.
+
+ ``reaggregate_component_dimensions`` is populated only when the requested
+ output grain omitted the protected dimension. In that case a pre-agg at the
+ output grain would have already collapsed away the key needed by MAX_BY /
+ MIN_BY / min / max in the metrics layer, so it is not substitutable.
+ """
+ return {
+ canonical_dimension_ref(ctx, node_rev_id, dimension_ref)
+ for dimension_ref in grain_group.reaggregate_component_dimensions.values()
+ }
+
+
def canonical_dimension_ref(
ctx: BuildContext,
node_rev_id: int,
@@ -256,6 +281,12 @@ def find_matching_preagg(
if not required_measures:
return None
+ reaggregate_grain = required_reaggregate_grain(
+ ctx,
+ node_rev_id,
+ grain_group,
+ )
+
requested_grain_set = {
canonical_dimension_ref(ctx, node_rev_id, rdim.original_ref)
for rdim in resolved_dimensions
@@ -282,6 +313,14 @@ def find_matching_preagg(
for ref in (preagg.grain_columns or [])
}
+ if not reaggregate_grain.issubset(preagg_grain_set):
+ logger.debug(
+ f"[BuildV3] Pre-agg {preagg.id} grain {preagg_grain_set} "
+ f"doesn't retain semi-additive protected grain "
+ f"{reaggregate_grain}",
+ )
+ continue
+
# Check grain compatibility. For additive measures the pre-agg may be at
# the same or a finer grain (roll-up allowed → subset match). For
# non-additive measures the pre-agg grain must exactly match the request.
@@ -315,13 +354,9 @@ def find_matching_preagg(
continue
join_back = coverage
- # Coverage check on (expression hash, Phase-1 aggregation) -- see
- # get_required_measure_identities for why the aggregation is part of it.
- preagg_measures = {
- (measure.expr_hash, measure.normalized_aggregation)
- for measure in preagg.measures
- if measure.expr_hash
- }
+ # Coverage check on measure identity -- see
+ # get_required_measure_identities for what that comprises.
+ preagg_measures = get_measure_identities(preagg.measures)
if not required_measures.issubset(preagg_measures):
logger.debug(
f"[BuildV3] Pre-agg {preagg.id} measures {preagg_measures} "
@@ -373,13 +408,21 @@ def get_preagg_measure_column(
Returns:
Column name in the pre-agg, or None if not found
"""
- target = (
+ target = measure_identity_token(
compute_expression_hash(component.expression),
- component.normalized_aggregation,
+ component.aggregation,
+ component.params,
)
for measure in preagg.measures:
- if (measure.expr_hash, measure.normalized_aggregation) == target:
+ if (
+ measure_identity_token(
+ measure.expr_hash,
+ measure.aggregation,
+ measure.params,
+ )
+ == target
+ ):
# Externally-registered pre-aggs bind the measure to a physical
# column name that may differ from the DJ component name.
return measure.source_column or measure.name
diff --git a/datajunction-server/datajunction_server/construction/build_v3/types.py b/datajunction-server/datajunction_server/construction/build_v3/types.py
index faac2e34c..2d3f6523c 100644
--- a/datajunction-server/datajunction_server/construction/build_v3/types.py
+++ b/datajunction-server/datajunction_server/construction/build_v3/types.py
@@ -13,6 +13,7 @@
from datajunction_server.errors import DJInvalidInputException, DJWarning
from datajunction_server.models.decompose import Aggregability, MetricComponent
from datajunction_server.models.dialect import Dialect
+from datajunction_server.models.materialization import MaterializationTarget
from datajunction_server.models.node_type import NodeType
from datajunction_server.sql.parsing import ast
from datajunction_server.sql.parsing.ast import to_sql
@@ -43,6 +44,9 @@ class BuildContext:
dimensions: list[str]
filters: list[str] = field(default_factory=list)
dialect: Dialect = Dialect.SPARK
+ # Set only when this build is producing a measures table for materialization.
+ # None for query-time builds, which never serialize.
+ materialization_target: MaterializationTarget | None = None
alias_registry: AliasRegistry = field(default_factory=AliasRegistry)
# Filter classification (populated early in setup)
@@ -316,6 +320,10 @@ class GrainGroupSQL:
# Used by metrics SQL to correctly reference component columns
component_aliases: dict[str, str] = field(default_factory=dict)
+ # Active semi-additive components that need collapse in the metrics layer.
+ # Maps component.name -> protected dimension column alias emitted by this CTE.
+ reaggregate_dimension_aliases: dict[str, str] = field(default_factory=dict)
+
# Merge tracking: when True, aggregations happen in final SELECT, not in CTE
is_merged: bool = False
@@ -743,6 +751,10 @@ class GrainGroup:
# between decompose.py component identifiers and measures.py SQL aliases.
grain_col_aliases: dict[str, str] = field(default_factory=dict)
+ # Active semi-additive components that require an internal protected dimension
+ # in this grain group. Maps component.name -> protected dimension ref.
+ reaggregate_component_dimensions: dict[str, str] = field(default_factory=dict)
+
# Non-decomposable metrics that couldn't be broken into components
# These need their raw metric expression applied in the final SELECT
non_decomposable_metrics: list[DecomposedMetricInfo] = field(default_factory=list)
diff --git a/datajunction-server/datajunction_server/database/measure.py b/datajunction-server/datajunction_server/database/measure.py
index bfdb81c11..32d0bc079 100644
--- a/datajunction-server/datajunction_server/database/measure.py
+++ b/datajunction-server/datajunction_server/database/measure.py
@@ -119,6 +119,12 @@ class FrozenMeasure(Base):
# How to aggregate the resolved expression (e.g., SUM, COUNT, AVG)
aggregation: Mapped[str]
+ # Tuning parameters for `aggregation`, for sketch-backed measures whose
+ # function name alone doesn't pin down the accumulate (a t-digest still needs
+ # a compression). Part of the measure's identity: a sketch frozen at one
+ # accuracy must not be reused for a metric asking for another.
+ params: Mapped[dict | None] = mapped_column(JSON, nullable=True)
+
# Additivity or rollup rule - tells the planner if this measure can be summed,
# needs special handling, or is non-additive.
rule: Mapped[MeasureAggregationRule] = mapped_column(MeasureAggregationRuleType)
diff --git a/datajunction-server/datajunction_server/database/node.py b/datajunction-server/datajunction_server/database/node.py
index d93acc753..942bc2556 100644
--- a/datajunction-server/datajunction_server/database/node.py
+++ b/datajunction-server/datajunction_server/database/node.py
@@ -88,6 +88,7 @@
from datajunction_server.models.custom_metadata import CustomMetadataFilter
from datajunction_server.models.node_type import NodeType
from datajunction_server.models.partition import PartitionType
+from datajunction_server.models.reaggregate import parse_reaggregate_spec
from datajunction_server.models.unit import (
AtomicUnit,
CompoundUnit,
@@ -678,6 +679,7 @@ async def to_spec(self, session: AsyncSession) -> NodeSpec:
)
extra_kwargs.update(
required_dimensions=required_dimensions_spec,
+ reaggregate=parse_reaggregate_spec(self.current.reaggregate),
direction=self.current.metric_metadata.direction
if self.current.metric_metadata
else None,
@@ -1721,6 +1723,14 @@ class NodeRevision(
uselist=False,
)
+ # Declares how a metric should roll up across dimensions.
+ # Stored as JSON here and validated at the API/deployment boundaries.
+ reaggregate: Mapped[dict[str, Any] | None] = mapped_column(
+ JSON,
+ nullable=True,
+ default=None,
+ )
+
# Filters that are always applied when generating SQL for a cube node
cube_filters: Mapped[list[str] | None] = mapped_column(
JSON,
@@ -2067,6 +2077,12 @@ def extra_validation(self) -> None:
"bound dimensions which are only for metrics.",
)
+ if self.type != NodeType.METRIC and self.reaggregate:
+ raise DJInvalidInputException(
+ f"Node {self.name} of type {self.type} cannot have "
+ "reaggregate settings which are only for metrics.",
+ )
+
if self.type == NodeType.METRIC:
self.check_metric()
diff --git a/datajunction-server/datajunction_server/database/preaggregation.py b/datajunction-server/datajunction_server/database/preaggregation.py
index 2572ff418..ce37033ad 100644
--- a/datajunction-server/datajunction_server/database/preaggregation.py
+++ b/datajunction-server/datajunction_server/database/preaggregation.py
@@ -6,6 +6,7 @@
import json
from datetime import UTC, datetime
from functools import partial
+from typing import Any
import sqlalchemy as sa
from sqlalchemy import (
@@ -80,16 +81,64 @@ def compute_grain_group_hash(
return hashlib.md5(content.encode()).hexdigest()
-def measure_identity_token(expr_hash: str, aggregation: str | None) -> str:
+def canonical_params(params: dict[str, Any] | None) -> str:
+ """
+ Stable string form of a component's tuning parameters, for identity.
+
+ Keys are sorted and numeric values normalized, so ``{"compression": 200}``
+ and ``{"compression": 200.0}`` cannot yield two identities for one sketch.
+ """
+ if not params:
+ return ""
+
+ def norm(value: Any) -> Any:
+ # 200 and 200.0 are the same compression; don't let the literal's
+ # spelling fork the identity. (bools are not floats, so they pass
+ # through untouched.)
+ if isinstance(value, float) and value.is_integer():
+ return int(value)
+ return value
+
+ # `default=str` so an exotic value can't raise from inside pre-agg matching.
+ # Params arrive as parsed YAML/JSON, so this should be unreachable; an opaque
+ # TypeError deep in the matcher is a bad way to find out otherwise.
+ return json.dumps(
+ {key: norm(params[key]) for key in sorted(params)},
+ sort_keys=True,
+ separators=(",", ":"),
+ default=str,
+ )
+
+
+def measure_identity_token(
+ expr_hash: str,
+ aggregation: str | None,
+ params: dict[str, Any] | None = None,
+) -> str:
"""
Canonical token identifying one measure: expression hash plus Phase-1
- aggregation.
+ aggregation, plus tuning parameters when the aggregation takes them.
The hash alone is not an identity -- ``SUM(x)`` and ``MAX(x)`` hash alike --
and a partial is reusable only by a metric that accumulates the same way.
Everything comparing or hashing measures goes through this.
+
+ Parameters extend that reasoning to sketches: a t-digest accumulated at
+ ``compression=100`` is not interchangeable with one at ``compression=1000``,
+ though both are ``nflx_tdigest_agg`` over the same expression. The segment is
+ appended **only** when params are present, so tokens for the overwhelming
+ majority of components -- plain ``SUM``, ``COUNT``, ``MAX`` -- are byte-identical
+ to what this returned before params existed. Stored ``preagg_hash`` values are
+ built from these tokens, so that stability is a compatibility requirement, not
+ a nicety.
+
+ Note the quantile *fractions* a metric asks for are deliberately not here. One
+ sketch column serves p50, p95 and p99; folding fractions in would fragment it
+ into three identical columns.
"""
- return f"{expr_hash}:{(aggregation or '').strip().upper()}"
+ base = f"{expr_hash}:{(aggregation or '').strip().upper()}"
+ suffix = canonical_params(params)
+ return f"{base}:{suffix}" if suffix else base
def get_measure_identities(measures: list[PreAggMeasure]) -> set[str]:
@@ -103,7 +152,7 @@ def get_measure_identities(measures: list[PreAggMeasure]) -> set[str]:
Set of identity tokens
"""
return {
- measure_identity_token(m.expr_hash, m.aggregation)
+ measure_identity_token(m.expr_hash, m.aggregation, m.params)
for m in measures
if m.expr_hash
}
@@ -173,36 +222,7 @@ def compute_preagg_hash(
class PreAggregation(Base):
- """
- First-class pre-aggregation entity that can be shared across cubes.
-
- A pre-aggregation represents a materialized grouping of measures at a specific grain,
- enabling efficient metric calculations by pre-computing aggregations.
-
- Pre-aggregations are ALWAYS created by DJ (via /preaggs/plan endpoint) from
- metrics + dimensions. Users never manually construct them - this ensures
- consistency between DJ-managed (Flow A) and user-managed (Flow B) materialization.
-
- Key concepts:
- - `node_revision`: The specific node revision this pre-agg is based on
- - `grain_columns`: Fully qualified dimension references that define the aggregation level
- - `measures`: Full MetricComponent info for matching and re-aggregation
- - `sql`: The generated SQL for materializing this pre-agg
- - `grain_group_hash`: Hash of (node_revision_id + sorted(grain_columns)) for grouping
-
- Measure format (MetricComponent):
- - name: Column name in materialized table
- - expression: The raw SQL expression
- - expr_hash: Hash of expression for identity matching
- - aggregation: Phase 1 function (e.g., "SUM")
- - merge: Phase 2 re-aggregation function
- - rule: Aggregation rules (type, level)
-
- Availability tracking:
- - Materialization status is tracked via AvailabilityState
- - Flow A: DJ's query service posts availability after materialization
- - Flow B: User's query service posts to /preaggs/{id}/availability/
- """
+ """First-class pre-aggregation entity that can be shared across cubes."""
__tablename__ = "pre_aggregation"
@@ -441,33 +461,7 @@ async def find_matching(
grain_columns: list[str],
measure_identities: set[str],
) -> PreAggregation | None:
- """
- Find the row this exact declaration already occupies, if any.
-
- This is the upsert's identity check -- "is this the same declaration I am
- about to write?" -- so it matches the uniqueness key exactly: same
- revision and grain (via ``grain_group_hash``) and the SAME set of measure
- identities, not merely a covering one.
-
- Covering was wrong here, in two compounding ways. Callers replace the
- matched row's contents wholesale, so a narrow declaration could match a
- wider pre-agg and silently strip measures off it, breaking routing for
- whatever metric depended on the dropped ones. And ``preagg_hash`` is
- UNIQUE over exactly ``(node_revision_id, grain_columns,
- measure_identities)`` and frozen at insert, so a covering match could
- hand back a row whose stored hash no longer described its own contents.
- A declaration that genuinely differs now gets its own row instead.
-
- Identity tokens rather than bare expression hashes, because the latter
- made SUM- and MAX-backed pre-aggs look like one row, so registering
- either silently overwrote the other.
-
- Contrast ``find_latest_for_node``, which asks the other question -- "what
- did this declaration look like before?" -- and does want covering.
-
- Returns:
- Matching PreAggregation if found, None otherwise
- """
+ """Find the row this exact declaration already occupies, if any."""
grain_group_hash = compute_grain_group_hash(node_revision_id, grain_columns)
candidates = await cls.get_by_grain_group_hash(session, grain_group_hash)
diff --git a/datajunction-server/datajunction_server/internal/client.py b/datajunction-server/datajunction_server/internal/client.py
index 7964b4cd2..37c9d24ed 100644
--- a/datajunction-server/datajunction_server/internal/client.py
+++ b/datajunction-server/datajunction_server/internal/client.py
@@ -192,6 +192,11 @@ async def python_client_create_node(
"required_dimensions": [ # type: ignore
col.name for col in node.current.required_dimensions
],
+ **(
+ {"reaggregate": node.current.reaggregate} # type: ignore
+ if node.current.reaggregate
+ else {}
+ ),
**(
{
"direction": ( # type: ignore
diff --git a/datajunction-server/datajunction_server/internal/cube_materializations.py b/datajunction-server/datajunction_server/internal/cube_materializations.py
index f5531ff9d..aa529c30b 100644
--- a/datajunction-server/datajunction_server/internal/cube_materializations.py
+++ b/datajunction-server/datajunction_server/internal/cube_materializations.py
@@ -24,7 +24,10 @@
UpsertCubeMaterialization,
)
from datajunction_server.models.dialect import Dialect
-from datajunction_server.models.materialization import MaterializationStrategy
+from datajunction_server.models.materialization import (
+ MaterializationStrategy,
+ MaterializationTarget,
+)
from datajunction_server.models.node_type import NodeNameVersion
from datajunction_server.models.partition import Granularity
from datajunction_server.models.query import ColumnMetadata
@@ -371,6 +374,11 @@ async def build_cube_materialization(
filters=(current_revision.cube_filters or []) + extra_filters,
dialect=Dialect.SPARK,
use_materialized=True,
+ # Measures SQL runs in Spark for every target, so the dialect above
+ # cannot say where the output is headed. Druid is the only cube target
+ # (`UpsertCubeMaterialization.job` is Literal["druid_cube"]), and this is
+ # the one caller that knows it.
+ materialization_target=MaterializationTarget.DRUID,
)
measures_queries = sorted(
[
diff --git a/datajunction-server/datajunction_server/internal/deployment/orchestrator.py b/datajunction-server/datajunction_server/internal/deployment/orchestrator.py
index adbb15ee3..365c712f3 100644
--- a/datajunction-server/datajunction_server/internal/deployment/orchestrator.py
+++ b/datajunction-server/datajunction_server/internal/deployment/orchestrator.py
@@ -122,6 +122,7 @@
NodeStatus,
NodeType,
)
+from datajunction_server.models.reaggregate import dump_reaggregate_spec
from datajunction_server.models.unit import (
AtomicUnit,
CompoundUnit,
@@ -5367,6 +5368,9 @@ async def _create_node_revision(
dependency_nodes,
)
new_revision.required_dimensions = matched_columns
+ new_revision.reaggregate = dump_reaggregate_spec(
+ metric_spec.rendered_reaggregate,
+ )
return new_revision
def _resolve_metric_unit(
diff --git a/datajunction-server/datajunction_server/internal/deployment/validation.py b/datajunction-server/datajunction_server/internal/deployment/validation.py
index d6d65f3bf..92c7369cd 100644
--- a/datajunction-server/datajunction_server/internal/deployment/validation.py
+++ b/datajunction-server/datajunction_server/internal/deployment/validation.py
@@ -29,6 +29,9 @@
)
from datajunction_server.models.dimensionlink import JoinType
from datajunction_server.models.node import NodeStatus, NodeType
+from datajunction_server.models.reaggregate import (
+ unsupported_dimension_reaggregate_functions,
+)
from datajunction_server.sql.dag import get_dimensions
from datajunction_server.sql.parsing.ast import fast_parse_mode
from datajunction_server.sql.parsing.backends.antlr4 import ast, parse_rule
@@ -153,7 +156,7 @@ async def validate(self, node_specs: list[NodeSpec]) -> list[NodeValidationResul
specs_needing_parse.append(spec)
spec_indices.append(i)
- # Pre-fetch dimension nodes for required_dimensions validation
+ # Pre-fetch dimension nodes for metric dimension-reference validation
await self._prefetch_required_dimension_nodes(specs_needing_parse)
# Pre-fetch dimension sets for cross-fact derived metric validation
@@ -520,6 +523,10 @@ def validate_query_node(
if req_dim_error is not None:
errors.append(req_dim_error)
+ reaggregate_error = self._check_reaggregate_dimension(spec)
+ if reaggregate_error is not None:
+ errors.append(reaggregate_error)
+
cross_fact_error = self._check_cross_fact_dimensions(spec)
if cross_fact_error is not None:
errors.append(cross_fact_error) # pragma: no cover
@@ -655,9 +662,9 @@ async def _prefetch_required_dimension_nodes(
specs: list[NodeSpec],
) -> None:
"""
- Collect all dimension node names referenced via required_dimensions across the
- batch, fetch any not already in dependency_nodes in a single DB query, and
- store the combined map in self._all_dim_nodes.
+ Collect all dimension node names referenced via metric-only dimension
+ declarations across the batch, fetch any not already in dependency_nodes in a
+ single DB query, and store the combined map in self._all_dim_nodes.
"""
req_dim_node_names: set[str] = set()
for spec in specs:
@@ -667,6 +674,12 @@ async def _prefetch_required_dimension_nodes(
if SEPARATOR in req_dim:
dim_node_name = req_dim.rsplit(SEPARATOR, 1)[0]
req_dim_node_names.add(dim_node_name)
+ reaggregate = getattr(spec, "rendered_reaggregate", None)
+ if reaggregate:
+ for rule in reaggregate.rules:
+ if SEPARATOR in rule.dimension:
+ dim_node_name = rule.dimension.rsplit(SEPARATOR, 1)[0]
+ req_dim_node_names.add(dim_node_name)
self._all_dim_nodes = dict(self.context.dependency_nodes)
@@ -780,6 +793,54 @@ def _check_required_dimensions(self, spec: NodeSpec) -> DJError | None:
debug={"invalid_required_dimensions": list(invalid)},
)
+ def _check_reaggregate_dimension(self, spec: NodeSpec) -> DJError | None:
+ """
+ Validate that reaggregate rule dimensions resolve to real columns.
+ """
+ reaggregate = getattr(spec, "rendered_reaggregate", None)
+ if not reaggregate or not reaggregate.rules:
+ return None
+
+ invalid_functions = unsupported_dimension_reaggregate_functions(reaggregate)
+ if invalid_functions:
+ return DJError(
+ code=ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION,
+ message=(
+ "Node definition contains unsupported dimension "
+ "reaggregate function(s)."
+ ),
+ debug={
+ "invalid_reaggregate_functions": invalid_functions,
+ },
+ )
+
+ dep_names = self.context.node_graph.get(spec.rendered_name, [])
+ parent_columns = [
+ col
+ for dep_name in dep_names
+ for dep_node in [self.context.dependency_nodes.get(dep_name)]
+ if dep_node and dep_node.current
+ for col in dep_node.current.columns
+ ]
+
+ invalid, _ = _resolve_required_dimensions(
+ [rule.dimension for rule in reaggregate.rules],
+ parent_columns,
+ self._all_dim_nodes,
+ )
+
+ if not invalid:
+ return None
+
+ return DJError(
+ code=ErrorCode.INVALID_COLUMN,
+ message=(
+ "Node definition contains references to columns as "
+ "reaggregate dimensions that are not on parent nodes."
+ ),
+ debug={"invalid_reaggregate_dimensions": list(invalid)},
+ )
+
async def _prefetch_metric_dimensions(
self,
specs: list[NodeSpec],
diff --git a/datajunction-server/datajunction_server/internal/materializations.py b/datajunction-server/datajunction_server/internal/materializations.py
index eafc57440..0535a215e 100644
--- a/datajunction-server/datajunction_server/internal/materializations.py
+++ b/datajunction-server/datajunction_server/internal/materializations.py
@@ -475,36 +475,6 @@ async def reconcile_declared_materializations(
Returns what each declared block resolved to, and the materializations the
blocks together superseded.
-
- A cube may declare more than one block -- typically an `incremental_time` build
- for freshness beside a periodic `full` rebuild that corrects late-arriving data
- -- and each is reconciled the same way. They cannot collide: the row name is
- derived from job, strategy and partition, the job is fixed for a cube and the
- strategies are unique by validation, so every declared block owns a row of its
- own.
-
- Mirrors what `POST /nodes/{name}/materialization/` does -- build the config, then
- update the row of the same name in place rather than inserting a second one,
- since `(name, node_revision_id)` is unique -- with one deliberate difference. The
- endpoint decides "unchanged" on `config` alone, but `schedule` and `strategy` are
- columns rather than config keys, so by that test a schedule-only edit compares
- equal and is silently dropped. Rescheduling is the whole point of a declared
- block, so all three are compared here.
-
- The declared blocks describe *the* materializations for their cube, so every
- other active row on the revision is deactivated. The rule is "any active row
- whose name no block built" rather than "any row of the same job type", because a
- cube's materializations are all writing one Druid datasource and a full rebuild
- replaces that datasource wholesale -- so a legacy `druid_measures_cube` row is
- just as much a competing writer as a second `druid_cube` row, and matching on job
- type would leave it running. The name is what carries the difference: it is
- derived from job, strategy and partition, so declaring a strategy the cube was
- not already materialized with builds a differently named row, and without this
- the cube ends up with two live workflows deleting each other's data.
-
- A cube planner row is the one thing left alone. It writes a datasource of its own
- and DJ cannot rebuild it from a declared block, so superseding it would stop a
- workflow nothing here can replace.
"""
# Snapshotted before the builds, each of which sets the new materialization's
# backref and so appends it to this very collection -- searching afterwards would
@@ -584,27 +554,7 @@ async def swap_cube_materializations(
) -> CubeMaterializationSwap | None:
"""
Rebuild a cube's materializations against a new revision and retire the old ones.
-
- Materializations belong to a single `NodeRevision` and availability is scoped to
- the revision encoded in the materialized table name, so without this a new cube
- revision -- including a metadata-only one -- has neither: the cube silently falls
- back to live queries while the superseded revision's workflow keeps posting
- availability for a table built from the old definition. Every new revision
- therefore swaps, no matter how insignificant the change was.
-
- Rebuilt rather than copied: a stored config embeds the cube version plus combiner
- SQL and a Druid spec derived from the old definition, so the new revision goes
- back through `create_new_materialization`. The old materialization is the default
- source of the user's intent because it is often the only record of it -- a cube
- materialized through the UI has no YAML to read it from. A cube that does declare
- `materialization:` passes it as `declared`, which wins, so a push that edits both
- a metric and the schedule rebuilds with the new schedule rather than the old.
-
- `previous_table_usable` is the caller's answer to whether the superseded
- revision's materialized table is still valid (`is_non_trivial_cube_change`
- inverted), recorded on the history event so an operator can tell whether the
- rebuild can adopt the existing data or needs a fresh build and backfill.
-
+
Touches only DJ-side state, and returns the query service work still owed --
`None` when the cube had nothing materialized and there is no work at all. The
caller commits and then hands the result to `apply_cube_materialization_swap`, so
@@ -650,18 +600,8 @@ async def swap_cube_materializations(
try:
upsert = _upsert_from_materialization(materialization)
if declared and isinstance(upsert, UpsertCubeMaterialization):
- # The declared block wins over the recovered intent: a cube that
- # declares `materialization:` has its config in the repo, so a
- # rebuild triggered by the same deploy must build what the YAML now
- # says. Only cube materializations can be declared; anything else
- # keeps what was recovered.
- #
- # Which block, when the cube declares several: the one naming the
- # strategy this row was built with, since that is what identifies a
- # declared entry. A row whose strategy nothing declares falls to the
- # first block, which is what a cube declaring exactly one has always
- # done -- and is the only sensible answer, since a rebuild has to
- # produce something for a row that is being retired either way.
+ # The declared block wins over the recovered intent.
+ # If there are several, match by strategy, otherwise use the first.
block = next(
(
candidate
diff --git a/datajunction-server/datajunction_server/internal/namespaces.py b/datajunction-server/datajunction_server/internal/namespaces.py
index d3eb5f10b..32bae088f 100644
--- a/datajunction-server/datajunction_server/internal/namespaces.py
+++ b/datajunction-server/datajunction_server/internal/namespaces.py
@@ -1229,6 +1229,7 @@ def _metric_project_config(node: Node, namespace_requested: str) -> dict:
"query": node.current.query,
"tags": [tag.name for tag in node.tags],
"required_dimensions": [dim.name for dim in node.current.required_dimensions],
+ "reaggregate": node.current.reaggregate,
"direction": (
node.current.metric_metadata.direction.name.lower()
if node.current.metric_metadata and node.current.metric_metadata.direction
@@ -1642,6 +1643,14 @@ async def get_node_specs_for_export(
)
for required_dim in metric_spec.required_dimensions
]
+ if metric_spec.reaggregate:
+ for rule in metric_spec.reaggregate.rules:
+ rule.dimension = _inject_prefix_for_cube_ref(
+ rule.dimension,
+ namespace,
+ parent_namespace,
+ namespace_suffixes,
+ )
if node_spec.node_type in (
NodeType.SOURCE,
NodeType.TRANSFORM,
diff --git a/datajunction-server/datajunction_server/internal/nodes.py b/datajunction-server/datajunction_server/internal/nodes.py
index 9f81e399b..13432e3e3 100644
--- a/datajunction-server/datajunction_server/internal/nodes.py
+++ b/datajunction-server/datajunction_server/internal/nodes.py
@@ -65,6 +65,7 @@
)
from datajunction_server.internal.access.authorization.context import AuthContext
from datajunction_server.internal.caching.interface import Cache
+from datajunction_server.internal.custom_metadata import validate_custom_metadata
from datajunction_server.internal.history import ActivityType, EntityType
from datajunction_server.internal.materializations import (
apply_cube_materialization_swap,
@@ -95,6 +96,12 @@
UpsertCubeMaterialization,
principal_refs,
)
+from datajunction_server.models.decompose import (
+ AggregationRule as DecomposeAggregationRule,
+)
+from datajunction_server.models.decompose import (
+ MetricComponent,
+)
from datajunction_server.models.deployment import (
ChangeTier,
CubeSpec,
@@ -127,6 +134,7 @@
)
from datajunction_server.models.node_type import NodeType
from datajunction_server.models.query import QueryCreate
+from datajunction_server.models.reaggregate import dump_reaggregate_spec
from datajunction_server.models.table_metadata import TableMetadata, TableOwner
from datajunction_server.service_clients import QueryServiceClient
from datajunction_server.sql.dag import (
@@ -140,7 +148,6 @@
from datajunction_server.sql.parsing.ast import CompileContext
from datajunction_server.sql.parsing.backends.antlr4 import parse, parse_rule
from datajunction_server.typing import UTCDatetime
-from datajunction_server.internal.custom_metadata import validate_custom_metadata
from datajunction_server.utils import (
SEPARATOR,
Version,
@@ -585,6 +592,11 @@ async def create_node_revision(
query=data.query,
mode=data.mode,
required_dimensions=data.required_dimensions or [],
+ reaggregate=(
+ dump_reaggregate_spec(data.reaggregate)
+ if node_type == NodeType.METRIC
+ else None
+ ),
created_by_id=current_user.id,
custom_metadata=data.custom_metadata,
)
@@ -886,12 +898,15 @@ async def _derive_frozen_measures_impl(
session=session,
name=measure.name,
)
+ if frozen_measure:
+ _raise_if_frozen_measure_conflicts(frozen_measure, measure)
if not frozen_measure and measure.aggregation:
frozen_measure = FrozenMeasure(
name=measure.name,
upstream_revision_id=upstream_revision_id,
expression=measure.expression,
aggregation=measure.aggregation,
+ params=measure.params,
rule=measure.rule,
used_by_node_revisions=[],
)
@@ -1019,14 +1034,51 @@ async def derive_frozen_measures_bulk(
upstream_revision_id=upstream_revision_id,
expression=measure.expression,
aggregation=measure.aggregation,
- rule=measure.rule,
+ params=measure.params,
+ rule=_frozen_measure_rule(measure.rule),
used_by_node_revisions=[],
)
session.add(frozen_measure)
fm_by_name[measure.name] = frozen_measure
+ else:
+ _raise_if_frozen_measure_conflicts(frozen_measure, measure)
frozen_measure.used_by_node_revisions.append(rev)
+def _aggregation_rule_identity(rule: DecomposeAggregationRule) -> dict[str, Any]:
+ """Return the stable JSON shape used for frozen-measure rule comparison."""
+ return rule.model_dump(mode="json", exclude_none=True, exclude={"reaggregate"})
+
+
+def _frozen_measure_rule(rule: DecomposeAggregationRule) -> DecomposeAggregationRule:
+ """
+ Return the metric-independent rule persisted on a shared frozen measure.
+ """
+ return rule.model_copy(update={"reaggregate": None})
+
+
+def _raise_if_frozen_measure_conflicts(
+ frozen_measure: FrozenMeasure,
+ measure: MetricComponent,
+) -> None:
+ """
+ Prevent component-name collisions from reusing a different frozen measure.
+ """
+ if (
+ frozen_measure.expression == measure.expression
+ and frozen_measure.aggregation == measure.aggregation
+ and (frozen_measure.params or None) == (measure.params or None)
+ and _aggregation_rule_identity(frozen_measure.rule)
+ == _aggregation_rule_identity(measure.rule)
+ ):
+ return
+
+ raise DJInvalidInputException(
+ f"Frozen measure `{measure.name}` already exists with a different "
+ "expression, aggregation, aggregation rule, or tuning parameters.",
+ )
+
+
async def save_node(
session: AsyncSession,
node_revision: NodeRevision,
@@ -1128,6 +1180,7 @@ async def copy_to_new_node(
table=old_revision.table,
required_dimensions=list(old_revision.required_dimensions),
metric_metadata=old_revision.metric_metadata,
+ reaggregate=old_revision.reaggregate,
cube_elements=list(old_revision.cube_elements),
cube_filters=old_revision.cube_filters,
status=old_revision.status,
@@ -1866,6 +1919,7 @@ async def cube_metric_component_identities(
FrozenMeasure.name,
FrozenMeasure.expression,
FrozenMeasure.aggregation,
+ FrozenMeasure.params,
)
.select_from(NodeRevisionFrozenMeasure)
.join(
@@ -1881,8 +1935,12 @@ async def cube_metric_component_identities(
rows = (await session.execute(statement)).all()
return {
f"{metric_name}:{component_name}:"
- + measure_identity_token(compute_expression_hash(expression), aggregation)
- for metric_name, component_name, expression, aggregation in rows
+ + measure_identity_token(
+ compute_expression_hash(expression),
+ aggregation,
+ params,
+ )
+ for metric_name, component_name, expression, aggregation, params in rows
}
@@ -2392,14 +2450,7 @@ async def _propagate_update_downstream(
cache.delete(upstream_cache_key)
if downstream.type == NodeType.CUBE:
- # Any tier rebuilds, and the churn is deliberate. Narrowing this by
- # comparing the upstream's resolved columns was rejected: a query edit
- # can move a filter, a join or a CASE threshold while leaving every
- # column and type identical, and each changes every row the cube serves.
- # Nothing short of reading the SQL tells those apart, so a changed query
- # makes anything built from it suspect. Only NONE is skipped, the one
- # case where DJ knows nothing material happened.
- #
+ # Any tier rebuilds except NONE.
# A rebuild can fail, and one cube's failure must not cost the remaining
# downstreams theirs.
if change_tier is not ChangeTier.NONE:
@@ -2505,6 +2556,7 @@ def copy_existing_node_revision(old_revision: NodeRevision, current_user: User):
status=old_revision.status,
required_dimensions=list(old_revision.required_dimensions),
metric_metadata=old_revision.metric_metadata,
+ reaggregate=old_revision.reaggregate,
dimension_links=[
DimensionLink(
dimension_id=link.dimension_id,
@@ -2727,8 +2779,20 @@ async def create_new_revision_from_existing(
and {col.name for col in old_revision.required_dimensions}
!= set(data.required_dimensions)
)
+ reaggregate_was_set = bool(
+ data and "reaggregate" in data.model_fields_set,
+ )
+ reaggregate_changes = (
+ reaggregate_was_set
+ and old_revision.reaggregate
+ != dump_reaggregate_spec(data.reaggregate if data else None)
+ )
major_changes = (
- query_changes or column_changes or pk_changes or required_dim_changes
+ query_changes
+ or column_changes
+ or pk_changes
+ or required_dim_changes
+ or reaggregate_changes
)
# If nothing has changed, do not create the new node revision
@@ -2778,6 +2842,13 @@ async def create_new_revision_from_existing(
if data and data.metric_metadata
else old_revision.metric_metadata
),
+ reaggregate=(
+ dump_reaggregate_spec(data.reaggregate)
+ if data and data.reaggregate is not None
+ else None
+ if reaggregate_was_set
+ else old_revision.reaggregate
+ ),
dimension_links=[
DimensionLink(
dimension_id=link.dimension_id,
@@ -4256,17 +4327,6 @@ async def revalidate_node(
]
# A tier rather than a version, so propagation can hand the same value to
# `bump_version` for downstream cubes.
- #
- # Any column change is major. An addition looks harmless -- nothing could
- # already reference a column that did not exist -- but the query produced it,
- # and a query edit can move a filter or a join while leaving the rest of the
- # projection identical. Demoting additions to MINOR buys nothing anyway: the
- # cube rebuild below skips only NONE, so a minor bump rebuilds all the same.
- #
- # `order_fixed` earns no tier. DJ filling in a missing projection index is its
- # own bookkeeping, not a change to the node, so a revision would describe
- # nothing and any tier above NONE would rebuild every cube below. It is applied
- # to the current revision in place instead, below.
change_tier = fold_change_tiers(
[
ChangeTier.MAJOR
diff --git a/datajunction-server/datajunction_server/internal/preaggregations.py b/datajunction-server/datajunction_server/internal/preaggregations.py
index 956053fe6..e1347e9d1 100644
--- a/datajunction-server/datajunction_server/internal/preaggregations.py
+++ b/datajunction-server/datajunction_server/internal/preaggregations.py
@@ -28,6 +28,7 @@
compute_grain_group_hash,
compute_preagg_hash,
get_measure_identities,
+ measure_identity_token,
)
from datajunction_server.errors import DJInvalidInputException
from datajunction_server.models.decompose import PreAggMeasure
@@ -195,7 +196,7 @@ async def register_external_preaggregations(
# identity -- (expression hash, Phase-1 aggregation) -- to the declared
# column. Keying on the hash alone would collapse SUM(x) and MAX(x),
# silently discarding one metric's declared column.
- measure_identity_to_column: dict[tuple[str, str], str] = {}
+ measure_identity_to_column: dict[str, str] = {}
for metric_name, physical_column in measure_columns.items():
node = await Node.get_by_name(
session,
@@ -224,9 +225,10 @@ async def register_external_preaggregations(
# is_measure guarantees exactly one component.
component = components[0]
measure_identity_to_column[
- (
+ measure_identity_token(
compute_expression_hash(component.expression),
- component.normalized_aggregation,
+ component.aggregation,
+ component.params,
)
] = physical_column
@@ -311,7 +313,11 @@ async def register_external_preaggregations(
grain_measures: list[PreAggMeasure] = []
for component in grain_group.components:
expr_hash = compute_expression_hash(component.expression)
- identity = (expr_hash, component.normalized_aggregation)
+ identity = measure_identity_token(
+ expr_hash,
+ component.aggregation,
+ component.params,
+ )
if identity not in measure_identity_to_column:
raise DJInvalidInputException(
message=(
diff --git a/datajunction-server/datajunction_server/internal/sql.py b/datajunction-server/datajunction_server/internal/sql.py
index 26689d09e..97644851e 100644
--- a/datajunction-server/datajunction_server/internal/sql.py
+++ b/datajunction-server/datajunction_server/internal/sql.py
@@ -242,6 +242,7 @@ async def generate_metrics_sql(
matched_cube,
dimensions,
merged_filters,
+ metrics,
)
# Auto-resolve dialect if not explicitly provided.
diff --git a/datajunction-server/datajunction_server/internal/validation.py b/datajunction-server/datajunction_server/internal/validation.py
index 6eb1af905..55df63941 100644
--- a/datajunction-server/datajunction_server/internal/validation.py
+++ b/datajunction-server/datajunction_server/internal/validation.py
@@ -23,6 +23,10 @@
from datajunction_server.models.base import labelize
from datajunction_server.models.node import NodeRevisionBase, NodeStatus
from datajunction_server.models.node_type import NodeType
+from datajunction_server.models.reaggregate import (
+ parse_reaggregate_spec,
+ unsupported_dimension_reaggregate_functions,
+)
from datajunction_server.sql.parsing import ast
from datajunction_server.sql.parsing.backends.antlr4 import SqlSyntaxError, parse
from datajunction_server.sql.parsing.backends.exceptions import DJParseException
@@ -357,11 +361,34 @@ async def validate_node_data(
parent_columns,
)
node_validator.required_dimensions = matched_bound_columns
+ reaggregate_spec = parse_reaggregate_spec(validated_node.reaggregate)
+ invalid_reaggregate_dimensions: set[str] = set()
+ invalid_reaggregate_functions: list[str] = []
+ if reaggregate_spec and reaggregate_spec.rules:
+ (
+ invalid_reaggregate_dimensions,
+ _,
+ ) = await find_required_dimensions(
+ session,
+ [rule.dimension for rule in reaggregate_spec.rules],
+ parent_columns,
+ )
+ invalid_reaggregate_functions = unsupported_dimension_reaggregate_functions(
+ reaggregate_spec,
+ )
except MissingGreenlet:
invalid_required_dimensions = set()
+ invalid_reaggregate_dimensions = set()
+ invalid_reaggregate_functions = []
node_validator.required_dimensions = []
- if missing_parents_map or type_inference_failures or invalid_required_dimensions:
+ if (
+ missing_parents_map
+ or type_inference_failures
+ or invalid_required_dimensions
+ or invalid_reaggregate_dimensions
+ or invalid_reaggregate_functions
+ ):
# update status
node_validator.status = NodeStatus.INVALID
# build errors
@@ -410,10 +437,46 @@ async def validate_node_data(
if invalid_required_dimensions
else []
)
+ invalid_reaggregate_dimensions_error = (
+ [
+ DJError(
+ code=ErrorCode.INVALID_COLUMN,
+ message=(
+ "Node definition contains references to columns as "
+ "reaggregate dimensions that are not on parent nodes."
+ ),
+ debug={
+ "invalid_reaggregate_dimensions": list(
+ invalid_reaggregate_dimensions,
+ ),
+ },
+ ),
+ ]
+ if invalid_reaggregate_dimensions
+ else []
+ )
+ invalid_reaggregate_functions_error = (
+ [
+ DJError(
+ code=ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION,
+ message=(
+ "Node definition contains unsupported dimension "
+ "reaggregate function(s)."
+ ),
+ debug={
+ "invalid_reaggregate_functions": invalid_reaggregate_functions,
+ },
+ ),
+ ]
+ if invalid_reaggregate_functions
+ else []
+ )
errors = (
missing_parents_error
+ type_inference_error
+ invalid_required_dimensions_error
+ + invalid_reaggregate_dimensions_error
+ + invalid_reaggregate_functions_error
)
node_validator.errors.extend(errors)
@@ -705,10 +768,30 @@ async def validate_node_data_v2(
parent_columns,
)
node_validator.required_dimensions = matched_bound_columns
+ reaggregate_spec = parse_reaggregate_spec(validated_node.reaggregate)
+ invalid_reaggregate_dimensions: set[str] = set()
+ invalid_reaggregate_functions: list[str] = []
+ if reaggregate_spec and reaggregate_spec.rules:
+ (
+ invalid_reaggregate_dimensions,
+ _,
+ ) = await find_required_dimensions(
+ session,
+ [rule.dimension for rule in reaggregate_spec.rules],
+ parent_columns,
+ )
+ invalid_reaggregate_functions = unsupported_dimension_reaggregate_functions(
+ reaggregate_spec,
+ )
# --- Step 12: final error assembly for missing parents + invalid required
# dims (matches legacy code shapes).
- if node_validator.missing_parents_map or invalid_required_dimensions:
+ if (
+ node_validator.missing_parents_map
+ or invalid_required_dimensions
+ or invalid_reaggregate_dimensions
+ or invalid_reaggregate_functions
+ ):
node_validator.status = NodeStatus.INVALID
if node_validator.missing_parents_map:
node_validator.errors.append(
@@ -740,6 +823,34 @@ async def validate_node_data_v2(
},
),
)
+ if invalid_reaggregate_dimensions:
+ node_validator.errors.append(
+ DJError(
+ code=ErrorCode.INVALID_COLUMN,
+ message=(
+ "Node definition contains references to columns as "
+ "reaggregate dimensions that are not on parent nodes."
+ ),
+ debug={
+ "invalid_reaggregate_dimensions": list(
+ invalid_reaggregate_dimensions,
+ ),
+ },
+ ),
+ )
+ if invalid_reaggregate_functions:
+ node_validator.errors.append(
+ DJError(
+ code=ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION,
+ message=(
+ "Node definition contains unsupported dimension "
+ "reaggregate function(s)."
+ ),
+ debug={
+ "invalid_reaggregate_functions": invalid_reaggregate_functions,
+ },
+ ),
+ )
return node_validator
diff --git a/datajunction-server/datajunction_server/models/cube_materialization.py b/datajunction-server/datajunction_server/models/cube_materialization.py
index 1e12ebb40..23abfbd2b 100644
--- a/datajunction-server/datajunction_server/models/cube_materialization.py
+++ b/datajunction-server/datajunction_server/models/cube_materialization.py
@@ -16,11 +16,11 @@
)
from datajunction_server.models.materialization import (
DEFAULT_CUBE_RETENTION,
- DRUID_AGG_MAPPING,
CoverageSpec,
MaterializationJobTypeEnum,
MaterializationStrategy,
SparkSpec,
+ get_druid_aggregator_spec,
)
from datajunction_server.models.node_type import NodeNameVersion
from datajunction_server.models.partition import Granularity
@@ -385,22 +385,19 @@ def metrics_spec(self) -> list[dict[str, Any]]:
Returns the Druid metrics spec for ingestion
"""
column_mapping = {col.name: col.type for col in self.columns} # type: ignore
- return [
- {
- "fieldName": measure.name,
- "name": measure.name,
- "type": DRUID_AGG_MAPPING[
- (column_mapping[measure.name], measure.aggregation.lower())
- ],
- }
- for measure in self.measures
- if measure.aggregation
- and (
- column_mapping.get(measure.name),
- measure.aggregation.lower(),
+ specs = (
+ get_druid_aggregator_spec(
+ column_name=measure.name,
+ column_type=column_mapping.get(measure.name),
+ aggregation=measure.aggregation,
+ merge=measure.merge,
+ params=measure.params,
)
- in DRUID_AGG_MAPPING
- ]
+ for measure in self.measures
+ )
+ # Unmappable measures are omitted, as they always have been here -- the
+ # cube API substitutes longSum instead. See get_druid_aggregator_spec.
+ return [spec for spec in specs if spec is not None]
@computed_field # type: ignore[misc]
@property
diff --git a/datajunction-server/datajunction_server/models/decompose.py b/datajunction-server/datajunction_server/models/decompose.py
index 5e5655838..f8966434d 100644
--- a/datajunction-server/datajunction_server/models/decompose.py
+++ b/datajunction-server/datajunction_server/models/decompose.py
@@ -9,9 +9,13 @@
- DecomposedMetric: A metric broken into components + combiner expression
"""
-from pydantic import BaseModel
+from typing import Any
+
+from pydantic import BaseModel, Field
from datajunction_server.enum import StrEnum
+from datajunction_server.models.materialization import MaterializationTarget
+from datajunction_server.models.reaggregate import DimensionReaggregateRule
class Aggregability(StrEnum):
@@ -54,6 +58,7 @@ class AggregationRule(BaseModel):
type: Aggregability = Aggregability.NONE
level: list[str] | None = None
+ reaggregate: DimensionReaggregateRule | None = None
class MetricComponent(BaseModel):
@@ -84,6 +89,10 @@ class MetricComponent(BaseModel):
{} placeholder ("SUM(POWER({}, 2))").
merge: The function name for combining pre-aggregated values (Phase 2).
rule: Aggregation rules defining how/when the component can be aggregated.
+
+ The sketch-backed fields added alongside these -- params, merge_args,
+ serialize, serialize_targets, serialize_type -- are documented inline
+ below, where the reasoning sits next to the declaration.
"""
name: str
@@ -96,6 +105,47 @@ class MetricComponent(BaseModel):
# ("order_id"); for complex expressions it is component.name so the identifier stays
# valid and consistent with what decompose.py computed.
grain_alias: str | None = None
+ # Tuning parameters for `aggregation`/`merge`, for sketch-backed components
+ # whose function name alone doesn't pin down the accumulate (a t-digest still
+ # needs a compression, a KLL a `k`). Part of measure identity -- see
+ # `measure_identity_token` -- because a sketch built at one accuracy must not
+ # satisfy a query asking for another.
+ params: dict[str, Any] | None = None
+ # Fixed arguments appended after the column in the Phase 2 merge call, as SQL
+ # literals: `nflx_tdigest_agg(col, 200.0)` is merge="nflx_tdigest_agg" plus
+ # merge_args=["200.0"]. `merge` stays a bare function name because two things
+ # match on it -- the Druid aggregator lookup key and the semi-additive rewrite
+ # in `_replace_reaggregate_merge_expression` -- so the tuning cannot be folded
+ # into it as a template. Rendered from `params` by the decomposition; `params`
+ # remains the identity record, this is the emission form.
+ merge_args: list[str] = Field(default_factory=list)
+
+ # Conversion applied when writing this component to a materialized table for
+ # a target listed in `serialize_targets` -- see `ComponentDef.serialize`. The
+ # value is a template expanded against the accumulated expression.
+ serialize: str | None = None
+ # A list rather than a set: this model is dumped straight to JSON in cube
+ # materialization configs, and `json.dumps` cannot encode a set. StrEnum
+ # members are fine -- they subclass `str`.
+ serialize_targets: list[MaterializationTarget] = Field(default_factory=list)
+ # Column type `serialize` produces. See `ComponentDef.serialize_type`.
+ serialize_type: str | None = None
+
+ def serializes_for(self, materialization_target: Any | None) -> bool:
+ """
+ Whether this component converts its accumulated value for `target`.
+
+ Two things key off this and they must agree: the SQL that writes the
+ column, and the type recorded for it. If they disagree the measures
+ table holds one representation while the catalog claims another, and the
+ Druid aggregator lookup -- which keys on the column type -- silently
+ finds nothing and drops the measure.
+ """
+ return bool(
+ self.serialize
+ and materialization_target is not None
+ and materialization_target in self.serialize_targets,
+ )
@property
def normalized_aggregation(self) -> str:
diff --git a/datajunction-server/datajunction_server/models/deployment.py b/datajunction-server/datajunction_server/models/deployment.py
index 1336b6aa7..67ac67fbe 100644
--- a/datajunction-server/datajunction_server/models/deployment.py
+++ b/datajunction-server/datajunction_server/models/deployment.py
@@ -40,6 +40,7 @@
NodeType,
)
from datajunction_server.models.partition import Granularity, PartitionType
+from datajunction_server.models.reaggregate import ReaggregateSpec
from datajunction_server.models.unit import (
Unit,
legacy_unit_to_structured,
@@ -200,37 +201,10 @@ class HierarchySpec(NamespacedSpec):
class PreAggSpec(NamespacedSpec):
"""
Specification for an externally-built pre-aggregation table adopted at deploy
- time (equivalent to POST /preaggs/register). ``name`` is a stable handle used
- for reconciliation and availability callbacks. Metric/dimension references may
- use ``${prefix}`` or be fully qualified; they are rendered against the
- deployment namespace.
-
- Every metric and every dimension is declared together with the physical
- column of the external table that holds it, as a map::
-
- metrics:
- ${prefix}paid_members: paid_members_sum
- dimensions:
- ${prefix}country_dim.country_iso: country
- ${prefix}date_dim.utc_date: utc_date
-
- Both maps require a value for every key -- including a dimension whose
- physical column happens to match its DJ column name, which is written out
- rather than left empty. An optional value would make the map not really a
- mapping, a trailing colon is easy to write by accident, and spelling the
- physical name out documents the table in the file that declares it.
-
- What goes under ``metrics`` are the measures the table stores. A derived
- metric (a ratio of two others, say) is not listed and cannot be: it has no
- column of its own. It is covered anyway, because both registration and
- query-time matching work on decomposed measure identities rather than metric
- names, so any metric that decomposes into the stored measures resolves to
- this table.
-
- The earlier four-field form -- ``metrics``/``dimensions`` as lists alongside
- separate ``measure_columns``/``dimension_columns`` maps -- is no longer
- accepted, and a spec still using it is rejected with a message describing
- what to write instead.
+ time. ``name`` is a stable handle used for reconciliation.
+
+ Every metric and dimension is declared together with its physical column
+ in the external table.
"""
# Metric/dimension reference -> the physical column of the external table
@@ -345,16 +319,7 @@ class PartitionSpec(BaseModel):
class MaterializationSpec(BaseModel):
- """
- Declarative materialization config for a cube.
-
- Deliberately carries only what the author decides: when to build, how, how far
- back to look, how much of history to serve, how long the result is kept, and the
- Druid, Spark and platform settings their own site needs. The backend that runs it
- (and everything it derives -- the measures queries, combiner SQL, the rest of the
- Druid spec, output tables) is DJ's choice, so no `job` field is exposed here and
- the block stays portable if that choice changes.
- """
+ """Declarative materialization config for a cube."""
schedule: str
strategy: MaterializationStrategy = MaterializationStrategy.INCREMENTAL_TIME
@@ -1022,6 +987,7 @@ class MetricSpec(NodeSpec):
# Excluded from serialization so it's never exported.
columns: list[ColumnSpec] | None = Field(default=None, exclude=True)
required_dimensions: list[str] | None = None # Field(default_factory=list)
+ reaggregate: ReaggregateSpec | None = None
direction: MetricDirection | None = None
unit_enum: MetricUnit | None = Field(default=None, exclude=True)
# Structured unit form at the metric level — peer of `unit_enum`.
@@ -1038,6 +1004,7 @@ class MetricSpec(NodeSpec):
"columns": ChangeTier.NONE,
# Required dimensions constrain which queries the metric can answer.
"required_dimensions": ChangeTier.MAJOR,
+ "reaggregate": ChangeTier.MAJOR,
# Everything below is presentation metadata on the metric's single output
# column -- the same set the PATCH path already treats as minor via
# `metric_metadata` in `create_new_revision_from_existing`.
@@ -1179,6 +1146,29 @@ def diff(self, other: "NodeSpec") -> list[str]:
changed.remove("required_dimensions")
return changed
+ @property
+ def rendered_reaggregate(self) -> ReaggregateSpec | None:
+ """
+ Reaggregate spec with `${prefix}` resolved to this spec's namespace.
+ """
+ if not self.reaggregate:
+ return None
+ rules = [
+ rule.model_copy(
+ update={
+ "dimension": render_prefixes(rule.dimension, self.namespace)
+ if "${prefix}" in rule.dimension
+ else rule.dimension,
+ },
+ )
+ for rule in self.reaggregate.rules
+ ]
+ return self.reaggregate.model_copy(
+ update={
+ "rules": rules,
+ },
+ )
+
def model_dump(self, **kwargs): # pragma: no cover
base = super().model_dump(**kwargs)
base["unit"] = self.unit
@@ -1218,6 +1208,7 @@ def __eq__(self, other: object) -> bool:
other.canonical_required_dimensions,
preserve_order=False,
)
+ and self.rendered_reaggregate == other.rendered_reaggregate
and eq_or_fallback(self.direction, other.direction, MetricDirection.NEUTRAL)
and self._normalized_unit() == other._normalized_unit()
and self.significant_digits == other.significant_digits
@@ -1241,20 +1232,11 @@ class CubeSpec(NodeSpec):
filters: list[str] | None = None
columns: list[ColumnSpec] | None = None
# Tri-state. A spec materializes the cube on the schedule it names; the `none`
- # sentinel tears down whatever the cube has materialized; absent -- and null,
- # which is what serializing an absent field produces -- means the cube's
- # materialization is not managed here and whatever exists is left alone. Only a
- # value can carry intent through serialization, so removal is spelled
- # `materialization: none` rather than inferred from a key being present.
+ # sentinel tears down whatever the cube has materialized; absent means the cube's
+ # materialization is not managed here and whatever exists is left alone.
#
- # A list declares more than one, which a cube legitimately needs: an
- # `incremental_time` build for freshness alongside a periodic `full` rebuild
- # that corrects late-arriving data, out-of-order events and dimension
- # backfills. `strategy` is what tells two entries apart -- `job` is not
- # authorable (see `MaterializationSpec`) and everything else is a knob rather
- # than an identity -- so two entries sharing one are rejected below. The scalar
- # form stays valid and means exactly what it always did; nothing has to be
- # rewritten as a one-element list.
+ # A list declares more than one (e.g. incremental + full). `strategy` is what tells
+ # two entries apart, so two entries sharing one are rejected below.
materialization: (
MaterializationSpec | list[MaterializationSpec] | MaterializationAction | None
) = None
@@ -1271,18 +1253,8 @@ class CubeSpec(NodeSpec):
# Only user-authored partition config is compared here (see __eq__);
# everything else about cube columns is auto-derived.
"columns": ChangeTier.MAJOR,
- # Materialization is not part of a cube's definition. Configuring one
- # through `POST /nodes/{name}/materialization/` has never created a node
- # revision, and a YAML-declared schedule has to behave the same way, or
- # the same edit would cut a version through one door and not the other.
- # Because `materialization` is also excluded from `__eq__`, a
- # materialization-only edit leaves the cube in the deployment's skip list
- # and never reaches the classifier at all; NONE records that intent rather
- # than describing a reachable code path. Nothing is lost by the exclusion:
- # reconciling the declared block against the persisted config is a separate
- # concern that runs over every declared cube whether or not the node
- # changed, which is also what catches a materialization changed outside
- # YAML.
+ # Materialization is not part of a cube's definition, so changing it
+ # does not mint a new node revision.
"materialization": ChangeTier.NONE,
}
diff --git a/datajunction-server/datajunction_server/models/materialization.py b/datajunction-server/datajunction_server/models/materialization.py
index 421e6269c..031e82c2b 100644
--- a/datajunction-server/datajunction_server/models/materialization.py
+++ b/datajunction-server/datajunction_server/models/materialization.py
@@ -32,6 +32,22 @@
if TYPE_CHECKING:
from datajunction_server.database.node import NodeRevision
+
+class MaterializationTarget(StrEnum):
+ """
+ Where a measures table is being written.
+
+ Distinct from ``Dialect``: measures SQL executes in Spark for a Druid cube
+ and an Iceberg pre-agg alike, so the dialect cannot tell the two apart. Only
+ the target can, and some components must be written differently depending on
+ it -- a sketch whose in-engine representation is not the one the destination
+ reads needs converting on the way out.
+ """
+
+ DRUID = "druid"
+ ICEBERG = "iceberg"
+
+
DRUID_AGG_MAPPING = {
("bigint", "sum"): "longSum",
("int", "sum"): "longSum",
@@ -54,8 +70,124 @@
("binary", "hll_sketch_agg"): "HLLSketchMerge",
}
-# Aggregation types that need special handling (extra config parameters)
-DRUID_SKETCH_TYPES = {"HLLSketchMerge"}
+# Aggregator types that carry extra config in the Druid metricsSpec, mapped to
+# their default configuration. A sketch aggregator is not fully specified by its
+# type: an HLL needs a precision, a t-digest a compression, a KLL a `k`. Defaults
+# apply when a metric declares no `reaggregate.params`.
+DRUID_SKETCH_CONFIG: dict[str, dict[str, Any]] = {
+ "HLLSketchMerge": {
+ "lgK": 12, # Log2 of K, controls precision (4-21)
+ "tgtHllType": "HLL_4", # HLL_4, HLL_6, or HLL_8
+ },
+}
+
+
+def register_druid_aggregator(
+ column_type: str,
+ merge_func: str,
+ aggregator: str,
+ default_config: dict[str, Any] | None = None,
+) -> None:
+ """
+ Register a Druid ingestion aggregator for a (column type, merge function) pair.
+
+ Args:
+ column_type: Measures-table column type, e.g. "binary"
+ merge_func: Phase-2 merge function name, e.g. "nflx_tdigest_agg"
+ aggregator: Druid aggregator type, e.g. "tDigestSketch"
+ default_config: Extra metricsSpec keys and their defaults, e.g.
+ ``{"compression": 200}``. Marks the aggregator as parameterized.
+
+ Raises:
+ DJInvalidInputException: If the pair is registered to a different aggregator,
+ or the aggregator has different defaults. Re-registering identical defaults
+ is a no-op.
+ """
+ key = (column_type, merge_func.lower())
+ existing = DRUID_AGG_MAPPING.get(key)
+ if existing is not None and existing != aggregator:
+ raise DJInvalidInputException(
+ message=(
+ f"Druid aggregator for {key} is already registered as "
+ f"`{existing}`; refusing to replace it with `{aggregator}`."
+ ),
+ )
+ new_config = dict(default_config) if default_config else None
+ existing_config = DRUID_SKETCH_CONFIG.get(aggregator)
+ if (
+ existing_config is not None
+ and new_config is not None
+ and existing_config != new_config
+ ):
+ raise DJInvalidInputException(
+ message=(
+ f"Druid aggregator `{aggregator}` is already registered with "
+ f"different defaults ({existing_config}); refusing to replace "
+ f"them with {new_config}."
+ ),
+ )
+ DRUID_AGG_MAPPING[key] = aggregator
+ if new_config:
+ DRUID_SKETCH_CONFIG[aggregator] = new_config
+
+
+def get_druid_aggregator_spec(
+ column_name: str,
+ column_type: str | None,
+ aggregation: str | None,
+ merge: str | None,
+ params: dict[str, Any] | None = None,
+) -> dict[str, Any] | None:
+ """
+ Build a Druid metricsSpec entry for a measure column, or None if unmappable.
+
+ The `merge` function takes precedence over `aggregation` since ingestion
+ operates on pre-aggregated partials.
+
+ Returns None for unmappable measures, allowing callers to determine fallback behavior.
+
+ Raises:
+ DJInvalidInputException: If `params` contains a key not declared by the aggregator.
+ """
+ agg_func = merge or aggregation
+ if not agg_func or column_type is None:
+ return None
+
+ aggregator = DRUID_AGG_MAPPING.get((column_type, agg_func.lower()))
+ if aggregator is None:
+ return None
+
+ family_config = DRUID_SKETCH_CONFIG.get(aggregator)
+ if params:
+ # Restrict `params` to declared tuning knobs to prevent overwriting
+ # structural keys (like `type` or `fieldName`) and catch typos.
+ allowed = set(family_config or ())
+ unknown = sorted(set(params) - allowed)
+ if unknown:
+ raise DJInvalidInputException(
+ message=(
+ f"Druid aggregator `{aggregator}` does not accept "
+ f"{', '.join(f'`{key}`' for key in unknown)}. "
+ + (
+ f"Supported: {', '.join(sorted(allowed))}."
+ if allowed
+ else "It takes no parameters."
+ )
+ ),
+ )
+
+ spec: dict[str, Any] = {
+ "fieldName": column_name,
+ "name": column_name,
+ "type": aggregator,
+ }
+ if family_config:
+ # Apply metric-specific params over family defaults.
+ spec.update(family_config)
+ if params:
+ spec.update(params)
+ return spec
+
# How long ingested cube data is kept in Druid. Without an explicit rule a datasource
# inherits the cluster default, which is unrelated to the span the cube holds, so an
@@ -458,16 +590,18 @@ def metrics_spec(self) -> dict:
for measure in measure_group.measures
if (measure.type.lower(), measure.agg.lower()) not in DRUID_AGG_MAPPING
]
- return {
- measure.name: {
- "fieldName": measure.field_name,
- "name": measure.field_name,
- "type": DRUID_AGG_MAPPING[(measure.type.lower(), measure.agg.lower())],
- }
+ # The V2 `Measure` model lacks a merge phase, so `agg` is used for lookup.
+ specs = {
+ measure.name: get_druid_aggregator_spec(
+ column_name=measure.field_name,
+ column_type=measure.type.lower(),
+ aggregation=measure.agg,
+ merge=None,
+ )
for measure_group in self.measures.values() # type: ignore
for measure in measure_group.measures
- if (measure.type.lower(), measure.agg.lower()) in DRUID_AGG_MAPPING
}
+ return {name: spec for name, spec in specs.items() if spec is not None}
def build_druid_spec(self, node_revision: "NodeRevision"):
"""
diff --git a/datajunction-server/datajunction_server/models/metric.py b/datajunction-server/datajunction_server/models/metric.py
index 2ec7e50b6..43813ccec 100644
--- a/datajunction-server/datajunction_server/models/metric.py
+++ b/datajunction-server/datajunction_server/models/metric.py
@@ -16,6 +16,7 @@
MetricMetadataOutput,
)
from datajunction_server.models.query import ColumnMetadata, V3ColumnMetadata
+from datajunction_server.models.reaggregate import ReaggregateSpec
from datajunction_server.models.sql import ScanEstimate, TranspiledSQL
from datajunction_server.models.unit import unit_to_dict
from datajunction_server.sql.decompose import MetricComponentExtractor
@@ -50,6 +51,7 @@ class Metric(BaseModel):
# going forward. `None` when no unit is set, regardless of input shape.
unit: dict | None = None
required_dimensions: list[str]
+ reaggregate: ReaggregateSpec | None = None
# Whether the metric is a single aggregation call (a "measure") that can map
# 1:1 to a column in an externally-built pre-aggregation table. Derived/ratio
@@ -104,6 +106,7 @@ async def parse_node(
node.current.columns[0].unit if node.current.columns else None,
),
required_dimensions=[dim.name for dim in node.current.required_dimensions],
+ reaggregate=node.current.reaggregate,
is_measure=node.current.is_measure,
incompatible_druid_functions=incompatible_druid_functions,
measures=measures,
diff --git a/datajunction-server/datajunction_server/models/node.py b/datajunction-server/datajunction_server/models/node.py
index e60ff6b67..13da677a4 100644
--- a/datajunction-server/datajunction_server/models/node.py
+++ b/datajunction-server/datajunction_server/models/node.py
@@ -33,6 +33,7 @@
from datajunction_server.models.materialization import MaterializationConfigOutput
from datajunction_server.models.node_type import NodeNameOutput, NodeType
from datajunction_server.models.partition import PartitionOutput
+from datajunction_server.models.reaggregate import ReaggregateSpec
from datajunction_server.models.tag import TagMinimum, TagOutput
from datajunction_server.models.unit import (
AtomicUnit,
@@ -878,6 +879,7 @@ class MetricNodeFields(BaseModel):
"""
required_dimensions: list[str] | None = None
+ reaggregate: ReaggregateSpec | None = None
metric_metadata: MetricMetadataInput | None = None
@@ -1033,6 +1035,7 @@ class NodeRevisionOutput(BaseModel):
materializations: list[MaterializationConfigOutput]
parents: list[NodeNameOutput]
metric_metadata: MetricMetadataOutput | None = None
+ reaggregate: ReaggregateSpec | None = None
dimension_links: list[LinkDimensionOutput] | None = None
custom_metadata: dict | None = None
@@ -1065,6 +1068,7 @@ class NodeOutput(GenericNodeOutputModel):
materializations: list[MaterializationConfigOutput]
parents: list[NodeNameOutput]
metric_metadata: MetricMetadataOutput | None = None
+ reaggregate: ReaggregateSpec | None = None
dimension_links: list[LinkDimensionOutput] = Field(default_factory=list)
created_at: UTCDatetime
created_by: UserNameOnly | None = None
diff --git a/datajunction-server/datajunction_server/models/reaggregate.py b/datajunction-server/datajunction_server/models/reaggregate.py
new file mode 100644
index 000000000..004b98944
--- /dev/null
+++ b/datajunction-server/datajunction_server/models/reaggregate.py
@@ -0,0 +1,153 @@
+"""Models for metric reaggregation declarations."""
+
+from typing import Any
+
+from pydantic import BaseModel, Field, model_validator
+
+from datajunction_server.enum import StrEnum
+
+
+class ReaggregationFunction(StrEnum):
+ """
+ Supported functions for metric reaggregation.
+ """
+
+ AUTO = "auto"
+ NONE = "none"
+ SUM = "sum"
+ AVG = "avg"
+ WEIGHTED_AVG = "weighted_avg"
+ LAST_VALUE = "last_value"
+ FIRST_VALUE = "first_value"
+ MIN = "min"
+ MAX = "max"
+ # Quantile sketch family. Unlike the functions above it does not describe a
+ # rollup arithmetic; it selects how a quantile metric is accumulated, merged
+ # and read back, which is what makes percentiles pre-aggregatable at all.
+ TDIGEST = "tdigest"
+
+
+DIMENSION_REAGGREGATE_FUNCTIONS = frozenset(
+ {
+ ReaggregationFunction.LAST_VALUE,
+ ReaggregationFunction.FIRST_VALUE,
+ ReaggregationFunction.MIN,
+ ReaggregationFunction.MAX,
+ },
+)
+
+
+PARAMETERIZED_REAGGREGATE_FUNCTIONS: frozenset[ReaggregationFunction] = frozenset(
+ {
+ # compression: centroids retained, trading sketch size for tail accuracy.
+ ReaggregationFunction.TDIGEST,
+ },
+)
+
+
+def is_parameterized_reaggregate_function(
+ function: ReaggregationFunction | None,
+) -> bool:
+ """
+ Return whether a function accepts tuning parameters in `params`.
+ """
+ return function in PARAMETERIZED_REAGGREGATE_FUNCTIONS
+
+
+def is_supported_dimension_reaggregate_function(
+ function: ReaggregationFunction,
+) -> bool:
+ """
+ Return whether a function can collapse across a protected dimension.
+ """
+ return function in DIMENSION_REAGGREGATE_FUNCTIONS
+
+
+def unsupported_dimension_reaggregate_functions(
+ spec: "ReaggregateSpec | dict | None",
+) -> list[str]:
+ """
+ Unsupported dimension-specific collapse functions in a reaggregate spec.
+ """
+ reaggregate = parse_reaggregate_spec(spec)
+ if not reaggregate:
+ return []
+ return sorted(
+ {
+ rule.fn.value
+ for rule in reaggregate.rules
+ if not is_supported_dimension_reaggregate_function(rule.fn)
+ },
+ )
+
+
+class DimensionReaggregateRule(BaseModel):
+ """
+ Dimension-specific metric reaggregation rule.
+ """
+
+ dimension: str
+ fn: ReaggregationFunction
+
+
+class ReaggregateSpec(BaseModel):
+ """
+ Declaration for how a metric rolls up from its accumulation grain.
+ """
+
+ fn: ReaggregationFunction | None = None
+ weight: str | None = None
+ rules: list[DimensionReaggregateRule] = Field(default_factory=list)
+
+ # Tuning parameters for `fn` (e.g., compression for t-digest).
+ # Rejected for functions that do not accept parameters.
+ params: dict[str, Any] | None = None
+
+ @model_validator(mode="after")
+ def check_params_supported(self) -> "ReaggregateSpec":
+ """
+ Reject `params` for unparameterized functions.
+ """
+ if self.params and not is_parameterized_reaggregate_function(self.fn):
+ named = f"`{self.fn.value}`" if self.fn else "no reaggregation function"
+ raise ValueError(
+ f"Reaggregation function {named} does not accept parameters, "
+ f"but got: {', '.join(sorted(self.params))}.",
+ )
+ return self
+
+
+def dump_reaggregate_spec(
+ spec: ReaggregateSpec | dict | None,
+) -> dict | None:
+ """
+ Return a JSON-serializable reaggregate spec dictionary.
+ """
+ if spec is None:
+ return None
+ if isinstance(spec, ReaggregateSpec):
+ return spec.model_dump(mode="json")
+ return ReaggregateSpec.model_validate(spec).model_dump(mode="json")
+
+
+def parse_reaggregate_spec(
+ spec: ReaggregateSpec | dict | None,
+) -> ReaggregateSpec | None:
+ """
+ Return a validated reaggregate spec model.
+ """
+ if spec is None:
+ return None
+ if isinstance(spec, ReaggregateSpec):
+ return spec
+ return ReaggregateSpec.model_validate(spec)
+
+
+def dimension_reaggregate_rules(
+ spec: ReaggregateSpec | dict | None,
+) -> list[DimensionReaggregateRule]:
+ """
+ Return the dimension-specific reaggregation rules from a spec.
+ """
+ reaggregate = parse_reaggregate_spec(spec)
+ return reaggregate.rules if reaggregate else []
diff --git a/datajunction-server/datajunction_server/semantic_fingerprints/v1.py b/datajunction-server/datajunction_server/semantic_fingerprints/v1.py
index 08cc05e83..d40f99c16 100644
--- a/datajunction-server/datajunction_server/semantic_fingerprints/v1.py
+++ b/datajunction-server/datajunction_server/semantic_fingerprints/v1.py
@@ -45,7 +45,7 @@
"primary_key",
"query",
),
- MetricSpec: ("query", "required_dimensions"),
+ MetricSpec: ("query", "required_dimensions", "reaggregate"),
CubeSpec: ("metrics", "dimensions", "filters", "columns"),
}
diff --git a/datajunction-server/datajunction_server/sql/decompose.py b/datajunction-server/datajunction_server/sql/decompose.py
index 3003ad291..cde8397ed 100644
--- a/datajunction-server/datajunction_server/sql/decompose.py
+++ b/datajunction-server/datajunction_server/sql/decompose.py
@@ -3,19 +3,29 @@
import hashlib
from abc import ABC, abstractmethod
from dataclasses import dataclass
-from typing import cast
+from typing import Any, cast
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import aliased
from datajunction_server.database.node import Node, NodeRelationship, NodeRevision
+from datajunction_server.errors import DJInvalidInputException
from datajunction_server.models.decompose import (
Aggregability,
AggregationRule,
MetricComponent,
)
+from datajunction_server.models.dialect import Dialect
+from datajunction_server.models.materialization import MaterializationTarget
from datajunction_server.models.node_type import NodeType
+from datajunction_server.models.reaggregate import (
+ ReaggregateSpec,
+ ReaggregationFunction,
+ dimension_reaggregate_rules,
+ is_supported_dimension_reaggregate_function,
+ parse_reaggregate_spec,
+)
from datajunction_server.naming import amenable_col_names
from datajunction_server.sql import functions as dj_functions
from datajunction_server.sql.parsing.backends.antlr4 import ast, parse
@@ -95,7 +105,29 @@ class ComponentDef:
suffix: str
accumulate: str
merge: str
+ # Fixed arguments appended after the column in the merge call, as SQL literals.
+ # `nflx_tdigest_agg(col, 200.0)` is merge="nflx_tdigest_agg" with
+ # merge_args=("200.0",). Kept separate from `merge` rather than templated into
+ # it because the Druid aggregator mapping and the semi-additive rewrite both
+ # match on the bare function name. Omitting a required tuning argument is not
+ # an error the engine reports: `nflx_tdigest_agg` has a one-argument overload
+ # that silently falls back to a near-useless compression and collapses the
+ # digest to a single centroid at the mean.
+ merge_args: tuple[str, ...] = ()
arg_index: int | None = 0 # Which arg to use, or None for multi-arg templates
+ # Conversion applied to the accumulated value when writing a measures table
+ # for a target that cannot read the in-engine representation. A template, as
+ # `accumulate` is: "nflx_tdigest_sketch({})". Applied only at materialization
+ # time and only for the targets in `serialize_targets`, so query-time SQL and
+ # every other destination keep the unwrapped value.
+ serialize: str | None = None
+ # Targets that need `serialize`. Empty means it never applies.
+ serialize_targets: tuple[MaterializationTarget, ...] = ()
+ # Column type the conversion produces, e.g. "binary". Declared rather than
+ # inferred: a sketch accumulate takes more arguments than type inference
+ # feeds it, so inference falls back to the metric type and would silently
+ # record the wrong type for the column.
+ serialize_type: str | None = None
class AggDecomposition(ABC):
@@ -115,9 +147,30 @@ class AggDecomposition(ABC):
def components(self) -> list[ComponentDef]:
"""Define the components needed for this aggregation."""
+ def __init__(self, params: dict[str, Any] | None = None):
+ """
+ Args:
+ params: Tuning parameters from the metric's ``reaggregate.params``.
+ Empty for the registry-by-function decompositions, which take
+ none; a sketch family reads its accuracy setting from here.
+ """
+ self.params = params or {}
+
@abstractmethod
- def combine(self, components: list[MetricComponent]) -> ast.Expression:
- """Build the combiner expression from merged metric components."""
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ) -> ast.Expression:
+ """
+ Build the combiner expression from merged metric components.
+
+ Returns Spark-canonical SQL; translation to other dialects happens in
+ the transpilation layer, so re-spelling function names, array literals
+ or index bases here applies them twice. Branch on `dialect` only for an
+ engine needing a structurally different expression.
+ """
# =============================================================================
@@ -127,6 +180,30 @@ def combine(self, components: list[MetricComponent]) -> ast.Expression:
DECOMPOSITION_REGISTRY: dict[type, type[AggDecomposition] | None] = {}
+# Decompositions selected by the metric's declared reaggregation family rather
+# than by its aggregation function. A family entry wins over the by-function
+# registry, which is what lets `APPROX_PERCENTILE(x, 0.95)` keep its spelling and
+# still decompose, but only for metrics that opted in with `reaggregate.fn`.
+FAMILY_DECOMPOSITION_REGISTRY: dict[ReaggregationFunction, type[AggDecomposition]] = {}
+
+
+def decomposes_family(fn: ReaggregationFunction):
+ """
+ Register a decomposition for a reaggregation family.
+
+ Exported for downstream deployments, which supply the engine-specific
+ functions a family needs. OSS registers none, so a metric declaring an
+ unregistered family falls through to the by-function registry and keeps the
+ aggregability it has today.
+ """
+
+ def decorator(decomp_class: type[AggDecomposition]):
+ FAMILY_DECOMPOSITION_REGISTRY[fn] = decomp_class
+ return decomp_class
+
+ return decorator
+
+
def decomposes(func_class: type):
"""Decorator to register a decomposition class for a function."""
@@ -155,7 +232,12 @@ class SumDecomposition(AggDecomposition):
def components(self) -> list[ComponentDef]:
return [ComponentDef("_sum", "SUM", "SUM")]
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func("SUM", components[0].name)
@@ -167,7 +249,12 @@ class MaxDecomposition(AggDecomposition):
def components(self) -> list[ComponentDef]:
return [ComponentDef("_max", "MAX", "MAX")]
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func("MAX", components[0].name)
@@ -179,7 +266,12 @@ class MinDecomposition(AggDecomposition):
def components(self) -> list[ComponentDef]:
return [ComponentDef("_min", "MIN", "MIN")]
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func("MIN", components[0].name)
@@ -191,7 +283,12 @@ class AnyValueDecomposition(AggDecomposition):
def components(self) -> list[ComponentDef]:
return [ComponentDef("_any_value", "ANY_VALUE", "ANY_VALUE")]
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func("ANY_VALUE", components[0].name)
@@ -208,7 +305,12 @@ class CountDecomposition(AggDecomposition):
def components(self) -> list[ComponentDef]:
return [ComponentDef("_count", "COUNT", "SUM")]
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func("SUM", components[0].name)
@@ -220,7 +322,12 @@ class CountIfDecomposition(AggDecomposition):
def components(self) -> list[ComponentDef]:
return [ComponentDef("_count_if", "COUNT_IF", "SUM")]
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func("SUM", components[0].name)
@@ -240,7 +347,12 @@ def components(self) -> list[ComponentDef]:
ComponentDef("_count", "COUNT", "SUM"),
]
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return ast.BinaryOp(
op=ast.BinaryOpKind.Divide,
left=make_func("SUM", components[0].name),
@@ -272,7 +384,12 @@ def components(self) -> list[ComponentDef]:
),
]
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func(
"hll_sketch_estimate",
make_func("hll_union_agg", components[0].name),
@@ -375,7 +492,12 @@ def _make_var_samp(self, components: list[MetricComponent]) -> ast.Expression:
class VarPopDecomposition(VarianceDecompositionBase):
"""Population variance: E[X²] - E[X]²"""
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return self._make_var_pop(components)
@@ -383,7 +505,12 @@ def combine(self, components: list[MetricComponent]):
class VarSampDecomposition(VarianceDecompositionBase):
"""Sample variance with Bessel's correction."""
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return self._make_var_samp(components)
@@ -391,7 +518,12 @@ def combine(self, components: list[MetricComponent]):
class VarianceDecomposition(VarianceDecompositionBase):
"""VARIANCE (alias for VAR_SAMP in Spark)."""
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return self._make_var_samp(components) # pragma: no cover
@@ -404,7 +536,12 @@ def combine(self, components: list[MetricComponent]):
class StddevPopDecomposition(VarianceDecompositionBase):
"""Population standard deviation: sqrt(VAR_POP)"""
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func("SQRT", self._make_var_pop(components))
@@ -412,7 +549,12 @@ def combine(self, components: list[MetricComponent]):
class StddevSampDecomposition(VarianceDecompositionBase):
"""Sample standard deviation: sqrt(VAR_SAMP)"""
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func("SQRT", self._make_var_samp(components))
@@ -420,7 +562,12 @@ def combine(self, components: list[MetricComponent]):
class StddevDecomposition(VarianceDecompositionBase):
"""STDDEV (alias for STDDEV_SAMP in Spark)."""
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return make_func("SQRT", self._make_var_samp(components)) # pragma: no cover
@@ -525,7 +672,12 @@ def _make_covar_samp(self, components: list[MetricComponent]) -> ast.Expression:
class CovarPopDecomposition(CovarianceDecompositionBase):
"""Population covariance: E[XY] - E[X]*E[Y]"""
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return self._make_covar_pop(components)
@@ -533,7 +685,12 @@ def combine(self, components: list[MetricComponent]):
class CovarSampDecomposition(CovarianceDecompositionBase):
"""Sample covariance with Bessel's correction."""
- def combine(self, components: list[MetricComponent]):
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ):
return self._make_covar_samp(components)
@@ -567,7 +724,12 @@ def components(self) -> list[ComponentDef]:
ComponentDef("_count", "COUNT({0})", "SUM", arg_index=0),
]
- def combine(self, components: list[MetricComponent]) -> ast.Expression:
+ def combine(
+ self,
+ components: list[MetricComponent],
+ func: ast.Function,
+ dialect: Dialect = Dialect.SPARK,
+ ) -> ast.Expression:
"""
Build CORR: COVAR(X,Y) / (STDDEV(X) * STDDEV(Y))
@@ -637,8 +799,27 @@ def combine(self, components: list[MetricComponent]) -> ast.Expression:
# =============================================================================
-def get_decomposition(func_class: type) -> AggDecomposition | None:
- """Get decomposition instance for a function class, or None if not decomposable."""
+def get_decomposition(
+ func_class: type,
+ reaggregate: ReaggregateSpec | None = None,
+) -> AggDecomposition | None:
+ """
+ Get the decomposition for an aggregation, or None if not decomposable.
+
+ A metric that declares a reaggregation family gets that family's
+ decomposition, which is how an opt-in sketch overrides the default handling
+ of its aggregation function. Everything else resolves by function class as
+ before.
+
+ A declared family with nothing registered for it falls through rather than
+ raising: OSS ships no family implementations, so the metric simply keeps the
+ aggregability it would have had.
+ """
+ if reaggregate is not None and reaggregate.fn is not None:
+ family_class = FAMILY_DECOMPOSITION_REGISTRY.get(reaggregate.fn)
+ if family_class is not None:
+ return family_class(params=reaggregate.params)
+
decomp_class = DECOMPOSITION_REGISTRY.get(func_class)
if decomp_class is None:
return None
@@ -746,6 +927,7 @@ class BaseMetricData:
name: str
query: str
+ reaggregate: ReaggregateSpec | dict | None = None
@dataclass
@@ -771,14 +953,30 @@ class MetricComponentExtractor:
For derived metrics: collects components from base metrics and substitutes references.
"""
- def __init__(self, node_revision_id: int):
+ def __init__(
+ self,
+ node_revision_id: int,
+ dialect: Dialect = Dialect.SPARK,
+ ):
"""
Extract metric components from a specific metric revision.
Args:
node_revision_id: ID of the metric node revision
+ dialect: Dialect the combiner will be rendered for. Every dialect
+ gets a combiner -- whether one exists at all is a property of
+ the aggregation function, not the engine -- but its shape can
+ differ: a sketch family whose engines expose different function
+ shapes needs the target, as Druid fuses a t-digest's merge and
+ combine where Spark and Trino keep them separate.
+
+ Defaults to Spark, which is right for the callers that render
+ for display or for frozen measures. Only the build_v3 path,
+ which knows the cube's actual target engine, passes anything
+ else.
"""
self._node_revision_id = node_revision_id
+ self._dialect = dialect
@classmethod
async def from_node_name( # pragma: no cover
@@ -919,7 +1117,10 @@ async def extract(
if is_parent_derived and parent_revision_id:
# Recursively extract the derived metric (inline expansion)
- parent_extractor = MetricComponentExtractor(parent_revision_id)
+ parent_extractor = MetricComponentExtractor(
+ parent_revision_id,
+ dialect=self._dialect,
+ )
base_components, derived_ast = await parent_extractor.extract(
session,
nodes_cache=nodes_cache,
@@ -930,7 +1131,10 @@ async def extract(
else:
# True base metric - decompose aggregations
base_ast = parse(base_metric.query)
- base_components, derived_ast = self._extract_base(base_ast)
+ base_components, derived_ast = self._extract_base(
+ base_ast,
+ parse_reaggregate_spec(base_metric.reaggregate),
+ )
for comp in base_components:
if comp.name not in components_tracker:
@@ -984,6 +1188,12 @@ def _build_metric_data_from_cache(
]
if metric_parents:
+ if metric_node.current.reaggregate:
+ raise DJInvalidInputException(
+ "Semi-additive metrics must be base metrics in V1. "
+ f"Derived metric `{metric_node.name}` declares reaggregate.",
+ )
+
# Derived metric - base metrics are the parent metrics
return MetricData(
query=metric_node.current.query,
@@ -992,6 +1202,7 @@ def _build_metric_data_from_cache(
BaseMetricData(
name=parent_name,
query=nodes_cache[parent_name].current.query,
+ reaggregate=nodes_cache[parent_name].current.reaggregate,
)
for parent_name in metric_parents
if nodes_cache[parent_name].current
@@ -1007,6 +1218,7 @@ def _build_metric_data_from_cache(
BaseMetricData(
name=metric_node.name,
query=metric_node.current.query,
+ reaggregate=metric_node.current.reaggregate,
),
],
)
@@ -1023,6 +1235,7 @@ async def _load_metric_data(self, session: AsyncSession) -> MetricData:
select(
Node.name.label("parent_name"),
NodeRevision.query.label("parent_query"),
+ NodeRevision.reaggregate.label("parent_reaggregate"),
)
.select_from(NodeRelationship)
.join(Node, NodeRelationship.parent_id == Node.id)
@@ -1043,6 +1256,7 @@ async def _load_metric_data(self, session: AsyncSession) -> MetricData:
this_metric_stmt = (
select(
NodeRevision.query,
+ NodeRevision.reaggregate,
Node.name,
)
.join(Node, NodeRevision.node_id == Node.id)
@@ -1055,12 +1269,22 @@ async def _load_metric_data(self, session: AsyncSession) -> MetricData:
this_row = this_result.one()
if parent_rows:
+ if this_row.reaggregate:
+ raise DJInvalidInputException(
+ "Semi-additive metrics must be base metrics in V1. "
+ f"Derived metric `{this_row.name}` declares reaggregate.",
+ )
+
# Derived metric - base metrics are the parents
return MetricData(
query=this_row.query,
is_derived=True,
base_metrics=[
- BaseMetricData(name=row.parent_name, query=row.parent_query)
+ BaseMetricData(
+ name=row.parent_name,
+ query=row.parent_query,
+ reaggregate=row.parent_reaggregate,
+ )
for row in parent_rows
],
)
@@ -1069,12 +1293,19 @@ async def _load_metric_data(self, session: AsyncSession) -> MetricData:
return MetricData(
query=this_row.query,
is_derived=False,
- base_metrics=[BaseMetricData(name=this_row.name, query=this_row.query)],
+ base_metrics=[
+ BaseMetricData(
+ name=this_row.name,
+ query=this_row.query,
+ reaggregate=this_row.reaggregate,
+ ),
+ ],
)
def _extract_base(
self,
query_ast: ast.Query,
+ reaggregate: ReaggregateSpec | None = None,
) -> tuple[list[MetricComponent], ast.Query]:
"""
Extract components from a base metric by decomposing aggregations.
@@ -1105,11 +1336,24 @@ def _extract_base(
# If any aggregation is non-decomposable, abort decomposition
# entirely — the metric is non-decomposable as a whole.
- if any(get_decomposition(dj_fn) is None for _, dj_fn in agg_funcs):
+ #
+ # The spec has to be passed here, not just to `_decompose` below:
+ # a family-gated metric decomposes precisely because it declared a
+ # family, and its aggregation function has no entry of its own. Ask
+ # without the spec and every such metric aborts at this gate and
+ # never reaches the family registry at all.
+ if any(
+ get_decomposition(dj_fn, reaggregate) is None for _, dj_fn in agg_funcs
+ ):
+ if dimension_reaggregate_rules(reaggregate):
+ self._raise_unsupported_reaggregate_shape(
+ "dimension-specific reaggregation requires a "
+ "decomposable aggregation",
+ )
return [], query_ast
for func, dj_function in agg_funcs:
- result = self._decompose(func, dj_function, query_ast)
+ result = self._decompose(func, dj_function, query_ast, reaggregate)
if result: # pragma: no branch
# Apply combiner to AST
func.parent.replace(from_=func, to=result.combiner) # type: ignore
@@ -1128,8 +1372,78 @@ def _extract_base(
for proj in query_ast.select.projection:
wrap_divisions_in_nullif(cast(ast.Expression, proj))
+ if reaggregate is not None and dimension_reaggregate_rules(reaggregate):
+ self._attach_reaggregate_spec(components, reaggregate)
+
+ if reaggregate is not None and reaggregate.params:
+ self._attach_reaggregate_params(components, reaggregate)
+
return components, query_ast
+ def _attach_reaggregate_params(
+ self,
+ components: list[MetricComponent],
+ reaggregate: ReaggregateSpec,
+ ) -> None:
+ """
+ Propagate tuning parameters from the reaggregate spec to aggregating components.
+ """
+ configurable = [
+ component
+ for component in components
+ if component.aggregation is not None and component.merge is not None
+ ]
+ if not configurable:
+ self._raise_unsupported_reaggregate_shape(
+ "parameterized reaggregation requires an aggregating component",
+ )
+ for component in configurable:
+ component.params = dict(reaggregate.params or {})
+
+ def _attach_reaggregate_spec(
+ self,
+ components: list[MetricComponent],
+ reaggregate: ReaggregateSpec,
+ ) -> None:
+ """
+ Attach a dimension-specific reaggregation rule to one ordinary measure.
+ """
+ dimension_rules = dimension_reaggregate_rules(reaggregate)
+ if len(dimension_rules) != 1:
+ self._raise_unsupported_reaggregate_shape(
+ "dimension-specific reaggregation requires exactly one rule",
+ )
+
+ if not is_supported_dimension_reaggregate_function(dimension_rules[0].fn):
+ self._raise_unsupported_reaggregate_shape(
+ f"unsupported dimension reaggregation function "
+ f"`{dimension_rules[0].fn.value}`",
+ )
+
+ if len(components) != 1:
+ self._raise_unsupported_reaggregate_shape(
+ "dimension-specific reaggregation requires exactly one component",
+ )
+
+ component = components[0]
+ if (
+ component.rule.type != Aggregability.FULL
+ or component.aggregation is None
+ or component.merge is None
+ ):
+ self._raise_unsupported_reaggregate_shape(
+ "dimension-specific reaggregation requires one non-distinct "
+ "fully-aggregatable component",
+ )
+
+ component.rule.reaggregate = dimension_rules[0]
+
+ @staticmethod
+ def _raise_unsupported_reaggregate_shape(reason: str) -> None:
+ raise DJInvalidInputException(
+ f"Unsupported reaggregate metric shape: {reason}.",
+ )
+
def _substitute_metric_references(
self,
query_ast: ast.Query,
@@ -1179,9 +1493,10 @@ def _decompose(
func: ast.Function,
dj_function: type,
query_ast: ast.Query,
+ reaggregate: ReaggregateSpec | None = None,
) -> DecompositionResult | None:
"""Decompose an aggregation function using the registry."""
- decomposition = get_decomposition(dj_function)
+ decomposition = get_decomposition(dj_function, reaggregate)
if decomposition is None: # pragma: no cover
# Defensive: ``_extract_base`` filters non-decomposable
@@ -1208,7 +1523,7 @@ def _decompose(
quantifier=ast.SetQuantifier.Distinct,
)
else:
- combiner_ast = decomposition.combine(components)
+ combiner_ast = decomposition.combine(components, func, self._dialect)
# Decomposed AVG / variance / stddev / covariance all build
# SUM(...) / SUM(count)-style combiners where the denominator
# can legitimately be 0. Wrap to produce NULL rather than
@@ -1288,11 +1603,15 @@ def _make_component(
expression=expression,
aggregation=None if is_distinct else accumulate_expr,
merge=None if is_distinct else comp_def.merge,
+ merge_args=[] if is_distinct else list(comp_def.merge_args),
rule=AggregationRule(
type=Aggregability.LIMITED if is_distinct else Aggregability.FULL,
level=[str(a) for a in func.args] if is_distinct else None,
),
grain_alias=grain_alias,
+ serialize=comp_def.serialize,
+ serialize_targets=list(comp_def.serialize_targets),
+ serialize_type=comp_def.serialize_type,
)
def _expand_template(self, template: str, args: list) -> str:
diff --git a/datajunction-server/datajunction_server/sql/functions.py b/datajunction-server/datajunction_server/sql/functions.py
index b194d46e4..87c6891ae 100644
--- a/datajunction-server/datajunction_server/sql/functions.py
+++ b/datajunction-server/datajunction_server/sql/functions.py
@@ -552,16 +552,16 @@ class ApproxPercentile(Function):
def infer_type(
col: ct.NumberType,
percentage: ct.ListType,
- accuracy: ct.NumberType | None,
-) -> ct.DoubleType:
+ accuracy: ct.NumberType | None = None,
+) -> ct.ListType:
return ct.ListType(element_type=col.type) # type: ignore
@ApproxPercentile.register
def infer_type(
col: ct.NumberType,
- percentage: ct.FloatType,
- accuracy: ct.NumberType | None,
+ percentage: ct.FloatingBase,
+ accuracy: ct.NumberType | None = None,
) -> ct.NumberType:
return col.type # type: ignore
diff --git a/datajunction-server/datajunction_server/sql/parsing/ast.py b/datajunction-server/datajunction_server/sql/parsing/ast.py
index 324bb2abb..8bd013d3f 100644
--- a/datajunction-server/datajunction_server/sql/parsing/ast.py
+++ b/datajunction-server/datajunction_server/sql/parsing/ast.py
@@ -1604,29 +1604,6 @@ async def add_ref_column(
"""
Add column referenced from this table. Returns True if the table has the column
and False otherwise.
-
- This function handles the following cases:
-
- Regular columns. For example:
- (1) non-aliased columns
- `SELECT country_id AS country1 FROM countries` should match the `country_id`
- column in the table `countries`
- (2) aliased columns
- `SELECT C.country_id AS country1 FROM countries C` should match the `country_id`
- column in the table `countries` with the column namespace/table alias `C`
-
- Struct columns. For example:
- (1) non-aliased struct columns
- `countries` has column `identifiers` with type:
- STRUCT
- `SELECT identifiers.country_name AS name FROM countries` should match the
- `identifier` -> `country_name` column in the table `countries`
- (2) aliased struct columns
- `countries` has column `identifiers` with type:
- STRUCT
- `SELECT C.identifiers.country_name AS name FROM countries C` should match the
- `identifier` -> `country_name` column in the table `countries` with the column namespace/
- table alias `C`
"""
if not self._columns:
if ctx is None:
diff --git a/datajunction-server/tests/api/cubes_build_metrics_spec_test.py b/datajunction-server/tests/api/cubes_build_metrics_spec_test.py
index 64c14bb34..e196fa151 100644
--- a/datajunction-server/tests/api/cubes_build_metrics_spec_test.py
+++ b/datajunction-server/tests/api/cubes_build_metrics_spec_test.py
@@ -4,7 +4,19 @@
This function builds Druid metricsSpec from measure columns and their components.
"""
+import pytest
+
from datajunction_server.api.cubes import _build_metrics_spec
+from datajunction_server.models.cube_materialization import MetricMeasures
+from datajunction_server.models.materialization import (
+ DRUID_AGG_MAPPING,
+ DRUID_SKETCH_CONFIG,
+ DruidMeasuresCubeConfig,
+ Measure,
+ get_druid_aggregator_spec,
+ register_druid_aggregator,
+)
+from datajunction_server.errors import DJInvalidInputException
from datajunction_server.models.decompose import AggregationRule, MetricComponent
from datajunction_server.models.query import ColumnMetadata
@@ -374,3 +386,293 @@ def test_preserves_column_order(self):
result = _build_metrics_spec(columns, components, aliases)
assert [r["name"] for r in result] == ["z_col", "a_col", "m_col"]
+
+
+class TestDruidAggregatorSpec:
+ """
+ Tests for the shared aggregator helper behind both metricsSpec builders.
+ """
+
+ def test_merge_is_preferred_over_accumulate(self):
+ """
+ Ingestion merges stored partials, so the merge function keys the lookup.
+ """
+ spec = get_druid_aggregator_spec(
+ column_name="orders_count",
+ column_type="bigint",
+ aggregation="COUNT",
+ merge="SUM",
+ )
+ assert spec == {
+ "fieldName": "orders_count",
+ "name": "orders_count",
+ "type": "longSum",
+ }
+
+ def test_falls_back_to_accumulate_without_merge(self):
+ """A component with no merge phase keys off its accumulate."""
+ spec = get_druid_aggregator_spec(
+ column_name="revenue_sum",
+ column_type="double",
+ aggregation="SUM",
+ merge=None,
+ )
+ assert spec is not None
+ assert spec["type"] == "doubleSum"
+
+ def test_unmappable_returns_none(self):
+ """
+ Return None for unmappable measures so callers determine fallback behavior.
+ """
+ assert (
+ get_druid_aggregator_spec(
+ column_name="x",
+ column_type="varchar",
+ aggregation="SUM",
+ merge="SUM",
+ )
+ is None
+ )
+ assert (
+ get_druid_aggregator_spec(
+ column_name="x",
+ column_type=None,
+ aggregation="SUM",
+ merge="SUM",
+ )
+ is None
+ )
+ assert (
+ get_druid_aggregator_spec(
+ column_name="x",
+ column_type="bigint",
+ aggregation=None,
+ merge=None,
+ )
+ is None
+ )
+
+ def test_sketch_gets_default_config(self):
+ """An HLL sketch carries its precision defaults into the spec."""
+ spec = get_druid_aggregator_spec(
+ column_name="accounts_hll",
+ column_type="binary",
+ aggregation="hll_sketch_agg",
+ merge="hll_union_agg",
+ )
+ assert spec == {
+ "fieldName": "accounts_hll",
+ "name": "accounts_hll",
+ "type": "HLLSketchMerge",
+ "lgK": 12,
+ "tgtHllType": "HLL_4",
+ }
+
+ def test_declared_params_override_defaults_partially(self):
+ """
+ Metric-level params override defaults for specific keys while retaining others.
+ """
+ spec = get_druid_aggregator_spec(
+ column_name="accounts_hll",
+ column_type="binary",
+ aggregation="hll_sketch_agg",
+ merge="hll_union_agg",
+ params={"lgK": 17},
+ )
+ assert spec is not None
+ assert spec["lgK"] == 17
+ assert spec["tgtHllType"] == "HLL_4"
+
+ def test_params_cannot_overwrite_structural_keys(self):
+ """
+ `params` must not overwrite structural keys like `type` or `fieldName`.
+ """
+ with pytest.raises(DJInvalidInputException) as excinfo:
+ get_druid_aggregator_spec(
+ column_name="accounts_hll",
+ column_type="binary",
+ aggregation="hll_sketch_agg",
+ merge="hll_union_agg",
+ params={"type": "doubleSum", "fieldName": "some_other_column"},
+ )
+ assert "does not accept" in str(excinfo.value)
+ assert "`fieldName`" in str(excinfo.value)
+ assert "`type`" in str(excinfo.value)
+
+ def test_unknown_param_is_rejected(self):
+ """Unknown parameters are rejected loudly."""
+ with pytest.raises(DJInvalidInputException) as excinfo:
+ get_druid_aggregator_spec(
+ column_name="accounts_hll",
+ column_type="binary",
+ aggregation="hll_sketch_agg",
+ merge="hll_union_agg",
+ params={"lgk": 17}, # lowercase k
+ )
+ assert "lgk" in str(excinfo.value)
+ assert "lgK" in str(excinfo.value) # the supported spelling is named
+
+ def test_params_rejected_for_unparameterized_aggregator(self):
+ """Params passed to an unparameterized aggregator raise an error."""
+ with pytest.raises(DJInvalidInputException) as excinfo:
+ get_druid_aggregator_spec(
+ column_name="orders_count",
+ column_type="bigint",
+ aggregation="COUNT",
+ merge="SUM",
+ params={"compression": 200},
+ )
+ assert "takes no parameters" in str(excinfo.value)
+
+
+class TestRegisterDruidAggregator:
+ """
+ Tests for registering custom sketch families.
+ """
+
+ def test_register_and_use(self):
+ """A registered family is mappable and carries its default config."""
+ try:
+ register_druid_aggregator(
+ column_type="binary",
+ merge_func="test_tdigest_agg",
+ aggregator="testTDigestSketch",
+ default_config={"compression": 200},
+ )
+ spec = get_druid_aggregator_spec(
+ column_name="latency_tdigest",
+ column_type="binary",
+ aggregation="test_tdigest",
+ merge="test_tdigest_agg",
+ )
+ assert spec == {
+ "fieldName": "latency_tdigest",
+ "name": "latency_tdigest",
+ "type": "testTDigestSketch",
+ "compression": 200,
+ }
+ # A metric's own params win over the family default.
+ tuned = get_druid_aggregator_spec(
+ column_name="latency_tdigest",
+ column_type="binary",
+ aggregation="test_tdigest",
+ merge="test_tdigest_agg",
+ params={"compression": 1000},
+ )
+ assert tuned is not None
+ assert tuned["compression"] == 1000
+ assert "testTDigestSketch" in DRUID_SKETCH_CONFIG
+ finally:
+ DRUID_AGG_MAPPING.pop(("binary", "test_tdigest_agg"), None)
+ DRUID_SKETCH_CONFIG.pop("testTDigestSketch", None)
+
+ def test_register_without_config(self):
+ """A family with nothing to tune registers without extra config keys."""
+ try:
+ register_druid_aggregator(
+ column_type="bigint",
+ merge_func="test_plain_merge",
+ aggregator="testPlainAgg",
+ )
+ spec = get_druid_aggregator_spec(
+ column_name="c",
+ column_type="bigint",
+ aggregation="test_plain",
+ merge="test_plain_merge",
+ )
+ assert spec == {
+ "fieldName": "c",
+ "name": "c",
+ "type": "testPlainAgg",
+ }
+ finally:
+ DRUID_AGG_MAPPING.pop(("bigint", "test_plain_merge"), None)
+
+ def test_conflicting_defaults_are_rejected(self):
+ """
+ Re-registering an aggregator with different defaults raises an error.
+ """
+ try:
+ register_druid_aggregator(
+ column_type="binary",
+ merge_func="test_conflict_agg",
+ aggregator="testConflictSketch",
+ default_config={"compression": 100},
+ )
+ with pytest.raises(DJInvalidInputException) as excinfo:
+ register_druid_aggregator(
+ column_type="binary",
+ merge_func="test_conflict_agg",
+ aggregator="testConflictSketch",
+ default_config={"compression": 200},
+ )
+ assert "already registered with different defaults" in str(excinfo.value)
+ # Re-registering exactly what is there is a no-op.
+ register_druid_aggregator(
+ column_type="binary",
+ merge_func="test_conflict_agg",
+ aggregator="testConflictSketch",
+ default_config={"compression": 100},
+ )
+ assert DRUID_SKETCH_CONFIG["testConflictSketch"] == {"compression": 100}
+ finally:
+ DRUID_AGG_MAPPING.pop(("binary", "test_conflict_agg"), None)
+ DRUID_SKETCH_CONFIG.pop("testConflictSketch", None)
+
+ def test_conflicting_aggregator_is_rejected(self):
+ """Two families cannot claim the same (column type, merge func) pair."""
+ try:
+ register_druid_aggregator(
+ column_type="binary",
+ merge_func="test_claimed",
+ aggregator="testFirstSketch",
+ )
+ with pytest.raises(DJInvalidInputException) as excinfo:
+ register_druid_aggregator(
+ column_type="binary",
+ merge_func="test_claimed",
+ aggregator="testSecondSketch",
+ )
+ assert "already registered as" in str(excinfo.value)
+ finally:
+ DRUID_AGG_MAPPING.pop(("binary", "test_claimed"), None)
+
+ def test_v2_measures_cube_config_gets_family_defaults(self):
+ """
+ The V2 path uses the aggregator fallback with `merge=None` to pull correct specs.
+ """
+ try:
+ register_druid_aggregator(
+ column_type="binary",
+ merge_func="test_v2_tdigest",
+ aggregator="testV2Sketch",
+ default_config={"compression": 200},
+ )
+ config = DruidMeasuresCubeConfig.model_construct(
+ dimensions=[],
+ measures={
+ "some.metric": MetricMeasures.model_construct(
+ metric="some.metric",
+ combiner="x",
+ measures=[
+ Measure(
+ name="latency_tdigest",
+ field_name="latency_tdigest",
+ agg="test_v2_tdigest",
+ type="binary",
+ ),
+ ],
+ ),
+ },
+ )
+ assert config.metrics_spec() == {
+ "latency_tdigest": {
+ "fieldName": "latency_tdigest",
+ "name": "latency_tdigest",
+ "type": "testV2Sketch",
+ "compression": 200,
+ },
+ }
+ finally:
+ DRUID_AGG_MAPPING.pop(("binary", "test_v2_tdigest"), None)
+ DRUID_SKETCH_CONFIG.pop("testV2Sketch", None)
diff --git a/datajunction-server/tests/api/cubes_test.py b/datajunction-server/tests/api/cubes_test.py
index aa00f9671..6d1b6245a 100644
--- a/datajunction-server/tests/api/cubes_test.py
+++ b/datajunction-server/tests/api/cubes_test.py
@@ -3584,10 +3584,16 @@ async def test_cube_materialization_metadata(
"aggregation": "COUNT",
"expression": "*",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"merge": "SUM",
"name": "count_c8e42e74",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -3595,10 +3601,16 @@ async def test_cube_materialization_metadata(
"aggregation": "SUM",
"expression": "if(discount > 0.0, 1, 0)",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"merge": "SUM",
"name": "discount_sum_30b84e6c",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -3606,10 +3618,16 @@ async def test_cube_materialization_metadata(
"aggregation": "COUNT",
"expression": "price",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"merge": "SUM",
"name": "price_count_935e7117",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -3617,10 +3635,16 @@ async def test_cube_materialization_metadata(
"aggregation": "SUM",
"expression": "price * discount",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"merge": "SUM",
"name": "price_discount_sum_e4ba5456",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -3628,10 +3652,16 @@ async def test_cube_materialization_metadata(
"aggregation": "SUM",
"expression": "price",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"merge": "SUM",
"name": "price_sum_935e7117",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -3639,10 +3669,16 @@ async def test_cube_materialization_metadata(
"aggregation": "COUNT",
"expression": "repair_order_id",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"merge": "SUM",
"name": "repair_order_id_count_bd241964",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -3650,10 +3686,16 @@ async def test_cube_materialization_metadata(
"aggregation": "SUM",
"expression": "total_repair_cost",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"merge": "SUM",
"name": "total_repair_cost_sum_67874507",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -3769,10 +3811,16 @@ async def test_cube_materialization_metadata(
"aggregation": "SUM",
"expression": "price",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"merge": "SUM",
"name": "price_sum_252381cf",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -4023,65 +4071,105 @@ async def test_cube_materialization_metadata(
"name": "count_c8e42e74",
"expression": "*",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"aggregation": "COUNT",
"merge": "SUM",
- "rule": {"type": "full", "level": None},
+ "rule": {"type": "full", "level": None, "reaggregate": None},
},
{
"name": "discount_sum_30b84e6c",
"expression": "if(discount > 0.0, 1, 0)",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"aggregation": "SUM",
"merge": "SUM",
- "rule": {"type": "full", "level": None},
+ "rule": {"type": "full", "level": None, "reaggregate": None},
},
{
"name": "price_count_935e7117",
"expression": "price",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"aggregation": "COUNT",
"merge": "SUM",
- "rule": {"type": "full", "level": None},
+ "rule": {"type": "full", "level": None, "reaggregate": None},
},
{
"name": "price_discount_sum_e4ba5456",
"expression": "price * discount",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"aggregation": "SUM",
"merge": "SUM",
- "rule": {"type": "full", "level": None},
+ "rule": {"type": "full", "level": None, "reaggregate": None},
},
{
"name": "price_sum_935e7117",
"expression": "price",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"aggregation": "SUM",
"merge": "SUM",
- "rule": {"type": "full", "level": None},
+ "rule": {"type": "full", "level": None, "reaggregate": None},
},
{
"name": "repair_order_id_count_bd241964",
"expression": "repair_order_id",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"aggregation": "COUNT",
"merge": "SUM",
- "rule": {"type": "full", "level": None},
+ "rule": {"type": "full", "level": None, "reaggregate": None},
},
{
"name": "total_repair_cost_sum_67874507",
"expression": "total_repair_cost",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"aggregation": "SUM",
"merge": "SUM",
- "rule": {"type": "full", "level": None},
+ "rule": {"type": "full", "level": None, "reaggregate": None},
},
{
"name": "price_sum_252381cf",
"expression": "price",
"grain_alias": None,
+ "params": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
"aggregation": "SUM",
"merge": "SUM",
- "rule": {"type": "full", "level": None},
+ "rule": {"type": "full", "level": None, "reaggregate": None},
},
],
"timestamp_column": "hire_date",
@@ -4804,6 +4892,133 @@ class TestCubeMaterializeV2SuccessPaths:
and mock the query service client to test the full endpoint flow.
"""
+ @pytest.fixture(autouse=True)
+ def _preserve_app_dependency_overrides(
+ self,
+ client_with_repairs_cube: AsyncClient,
+ ):
+ """Protect module-scoped client overrides from function-scoped clients."""
+ original_overrides = dict(client_with_repairs_cube.app.dependency_overrides)
+ yield
+ client_with_repairs_cube.app.dependency_overrides.clear()
+ client_with_repairs_cube.app.dependency_overrides.update(original_overrides)
+
+ @pytest.mark.asyncio
+ async def test_materialize_reaggregate_cube_without_protected_dimension_fails(
+ self,
+ client_with_build_v3: AsyncClient,
+ mocker,
+ ):
+ """Druid cube materialization refuses to bake in collapsed
+ semi-additive metrics without the protected dimension."""
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.daily_balance",
+ "description": "Semi-additive balance measured by order date",
+ "query": "SELECT SUM(line_total) FROM v3.order_details",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "v3.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ cube_name = "v3.test_daily_balance_materialization_guard"
+ response = await client_with_build_v3.post(
+ "/nodes/cube/",
+ json={
+ "name": cube_name,
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ "mode": "published",
+ "description": "Unsafe semi-additive materialization",
+ },
+ )
+ assert response.status_code == 201, response.json()
+
+ combiner = mocker.patch(
+ "datajunction_server.api.cubes.build_combiner_sql_from_preaggs",
+ )
+
+ response = await client_with_build_v3.post(
+ f"/cubes/{cube_name}/materialize",
+ json={"strategy": "full", "schedule": "0 0 * * *"},
+ )
+
+ assert response.status_code == 400, response.json()
+ assert "protected dimension" in response.json()["message"]
+ combiner.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_materialize_derived_reaggregate_cube_without_protected_dimension_fails(
+ self,
+ client_with_build_v3: AsyncClient,
+ mocker,
+ ):
+ """Druid cube materialization also refuses derived metrics whose base
+ metric has an omitted semi-additive protected dimension."""
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.daily_balance",
+ "description": "Semi-additive balance measured by order date",
+ "query": "SELECT SUM(line_total) FROM v3.order_details",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "v3.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.daily_balance_index",
+ "description": "Derived semi-additive balance index",
+ "query": "SELECT 10.0 / v3.daily_balance",
+ "mode": "published",
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ cube_name = "v3.test_daily_balance_index_materialization_guard"
+ response = await client_with_build_v3.post(
+ "/nodes/cube/",
+ json={
+ "name": cube_name,
+ "metrics": ["v3.daily_balance_index"],
+ "dimensions": ["v3.product.category"],
+ "mode": "published",
+ "description": "Unsafe derived semi-additive materialization",
+ },
+ )
+ assert response.status_code == 201, response.json()
+
+ combiner = mocker.patch(
+ "datajunction_server.api.cubes.build_combiner_sql_from_preaggs",
+ )
+
+ response = await client_with_build_v3.post(
+ f"/cubes/{cube_name}/materialize",
+ json={"strategy": "full", "schedule": "0 0 * * *"},
+ )
+
+ assert response.status_code == 400, response.json()
+ assert "protected dimension" in response.json()["message"]
+ combiner.assert_not_called()
+
@pytest.mark.asyncio
async def test_materialize_cube_full_strategy_success(
self,
@@ -5596,6 +5811,12 @@ async def test_materialize_cube_returns_metric_combiners(
mock_qs_client.materialize_cube_v2.return_value = mocker.MagicMock(
urls=["http://workflow/test-cube"],
)
+ had_query_service_override = get_query_service_client in (
+ client.app.dependency_overrides
+ )
+ original_query_service_override = client.app.dependency_overrides.get(
+ get_query_service_client,
+ )
client.app.dependency_overrides[get_query_service_client] = lambda: (
mock_qs_client
)
@@ -5783,8 +6004,12 @@ async def test_materialize_cube_returns_metric_combiners(
]
finally:
# Clean up dependency override
- if get_query_service_client in client.app.dependency_overrides:
- del client.app.dependency_overrides[get_query_service_client]
+ if had_query_service_override:
+ client.app.dependency_overrides[get_query_service_client] = (
+ original_query_service_override
+ )
+ else:
+ client.app.dependency_overrides.pop(get_query_service_client, None)
class TestCubeDeactivateSuccessPaths:
@@ -6020,6 +6245,12 @@ async def test_deactivate_cube_uses_stored_workflow_names(
urls=["http://workflow/cube-workflow"],
workflow_names=["cube_wf_name_1"],
)
+ had_query_service_override = get_query_service_client in (
+ client.app.dependency_overrides
+ )
+ original_query_service_override = client.app.dependency_overrides.get(
+ get_query_service_client,
+ )
client.app.dependency_overrides[get_query_service_client] = lambda: (
mock_qs_client
)
@@ -6047,8 +6278,12 @@ async def test_deactivate_cube_uses_stored_workflow_names(
# The old path should NOT have been called
mock_qs_client.deactivate_cube_workflow.assert_not_called()
finally:
- if get_query_service_client in client.app.dependency_overrides:
- del client.app.dependency_overrides[get_query_service_client]
+ if had_query_service_override:
+ client.app.dependency_overrides[get_query_service_client] = (
+ original_query_service_override
+ )
+ else:
+ client.app.dependency_overrides.pop(get_query_service_client, None)
class TestCubeBackfillSuccessPaths:
diff --git a/datajunction-server/tests/api/graphql/resolvers/test_node_resolver.py b/datajunction-server/tests/api/graphql/resolvers/test_node_resolver.py
index 6a47982db..f38a5501a 100644
--- a/datajunction-server/tests/api/graphql/resolvers/test_node_resolver.py
+++ b/datajunction-server/tests/api/graphql/resolvers/test_node_resolver.py
@@ -70,6 +70,86 @@ def test_columns_resolver_filters_by_attribute():
assert result[0].name == "user_id"
+def test_reaggregate_resolver_returns_none_for_non_metric():
+ """
+ Test that the reaggregate resolver only resolves for metric revisions.
+ """
+ from datajunction_server.api.graphql.scalars.node import NodeRevision
+ from datajunction_server.database import NodeRevision as DBNodeRevision
+ from datajunction_server.models.node_type import NodeType
+
+ db_node_revision = DBNodeRevision(
+ name="source_rev",
+ type=NodeType.SOURCE,
+ version="1",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+
+ result = NodeRevision.reaggregate(NodeRevision, root=db_node_revision)
+
+ assert result is None
+
+
+def test_reaggregate_resolver_returns_none_for_empty_metric_spec():
+ """
+ Test that the reaggregate resolver returns None for metrics without a spec.
+ """
+ from datajunction_server.api.graphql.scalars.node import NodeRevision
+ from datajunction_server.database import NodeRevision as DBNodeRevision
+ from datajunction_server.models.node_type import NodeType
+
+ db_node_revision = DBNodeRevision(
+ name="metric_rev",
+ type=NodeType.METRIC,
+ version="1",
+ reaggregate=None,
+ )
+
+ result = NodeRevision.reaggregate(NodeRevision, root=db_node_revision)
+
+ assert result is None
+
+
+def test_reaggregate_resolver_returns_metric_spec():
+ """
+ Test that the reaggregate resolver exposes metric reaggregation declarations.
+ """
+ from datajunction_server.api.graphql.scalars.node import NodeRevision
+ from datajunction_server.database import NodeRevision as DBNodeRevision
+ from datajunction_server.models.node_type import NodeType
+ from datajunction_server.models.reaggregate import ReaggregationFunction
+
+ db_node_revision = DBNodeRevision(
+ name="metric_rev",
+ type=NodeType.METRIC,
+ version="1",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+
+ result = NodeRevision.reaggregate(NodeRevision, root=db_node_revision)
+
+ assert result is not None
+ assert result.fn is None
+ assert result.weight is None
+ assert len(result.rules) == 1
+ assert result.rules[0].dimension == "default.date_dim.date"
+ assert result.rules[0].fn == ReaggregationFunction.LAST_VALUE
+
+
@pytest.mark.asyncio
@patch("datajunction_server.api.graphql.resolvers.nodes.DBNode.get_by_name")
@patch("datajunction_server.api.graphql.resolvers.nodes.load_node_options")
diff --git a/datajunction-server/tests/api/measures_test.py b/datajunction-server/tests/api/measures_test.py
index 15cb18343..4d3c0d943 100644
--- a/datajunction-server/tests/api/measures_test.py
+++ b/datajunction-server/tests/api/measures_test.py
@@ -292,6 +292,7 @@ async def test_list_frozen_measures(
"name": "repair_order_id_count_bd241964",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
"upstream_revision": {
diff --git a/datajunction-server/tests/api/metrics_test.py b/datajunction-server/tests/api/metrics_test.py
index b6d939b76..49c39146a 100644
--- a/datajunction-server/tests/api/metrics_test.py
+++ b/datajunction-server/tests/api/metrics_test.py
@@ -3,6 +3,7 @@
"""
from unittest.mock import patch
+from uuid import uuid4
import pytest
import pytest_asyncio
@@ -464,10 +465,16 @@ async def test_read_metrics(module__client_with_roads: AsyncClient) -> None:
"aggregation": "SUM",
"expression": "if(discount > 0.0, 1, 0)",
"grain_alias": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
+ "params": None,
"name": "discount_sum_30b84e6c",
"merge": "SUM",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -475,10 +482,16 @@ async def test_read_metrics(module__client_with_roads: AsyncClient) -> None:
"aggregation": "COUNT",
"expression": "*",
"grain_alias": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
+ "params": None,
"merge": "SUM",
"name": "count_c8e42e74",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -494,6 +507,111 @@ async def test_read_metrics(module__client_with_roads: AsyncClient) -> None:
assert data["custom_metadata"] is None
+@pytest.mark.asyncio
+async def test_metric_reaggregate_roundtrip_and_validation(
+ client_with_roads: AsyncClient,
+) -> None:
+ """
+ Semi-additive declarations round-trip and validate their protected dimension.
+ """
+ metric_name = f"default.reaggregate_repairs_{uuid4().hex}"
+ response = await client_with_roads.post(
+ "/nodes/metric/",
+ json={
+ "name": metric_name,
+ "description": "Repair orders with semi-additive declaration",
+ "query": "SELECT COUNT(repair_order_id) FROM default.repair_orders_fact",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "repair_order_id",
+ "fn": "last_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+ assert response.json()["reaggregate"] == {
+ "fn": None,
+ "weight": None,
+ "params": None,
+ "rules": [
+ {
+ "dimension": "repair_order_id",
+ "fn": "last_value",
+ },
+ ],
+ }
+
+ response = await client_with_roads.get(f"/nodes/{metric_name}/")
+ assert response.status_code == 200
+ assert response.json()["reaggregate"] == {
+ "fn": None,
+ "weight": None,
+ "params": None,
+ "rules": [
+ {
+ "dimension": "repair_order_id",
+ "fn": "last_value",
+ },
+ ],
+ }
+
+ response = await client_with_roads.get(f"/metrics/{metric_name}/")
+ assert response.status_code == 200
+ assert response.json()["reaggregate"] == {
+ "fn": None,
+ "weight": None,
+ "params": None,
+ "rules": [
+ {
+ "dimension": "repair_order_id",
+ "fn": "last_value",
+ },
+ ],
+ }
+
+ invalid_metric_name = f"default.invalid_reaggregate_{uuid4().hex}"
+ response = await client_with_roads.post(
+ "/nodes/metric/",
+ json={
+ "name": invalid_metric_name,
+ "description": "Invalid semi-additive declaration",
+ "query": "SELECT COUNT(repair_order_id) FROM default.repair_orders_fact",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "default.repair_orders_fact.nope",
+ "fn": "last_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code == 422
+ assert response.json() == {
+ "message": "Node definition contains references to columns as "
+ "reaggregate dimensions that are not on parent nodes.",
+ "errors": [
+ {
+ "code": "INVALID_COLUMN",
+ "message": "Node definition contains references to columns as "
+ "reaggregate dimensions that are not on parent nodes.",
+ "debug": {
+ "invalid_reaggregate_dimensions": [
+ "default.repair_orders_fact.nope",
+ ],
+ },
+ "context": "",
+ },
+ ],
+ "warnings": [],
+ }
+
+
@pytest_asyncio.fixture(scope="module")
async def module__current_user(module__session: AsyncSession) -> User:
"""
diff --git a/datajunction-server/tests/api/preaggregations_test.py b/datajunction-server/tests/api/preaggregations_test.py
index 72a8b9804..e87a1d523 100644
--- a/datajunction-server/tests/api/preaggregations_test.py
+++ b/datajunction-server/tests/api/preaggregations_test.py
@@ -736,11 +736,17 @@ async def test_get_preagg_by_id(self, client_with_preaggs):
"expr_hash": "83632b779d87",
"expression": "line_total",
"grain_alias": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
+ "params": None,
"merge": "SUM",
"name": "line_total_sum_e1f61696",
"source_column": None,
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
"used_by_metrics": [
@@ -803,11 +809,17 @@ async def test_get_preagg_by_id(self, client_with_preaggs):
"expr_hash": "221d2a4bfdae",
"expression": "quantity",
"grain_alias": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
+ "params": None,
"merge": "SUM",
"name": "quantity_sum_06b64d2e",
"source_column": None,
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
"used_by_metrics": [
diff --git a/datajunction-server/tests/api/sql_v2_test.py b/datajunction-server/tests/api/sql_v2_test.py
index 42c58eb52..fec7578f1 100644
--- a/datajunction-server/tests/api/sql_v2_test.py
+++ b/datajunction-server/tests/api/sql_v2_test.py
@@ -1201,9 +1201,18 @@ async def create_metric_distinct_single_column(client: AsyncClient):
"aggregation": None,
"expression": "hard_hat_id",
"grain_alias": "hard_hat_id",
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
+ "params": None,
"merge": None,
"name": "hard_hat_id",
- "rule": {"level": ["hard_hat_id"], "type": "limited"},
+ "rule": {
+ "level": ["hard_hat_id"],
+ "reaggregate": None,
+ "type": "limited",
+ },
},
]
assert metric_data["derived_expression"] == "COUNT( DISTINCT hard_hat_id)"
@@ -1228,9 +1237,18 @@ async def create_metric_distinct_expression(client: AsyncClient):
"aggregation": None,
"expression": "IF(hard_hat_id = 1, 1, 0)",
"grain_alias": "hard_hat_id_distinct_0291ee39",
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
+ "params": None,
"merge": None,
"name": "hard_hat_id_distinct_0291ee39",
- "rule": {"level": ["IF(hard_hat_id = 1, 1, 0)"], "type": "limited"},
+ "rule": {
+ "level": ["IF(hard_hat_id = 1, 1, 0)"],
+ "reaggregate": None,
+ "type": "limited",
+ },
},
]
assert (
@@ -1623,10 +1641,16 @@ async def test_metric_definitions_with_nonjoinable_dimensions(
"aggregation": "SUM",
"expression": "default.local_hard_hats_2.hard_hat_id",
"grain_alias": None,
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
+ "params": None,
"merge": "SUM",
"name": "default_DOT_local_hard_hats_2_DOT_hard_hat_id_sum_bf8a8419",
"rule": {
"level": None,
+ "reaggregate": None,
"type": "full",
},
},
@@ -1687,10 +1711,16 @@ async def test_metric_definitions_with_single_joinable_dimensions(
"aggregation": None,
"expression": "default.municipality_dim.contact_name",
"grain_alias": "contact_name",
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
+ "params": None,
"merge": None,
"name": "contact_name",
"rule": {
"level": ["default.municipality_dim.contact_name"],
+ "reaggregate": None,
"type": "limited",
},
},
@@ -1792,6 +1822,11 @@ async def test_metric_definition_with_multiple_joinable_dimensions(
"expression": "IF(default.hard_hat.state = 'NY', default.hard_hat.first_name, "
"NULL)",
"grain_alias": "default_DOT_hard_hat_DOT_state_default_DOT_hard_hat_DOT_first_name_distinct_1a99d6a7",
+ "merge_args": [],
+ "serialize": None,
+ "serialize_targets": [],
+ "serialize_type": None,
+ "params": None,
"merge": None,
"name": "default_DOT_hard_hat_DOT_state_default_DOT_hard_hat_DOT_first_name_distinct_1a99d6a7",
"rule": {
@@ -1799,6 +1834,7 @@ async def test_metric_definition_with_multiple_joinable_dimensions(
"IF(default.hard_hat.state = 'NY', "
"default.hard_hat.first_name, NULL)",
],
+ "reaggregate": None,
"type": "limited",
},
},
diff --git a/datajunction-server/tests/construction/build_v3/accumulate_type_test.py b/datajunction-server/tests/construction/build_v3/accumulate_type_test.py
new file mode 100644
index 000000000..43c048ca4
--- /dev/null
+++ b/datajunction-server/tests/construction/build_v3/accumulate_type_test.py
@@ -0,0 +1,152 @@
+"""
+Tests for typing an accumulate whose outermost call takes several arguments.
+
+Type inference feeds a component exactly one input type. That is right for
+every aggregation DJ ships, including the templated ones -- ``SUM(POWER(x, 2))``
+is still a single-argument ``SUM`` on the outside. A sketch is the first shape
+that breaks it: ``nflx_tdigest(latency_ms, CAST(200.0 AS DOUBLE))`` needs two,
+so inference raised ``TypeError``, the caller swallowed it, and the column was
+recorded as the metric's own type.
+
+That mattered beyond cosmetics. The recorded type is persisted on the
+pre-aggregation row and forwarded to the query service, which uses it to
+create the materialized table -- so a struct-valued sketch column was being
+described as ``double``.
+
+``POWER`` stands in for a sketch accumulate here: OSS registers no sketch
+families, and it is the available two-argument function whose inferred result
+differs from the fallback.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+
+from datajunction_server.construction.build_v3.measures import (
+ _multi_argument_accumulate_types,
+ infer_component_type,
+)
+from datajunction_server.models.decompose import AggregationRule, MetricComponent
+from datajunction_server.models.materialization import MaterializationTarget
+
+
+def _parent(column_type: str = "double"):
+ return SimpleNamespace(
+ current=SimpleNamespace(
+ columns=[SimpleNamespace(name="latency_ms", type=column_type)],
+ ),
+ )
+
+
+def _component(aggregation: str, **kwargs) -> MetricComponent:
+ return MetricComponent(
+ name="latency_acc_abc123",
+ expression="latency_ms",
+ aggregation=aggregation,
+ merge="SUM",
+ rule=AggregationRule(),
+ **kwargs,
+ )
+
+
+class TestMultiArgumentTypes:
+ """The helper resolves every argument, or declines entirely."""
+
+ def test_resolves_a_column_and_a_literal(self):
+ types = _multi_argument_accumulate_types("POWER(latency_ms, 2)", _parent())
+ assert [str(t) for t in types] == ["double", "int"]
+
+ def test_resolves_a_cast_without_binding_it_to_a_table(self):
+ # Literals and casts carry their own type; only columns need the parent.
+ types = _multi_argument_accumulate_types(
+ "POWER(latency_ms, CAST(200.0 AS DOUBLE))",
+ _parent(),
+ )
+ assert [str(t) for t in types] == ["double", "double"]
+
+ def test_declines_without_a_parent_to_resolve_columns_against(self):
+ assert _multi_argument_accumulate_types("POWER(latency_ms, 2)", None) is None
+
+ def test_declines_for_a_bare_function_name(self):
+ # "SUM" is a name, not a call -- the overwhelmingly common shape.
+ assert _multi_argument_accumulate_types("SUM", _parent()) is None
+
+ def test_declines_for_a_single_argument_call(self):
+ # Including templated ones, whose outermost call still takes one arg.
+ assert _multi_argument_accumulate_types("SUM(POWER(latency_ms, 2))", _parent()) is None
+
+ def test_declines_when_a_column_type_is_unrecognized(self):
+ # Better to fall back than to invent a type for the stored schema.
+ assert (
+ _multi_argument_accumulate_types(
+ "POWER(latency_ms, 2)",
+ _parent("some_unknown_type"),
+ )
+ is None
+ )
+
+
+class TestInferredColumnType:
+ """What the measures column ends up recorded as."""
+
+ def test_multi_argument_accumulate_is_typed_from_its_arguments(self):
+ # The regression: this used to return the metric type, because
+ # POWER.infer_type(double) raises and the error was swallowed.
+ assert (
+ infer_component_type(
+ _component("POWER(latency_ms, 2)"),
+ "bigint",
+ _parent(),
+ )
+ == "double"
+ )
+
+ def test_single_argument_accumulate_is_unchanged(self):
+ assert (
+ infer_component_type(_component("SUM"), "bigint", _parent()) == "double"
+ )
+
+ def test_unresolvable_multi_argument_accumulate_falls_back(self):
+ assert (
+ infer_component_type(
+ _component("POWER(latency_ms, 2)"),
+ "bigint",
+ _parent("some_unknown_type"),
+ )
+ == "bigint"
+ )
+
+ @pytest.mark.parametrize("target", [None, MaterializationTarget.ICEBERG])
+ def test_unconverted_targets_get_the_argument_derived_type(self, target):
+ assert (
+ infer_component_type(
+ _component("POWER(latency_ms, 2)"),
+ "bigint",
+ _parent(),
+ target,
+ )
+ == "double"
+ )
+
+ def test_declared_serialize_type_still_wins_for_its_target(self):
+ """
+ The conversion type takes precedence over the accumulated one.
+
+ Druid stores the converted representation, so the column must be
+ described as that -- the aggregator lookup keys on it.
+ """
+ component = _component(
+ "POWER(latency_ms, 2)",
+ serialize="to_druid_bytes({})",
+ serialize_targets=[MaterializationTarget.DRUID],
+ serialize_type="binary",
+ )
+ assert (
+ infer_component_type(
+ component,
+ "bigint",
+ _parent(),
+ MaterializationTarget.DRUID,
+ )
+ == "binary"
+ )
diff --git a/datajunction-server/tests/construction/build_v3/cube_matcher_test.py b/datajunction-server/tests/construction/build_v3/cube_matcher_test.py
index 89a3cf630..e8019e4b4 100644
--- a/datajunction-server/tests/construction/build_v3/cube_matcher_test.py
+++ b/datajunction-server/tests/construction/build_v3/cube_matcher_test.py
@@ -29,12 +29,49 @@
from datajunction_server.construction.build_v3.utils import (
extract_filter_dimension_refs,
)
+from datajunction_server.database.node import Node
from datajunction_server.errors import DJInvalidInputException
from datajunction_server.models.decompose import Aggregability
from datajunction_server.models.dialect import Dialect
from tests.construction.build_v3 import assert_sql_equal
+async def _create_daily_balance_metric(client_with_build_v3):
+ """Create a semi-additive metric protected by order date."""
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.daily_balance",
+ "description": "Semi-additive balance measured by order date",
+ "query": "SELECT SUM(line_total) FROM v3.order_details",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "v3.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+
+async def _create_daily_balance_index_metric(client_with_build_v3):
+ """Create a derived metric that depends on the semi-additive balance."""
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.daily_balance_index",
+ "description": "Derived semi-additive balance index",
+ "query": "SELECT 10.0 / v3.daily_balance",
+ "mode": "published",
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+
def test_materialized_dimension_lookup_preserves_roles():
"""Two roles with one short name resolve to distinct physical columns."""
cube = SimpleNamespace(
@@ -1156,6 +1193,197 @@ def _boom(_filters):
)
assert "could not be parsed" in str(exc.value)
+ @pytest.mark.asyncio
+ async def test_reaggregate_cube_missing_protected_dimension_not_matched(
+ self,
+ client_with_build_v3,
+ session,
+ ):
+ """A materialized cube that dropped a semi-additive protected dimension
+ is not safe to auto-route to."""
+ await _create_daily_balance_metric(client_with_build_v3)
+ response = await client_with_build_v3.post(
+ "/nodes/cube/",
+ json={
+ "name": "v3.test_daily_balance_category_cube",
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ "mode": "published",
+ "description": "Category cube missing semi-additive protected grain",
+ },
+ )
+ assert response.status_code == 201, response.json()
+
+ response = await client_with_build_v3.post(
+ "/data/v3.test_daily_balance_category_cube/availability/",
+ json={
+ "catalog": "default",
+ "schema_": "analytics",
+ "table": "daily_balance_category_cube",
+ "valid_through_ts": int(time.time() * 1000),
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ result = await find_matching_cube(
+ session,
+ metrics=["v3.daily_balance"],
+ dimensions=["v3.product.category"],
+ )
+ assert result is None
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_unsafe_cube_falls_back_to_live_sql(
+ self,
+ client_with_build_v3,
+ session,
+ ):
+ """An unsafe materialized cube is ignored, leaving the live path usable."""
+ await _create_daily_balance_metric(client_with_build_v3)
+ response = await client_with_build_v3.post(
+ "/nodes/cube/",
+ json={
+ "name": "v3.test_daily_balance_live_fallback_cube",
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ "mode": "published",
+ "description": "Unsafe cube for live fallback",
+ },
+ )
+ assert response.status_code == 201, response.json()
+
+ response = await client_with_build_v3.post(
+ "/data/v3.test_daily_balance_live_fallback_cube/availability/",
+ json={
+ "catalog": "default",
+ "schema_": "analytics",
+ "table": "daily_balance_live_fallback_cube",
+ "valid_through_ts": int(time.time() * 1000),
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ result = await build_metrics_sql(
+ session=session,
+ metrics=["v3.daily_balance"],
+ dimensions=["v3.product.category"],
+ )
+
+ assert result.cube_name is None
+ assert "daily_balance_live_fallback_cube" not in result.sql
+ assert "MAX_BY(" in result.sql
+ assert "date_id_order" in result.sql
+
+ @pytest.mark.asyncio
+ async def test_derived_reaggregate_cube_missing_protected_dimension_not_matched(
+ self,
+ client_with_build_v3,
+ session,
+ ):
+ """Derived metric cubes inherit base semi-additive materialization needs."""
+ await _create_daily_balance_metric(client_with_build_v3)
+ await _create_daily_balance_index_metric(client_with_build_v3)
+ response = await client_with_build_v3.post(
+ "/nodes/cube/",
+ json={
+ "name": "v3.test_daily_balance_index_category_cube",
+ "metrics": ["v3.daily_balance_index"],
+ "dimensions": ["v3.product.category"],
+ "mode": "published",
+ "description": "Derived cube missing semi-additive protected grain",
+ },
+ )
+ assert response.status_code == 201, response.json()
+
+ response = await client_with_build_v3.post(
+ "/data/v3.test_daily_balance_index_category_cube/availability/",
+ json={
+ "catalog": "default",
+ "schema_": "analytics",
+ "table": "daily_balance_index_category_cube",
+ "valid_through_ts": int(time.time() * 1000),
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ result = await find_matching_cube(
+ session,
+ metrics=["v3.daily_balance_index"],
+ dimensions=["v3.product.category"],
+ )
+ assert result is None
+
+ @pytest.mark.asyncio
+ async def test_pinned_reaggregate_cube_missing_protected_dimension_raises(
+ self,
+ client_with_build_v3,
+ session,
+ ):
+ """Pinned cube validation also blocks unsafe semi-additive materialization."""
+ await _create_daily_balance_metric(client_with_build_v3)
+ response = await client_with_build_v3.post(
+ "/nodes/cube/",
+ json={
+ "name": "v3.test_daily_balance_pinned_unsafe_cube",
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ "mode": "published",
+ "description": "Pinned unsafe semi-additive cube",
+ },
+ )
+ assert response.status_code == 201, response.json()
+ cube_node = await Node.get_cube_by_name(
+ session,
+ "v3.test_daily_balance_pinned_unsafe_cube",
+ )
+ assert cube_node and cube_node.current
+
+ with pytest.raises(DJInvalidInputException) as exc:
+ await validate_pinned_cube_covers_filters(
+ session,
+ cube_node.current,
+ dimensions=["v3.product.category"],
+ filters=[],
+ metrics=["v3.daily_balance"],
+ )
+ assert "protected dimension" in str(exc.value)
+
+ @pytest.mark.asyncio
+ async def test_pinned_derived_reaggregate_cube_missing_protected_dimension_raises(
+ self,
+ client_with_build_v3,
+ session,
+ ):
+ """Pinned derived metric cubes also inherit base semi-additive needs."""
+ await _create_daily_balance_metric(client_with_build_v3)
+ await _create_daily_balance_index_metric(client_with_build_v3)
+ response = await client_with_build_v3.post(
+ "/nodes/cube/",
+ json={
+ "name": "v3.test_daily_balance_index_pinned_unsafe_cube",
+ "metrics": ["v3.daily_balance_index"],
+ "dimensions": ["v3.product.category"],
+ "mode": "published",
+ "description": "Pinned unsafe derived semi-additive cube",
+ },
+ )
+ assert response.status_code == 201, response.json()
+ cube_node = await Node.get_cube_by_name(
+ session,
+ "v3.test_daily_balance_index_pinned_unsafe_cube",
+ )
+ assert cube_node and cube_node.current
+
+ with pytest.raises(DJInvalidInputException) as exc:
+ await validate_pinned_cube_covers_filters(
+ session,
+ cube_node.current,
+ dimensions=["v3.product.category"],
+ filters=[],
+ metrics=["v3.daily_balance_index"],
+ )
+ assert "protected dimension" in str(exc.value)
+
@pytest.mark.asyncio
async def test_pinned_cube_non_druid_dialect_skips_validation(
self,
diff --git a/datajunction-server/tests/construction/build_v3/decomposition_test.py b/datajunction-server/tests/construction/build_v3/decomposition_test.py
index 5a8a6af39..179a95372 100644
--- a/datajunction-server/tests/construction/build_v3/decomposition_test.py
+++ b/datajunction-server/tests/construction/build_v3/decomposition_test.py
@@ -5,9 +5,11 @@
import pytest
from datajunction_server.construction.build_v3.decomposition import (
+ _reaggregate_dimension_requested,
get_base_metrics_for_derived,
is_derived_metric,
)
+from datajunction_server.construction.build_v3.metrics import _source_dimension_alias
from datajunction_server.construction.build_v3.types import BuildContext
from datajunction_server.models.node_type import NodeType
@@ -223,6 +225,42 @@ def test_metric_with_only_dimension_parent_not_derived(self):
assert result is False
+@pytest.mark.parametrize(
+ ("protected_dimension", "requested_dimensions", "expected"),
+ [
+ ("v3.date.date_id[order]", ["v3.date.date_id[order]"], True),
+ ("v3.date.date_id[order]", ["v3.date.date_id[order]]"], True),
+ ("v3.date.date_id[order]", ["v3.date.date_id"], False),
+ ("v3.date.date_id[order]", ["v3.date.date_id[ship]"], False),
+ ("v3.date.date_id[order]", ["date_id"], False),
+ ("v3.date.date_id", ["v3.date.date_id"], True),
+ ("v3.date.date_id", ["date_id"], True),
+ ("v3.date.date_id", ["v3.date.date_id[order]"], False),
+ ],
+)
+def test_reaggregate_dimension_requested_is_role_sensitive(
+ protected_dimension,
+ requested_dimensions,
+ expected,
+):
+ """A role-less ref must not satisfy a role-qualified protected dimension."""
+ assert (
+ _reaggregate_dimension_requested(protected_dimension, requested_dimensions)
+ is expected
+ )
+
+
+def test_source_dimension_alias_does_not_fall_back_for_roled_refs():
+ """A roled protected dimension must collapse by that role's physical alias."""
+ ctx = BuildContext(session=MagicMock(), metrics=[], dimensions=[])
+ ctx.alias_registry.register("v3.date.date_id")
+ ctx.alias_registry.register("v3.date.date_id[ship]")
+
+ assert _source_dimension_alias(ctx, "v3.date.date_id[order]") is None
+ assert _source_dimension_alias(ctx, "v3.date.date_id[ship]") == "date_id_ship"
+ assert _source_dimension_alias(ctx, "v3.date.date_id") == "date_id"
+
+
@pytest.mark.asyncio
async def test_decomposition_with_dimension_parent_integration(
module__client_with_build_v3,
diff --git a/datajunction-server/tests/construction/build_v3/merge_args_test.py b/datajunction-server/tests/construction/build_v3/merge_args_test.py
new file mode 100644
index 000000000..e044b4361
--- /dev/null
+++ b/datajunction-server/tests/construction/build_v3/merge_args_test.py
@@ -0,0 +1,129 @@
+"""
+Tests for fixed merge arguments on a component's Phase 2 call.
+
+Sketch families need tuning passed to the merge as well as the accumulate --
+`nflx_tdigest_agg(digest, compression)` -- but `merge` itself has to stay a bare
+function name, because the Druid aggregator mapping and the semi-additive
+rewrite both match on it. `merge_args` carries the rest.
+
+The omission this guards against is silent: a sketch merge called without its
+compression is accepted by the engine and returns a digest collapsed to a single
+centroid, so every quantile comes back equal to the mean.
+"""
+
+from datajunction_server.construction.build_v3.decomposition import build_merge_call
+from datajunction_server.models.decompose import (
+ AggregationRule,
+ Aggregability,
+ MetricComponent,
+)
+from datajunction_server.sql.decompose import ComponentDef
+from datajunction_server.sql.parsing import ast
+
+
+def _column(name: str = "latency_tdigest_ab12") -> ast.Column:
+ return ast.Column(name=ast.Name(name))
+
+
+class TestBuildMergeCall:
+ """The helper both emission sites use."""
+
+ def test_no_args_is_the_plain_single_argument_call(self):
+ # Every non-sketch aggregation takes this path and must be unchanged.
+ expr = build_merge_call("SUM", [], _column("revenue_sum_ab12"))
+ assert str(expr) == "SUM(revenue_sum_ab12)"
+
+ def test_one_fixed_argument_is_appended_after_the_column(self):
+ expr = build_merge_call(
+ "nflx_tdigest_agg",
+ ["CAST(200.0 AS DOUBLE)"],
+ _column(),
+ )
+ assert str(expr) == (
+ "nflx_tdigest_agg(latency_tdigest_ab12, CAST(200.0 AS DOUBLE))"
+ )
+
+ def test_several_fixed_arguments_keep_their_order(self):
+ expr = build_merge_call("sketch_merge", ["12", "'HLL_4'"], _column("s_ab12"))
+ assert str(expr) == "sketch_merge(s_ab12, 12, 'HLL_4')"
+
+ def test_the_column_is_always_the_first_argument(self):
+ expr = build_merge_call("f", ["1"], _column("c"))
+ assert isinstance(expr, ast.Function)
+ assert str(expr.args[0]) == "c"
+
+ def test_the_merge_name_is_preserved_verbatim(self):
+ # Matched on elsewhere, so casing must not be normalized.
+ expr = build_merge_call("nflx_tdigest_agg", ["1.0"], _column("c"))
+ assert expr.name.name == "nflx_tdigest_agg"
+
+ def test_arguments_are_parsed_not_pasted(self):
+ # A literal arrives as an expression node, not raw text, so it renders
+ # through the same dialect machinery as the rest of the tree.
+ expr = build_merge_call("f", ["1 + 2"], _column("c"))
+ assert isinstance(expr.args[1], ast.Expression)
+ assert str(expr) == "f(c, 1 + 2)"
+
+
+class TestComponentDefDefaults:
+ """Existing decompositions must be untouched by the new field."""
+
+ def test_merge_args_defaults_to_empty(self):
+ assert ComponentDef(suffix="_sum", accumulate="SUM", merge="SUM").merge_args == ()
+
+ def test_merge_args_is_a_tuple_so_the_default_cannot_be_mutated(self):
+ first = ComponentDef(suffix="_sum", accumulate="SUM", merge="SUM")
+ second = ComponentDef(suffix="_cnt", accumulate="COUNT", merge="SUM")
+ assert first.merge_args == second.merge_args == ()
+ assert isinstance(first.merge_args, tuple)
+
+ def test_merge_args_is_declared_on_the_component_def(self):
+ comp = ComponentDef(
+ suffix="_tdigest",
+ accumulate="nflx_tdigest({}, CAST(200.0 AS DOUBLE))",
+ merge="nflx_tdigest_agg",
+ merge_args=("CAST(200.0 AS DOUBLE)",),
+ )
+ assert comp.merge == "nflx_tdigest_agg"
+ assert comp.merge_args == ("CAST(200.0 AS DOUBLE)",)
+
+
+class TestMetricComponentField:
+ """`merge_args` survives the model, which is dumped to JSON in cube configs."""
+
+ def _component(self, merge_args):
+ return MetricComponent(
+ name="latency_tdigest_ab12",
+ expression="latency_ms",
+ aggregation="nflx_tdigest",
+ merge="nflx_tdigest_agg",
+ merge_args=merge_args,
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+
+ def test_defaults_to_empty_list(self):
+ component = MetricComponent(
+ name="revenue_sum_ab12",
+ expression="revenue",
+ aggregation="SUM",
+ merge="SUM",
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+ assert component.merge_args == []
+
+ def test_round_trips_through_json(self):
+ # A list rather than a tuple: these models are serialized straight into
+ # materialization configs.
+ component = self._component(["CAST(200.0 AS DOUBLE)"])
+ assert component.model_dump()["merge_args"] == ["CAST(200.0 AS DOUBLE)"]
+
+ def test_feeds_the_merge_call(self):
+ component = self._component(["CAST(200.0 AS DOUBLE)"])
+ expr = build_merge_call(
+ component.merge,
+ component.merge_args,
+ _column(component.name),
+ )
+ assert str(expr) == (
+ "nflx_tdigest_agg(latency_tdigest_ab12, CAST(200.0 AS DOUBLE))"
+ )
diff --git a/datajunction-server/tests/construction/build_v3/metrics_sql_test.py b/datajunction-server/tests/construction/build_v3/metrics_sql_test.py
index 697457e2d..847053723 100644
--- a/datajunction-server/tests/construction/build_v3/metrics_sql_test.py
+++ b/datajunction-server/tests/construction/build_v3/metrics_sql_test.py
@@ -1,3 +1,5 @@
+import time
+
import pytest
from . import assert_sql_equal
@@ -99,6 +101,651 @@ async def test_simple_single_metric(self, client_with_build_v3):
},
]
+ @staticmethod
+ async def _create_daily_balance_metric(client_with_build_v3):
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.daily_balance",
+ "description": "Semi-additive balance measured by order date",
+ "query": "SELECT SUM(line_total) FROM v3.order_details",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "v3.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ @staticmethod
+ async def _create_scaled_daily_balance_metric(client_with_build_v3):
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.scaled_daily_balance",
+ "description": "Scaled semi-additive balance measured by order date",
+ "query": "SELECT SUM(line_total) / 100 FROM v3.order_details",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "v3.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ @staticmethod
+ async def _create_first_daily_balance_metric(client_with_build_v3):
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.first_daily_balance",
+ "description": "Semi-additive balance measured by first order date",
+ "query": "SELECT SUM(line_total) FROM v3.order_details",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "v3.date.date_id[order]",
+ "fn": "first_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ @staticmethod
+ async def _create_daily_balance_index_metric(client_with_build_v3):
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.daily_balance_index",
+ "description": "Derived semi-additive balance index",
+ "query": "SELECT 10.0 / v3.daily_balance",
+ "mode": "published",
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ @staticmethod
+ async def _create_wow_daily_balance_index_metric(client_with_build_v3):
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.wow_daily_balance_index",
+ "description": "Week-over-week daily balance index change",
+ "query": """
+ SELECT
+ (v3.daily_balance_index - LAG(v3.daily_balance_index, 1)
+ OVER (ORDER BY v3.date.week[order]))
+ / NULLIF(
+ LAG(v3.daily_balance_index, 1)
+ OVER (ORDER BY v3.date.week[order]),
+ 0
+ ) * 100
+ """,
+ "mode": "published",
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ @staticmethod
+ async def _create_product_balance_metric(client_with_build_v3):
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.product_balance",
+ "description": "Semi-additive balance protected by product",
+ "query": "SELECT SUM(line_total) FROM v3.order_details",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "v3.product.product_id",
+ "fn": "last_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ @staticmethod
+ async def _create_product_balance_index_metric(client_with_build_v3):
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.product_balance_index",
+ "description": "Derived semi-additive product balance index",
+ "query": "SELECT 10.0 / v3.product_balance",
+ "mode": "published",
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ @staticmethod
+ async def _create_wow_product_balance_index_metric(client_with_build_v3):
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.wow_product_balance_index",
+ "description": "Week-over-week product balance index change",
+ "query": """
+ SELECT
+ (v3.product_balance_index - LAG(v3.product_balance_index, 1)
+ OVER (ORDER BY v3.date.week[order]))
+ / NULLIF(
+ LAG(v3.product_balance_index, 1)
+ OVER (ORDER BY v3.date.week[order]),
+ 0
+ ) * 100
+ """,
+ "mode": "published",
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_collapses_when_protected_dimension_omitted(
+ self,
+ client_with_build_v3,
+ ):
+ """A semi-additive metric keeps its protected dimension as private grain."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ assert_sql_equal(
+ response.json()["sql"],
+ """
+ WITH
+ v3_order_details AS (
+ SELECT o.order_date, oi.product_id, oi.quantity * oi.unit_price AS line_total
+ FROM default.v3.orders o
+ JOIN default.v3.order_items oi ON o.order_id = oi.order_id
+ ),
+ v3_product AS (
+ SELECT product_id, category
+ FROM default.v3.products
+ ),
+ order_details_0 AS (
+ SELECT t2.category,
+ t1.order_date AS date_id_order,
+ SUM(t1.line_total) AS line_total_sum_e1f61696
+ FROM v3_order_details t1
+ LEFT OUTER JOIN v3_product t2 ON t1.product_id = t2.product_id
+ GROUP BY t2.category, t1.order_date
+ )
+ SELECT order_details_0.category AS category,
+ MAX_BY(
+ order_details_0.line_total_sum_e1f61696,
+ order_details_0.date_id_order
+ ) AS daily_balance
+ FROM order_details_0
+ GROUP BY order_details_0.category
+ """,
+ )
+ assert response.json()["columns"] == [
+ {
+ "name": "category",
+ "type": "string",
+ "semantic_entity": "v3.product.category",
+ "semantic_type": "dimension",
+ },
+ {
+ "name": "daily_balance",
+ "type": "double",
+ "semantic_entity": "v3.daily_balance",
+ "semantic_type": "metric",
+ },
+ ]
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_with_limited_metric_keeps_protected_grain(
+ self,
+ client_with_build_v3,
+ ):
+ """Merged metric queries must not add limited grain to the collapse CTE."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance", "v3.order_count"],
+ "dimensions": ["v3.product.category"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ sql = response.json()["sql"]
+ normalized_sql = " ".join(sql.split())
+ assert "order_details_0 AS" in sql
+ assert "GROUP BY t2.category, t1.order_date" in normalized_sql
+ assert "GROUP BY t2.category, t1.order_date, t1.order_id" not in normalized_sql
+ assert "order_details_1_agg AS" in sql
+ assert "COUNT( DISTINCT order_id)" in normalized_sql
+ assert "MAX_BY(" in sql
+ assert "order_details_0.date_id_order" in sql
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_preserves_single_component_combiner_wrapper(
+ self,
+ client_with_build_v3,
+ ):
+ """Semi-additive collapse preserves arithmetic around the aggregate."""
+ await self._create_scaled_daily_balance_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.scaled_daily_balance"],
+ "dimensions": ["v3.product.category"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ sql = response.json()["sql"]
+ normalized_sql = " ".join(sql.split())
+ assert "MAX_BY(" in sql
+ assert "order_details_0.date_id_order" in sql
+ assert "/ 100 AS scaled_daily_balance" in normalized_sql
+ assert (
+ "MAX_BY(order_details_0.line_total_sum_e1f61696, order_details_0.date_id_order) / 100"
+ in normalized_sql
+ )
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_druid_uses_latest_by_for_last_value(
+ self,
+ client_with_build_v3,
+ ):
+ """Druid live SQL uses Druid-native collapse functions."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ "dialect": "druid",
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+ assert response.json()["dialect"] == "druid"
+
+ assert_sql_equal(
+ response.json()["sql"],
+ """
+ WITH
+ v3_order_details AS (
+ SELECT o.order_date, oi.product_id, oi.quantity * oi.unit_price AS line_total
+ FROM default.v3.orders o
+ JOIN default.v3.order_items oi ON o.order_id = oi.order_id
+ ),
+ v3_product AS (
+ SELECT product_id, category
+ FROM default.v3.products
+ ),
+ order_details_0 AS (
+ SELECT t2.category,
+ t1.order_date AS date_id_order,
+ SUM(t1.line_total) AS line_total_sum_e1f61696
+ FROM v3_order_details t1
+ LEFT OUTER JOIN v3_product t2 ON t1.product_id = t2.product_id
+ GROUP BY t2.category, t1.order_date
+ )
+ SELECT order_details_0.category AS category,
+ LATEST_BY(
+ order_details_0.line_total_sum_e1f61696,
+ order_details_0.date_id_order
+ ) AS daily_balance
+ FROM order_details_0
+ GROUP BY order_details_0.category
+ """,
+ )
+ assert "MAX_BY" not in response.json()["sql"]
+ assert "ARG_MAX" not in response.json()["sql"]
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_auto_routed_druid_uses_latest_by(
+ self,
+ client_with_build_v3,
+ ):
+ """Auto-routed Druid cube SQL uses Druid-native collapse functions."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.post(
+ "/nodes/cube/",
+ json={
+ "name": "v3.daily_balance_category_cube",
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ "mode": "published",
+ "description": "Category-only decoy cube",
+ },
+ )
+ assert response.status_code == 201, response.json()
+
+ response = await client_with_build_v3.post(
+ "/data/v3.daily_balance_category_cube/availability/",
+ json={
+ "catalog": "default",
+ "schema_": "analytics",
+ "table": "daily_balance_category_cube",
+ "valid_through_ts": int(time.time() * 1000),
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ response = await client_with_build_v3.post(
+ "/nodes/cube/",
+ json={
+ "name": "v3.daily_balance_cube",
+ "metrics": ["v3.daily_balance"],
+ "dimensions": [
+ "v3.product.category",
+ "v3.date.date_id[order]",
+ ],
+ "mode": "published",
+ "description": "Daily balance cube at protected grain",
+ },
+ )
+ assert response.status_code == 201, response.json()
+
+ response = await client_with_build_v3.post(
+ "/data/v3.daily_balance_cube/availability/",
+ json={
+ "catalog": "default",
+ "schema_": "analytics",
+ "table": "daily_balance_cube",
+ "valid_through_ts": int(time.time() * 1000),
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ },
+ )
+ assert response.status_code == 200, response.json()
+ assert response.json()["dialect"] == "druid"
+
+ sql = response.json()["sql"]
+ assert "FROM daily_balance_cube" in sql
+ assert "daily_balance_category_cube" not in sql
+ assert "LATEST_BY(" in sql
+ assert "date_id_order" in sql
+ assert "MAX_BY" not in sql
+ assert "ARG_MAX" not in sql
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_druid_uses_earliest_by_for_first_value(
+ self,
+ client_with_build_v3,
+ ):
+ """Druid first-value collapse renders as EARLIEST_BY."""
+ await self._create_first_daily_balance_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.first_daily_balance"],
+ "dimensions": ["v3.product.category"],
+ "dialect": "druid",
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ sql = response.json()["sql"]
+ assert "EARLIEST_BY(" in sql
+ assert "MIN_BY" not in sql
+ assert "ARG_MIN" not in sql
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_derived_metric_uses_collapsed_base(
+ self,
+ client_with_build_v3,
+ ):
+ """Derived metrics inline semi-additive collapse in denominator position."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+ await self._create_daily_balance_index_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance_index"],
+ "dimensions": ["v3.product.category"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ sql = response.json()["sql"]
+ assert "MAX_BY(" in sql
+ assert "order_details_0.date_id_order" in sql
+ assert "MAX_BY" in sql.split(" AS daily_balance_index")[0]
+ assert "10.0 / NULLIF(MAX_BY(" in sql
+ assert "10.0 / NULLIF(SUM(" not in sql
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_derived_metric_preserves_requested_protected_dimension(
+ self,
+ client_with_build_v3,
+ ):
+ """Derived metrics use normal aggregation when the protected dimension is requested."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+ await self._create_daily_balance_index_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance_index"],
+ "dimensions": ["v3.date.date_id[order]"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ sql = response.json()["sql"]
+ assert "10.0 / NULLIF(SUM(" in sql
+ assert "date_id_order" in sql
+ assert "MAX_BY(" not in sql
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_nested_window_metric_reaggregates_with_collapse(
+ self,
+ client_with_build_v3,
+ ):
+ """Window reaggregation of derived metrics uses semi-additive parents."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+ await self._create_daily_balance_index_metric(client_with_build_v3)
+ await self._create_wow_daily_balance_index_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.wow_daily_balance_index"],
+ "dimensions": ["v3.product.category"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ sql = response.json()["sql"]
+ assert "base_metrics AS" in sql
+ assert "MAX_BY(" in sql
+ assert "order_details_0.date_id_order" in sql
+ assert "10.0 / NULLIF(MAX_BY(" in sql
+ assert "LAG(base_metrics.daily_balance_index, 1)" in sql
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_window_grain_reaggregation_uses_collapse(
+ self,
+ client_with_build_v3,
+ ):
+ """Window aggregation CTEs collapse semi-additive derived parents."""
+ await self._create_product_balance_metric(client_with_build_v3)
+ await self._create_product_balance_index_metric(client_with_build_v3)
+ await self._create_wow_product_balance_index_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.wow_product_balance_index"],
+ "dimensions": ["v3.date.date_id[order]"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ sql = response.json()["sql"]
+ assert "order_details_week_agg AS" in sql
+ assert "MAX_BY(" in sql
+ assert "product_id" in sql
+ assert "10.0 / NULLIF(MAX_BY(" in sql
+ assert "10.0 / NULLIF(SUM(" not in sql
+ assert "LAG(order_details_week_agg.product_balance_index, 1)" in sql
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_uses_normal_aggregation_when_protected_dimension_requested(
+ self,
+ client_with_build_v3,
+ ):
+ """Requesting the protected dimension means there is nothing to collapse."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.date.date_id[order]"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ assert_sql_equal(
+ response.json()["sql"],
+ """
+ WITH
+ v3_order_details AS (
+ SELECT o.order_date, oi.quantity * oi.unit_price AS line_total
+ FROM default.v3.orders o
+ JOIN default.v3.order_items oi ON o.order_id = oi.order_id
+ ),
+ order_details_0 AS (
+ SELECT t1.order_date AS date_id_order,
+ SUM(t1.line_total) AS line_total_sum_e1f61696
+ FROM v3_order_details t1
+ GROUP BY t1.order_date
+ )
+ SELECT order_details_0.date_id_order AS date_id_order,
+ SUM(order_details_0.line_total_sum_e1f61696) AS daily_balance
+ FROM order_details_0
+ GROUP BY order_details_0.date_id_order
+ """,
+ )
+ assert "MAX_BY" not in response.json()["sql"]
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_collapses_when_roleless_base_dimension_requested(
+ self,
+ client_with_build_v3,
+ ):
+ """A role-less dimension request does not satisfy a roled protected grain."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.date.date_id"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ sql = response.json()["sql"]
+ assert "MAX_BY(" in sql
+ assert "order_details_0.date_id_order" in sql
+ assert (
+ "SUM(order_details_0.line_total_sum_e1f61696) AS daily_balance" not in sql
+ )
+
+ @pytest.mark.asyncio
+ async def test_reaggregate_collapses_with_coarser_time_dimension(
+ self,
+ client_with_build_v3,
+ ):
+ """Coarser time output still collapses across the protected date grain."""
+ await self._create_daily_balance_metric(client_with_build_v3)
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.date.month[order]"],
+ "use_materialized": "false",
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ assert_sql_equal(
+ response.json()["sql"],
+ """
+ WITH
+ v3_date AS (
+ SELECT date_id, month
+ FROM default.v3.dates
+ ),
+ v3_order_details AS (
+ SELECT o.order_date, oi.quantity * oi.unit_price AS line_total
+ FROM default.v3.orders o
+ JOIN default.v3.order_items oi ON o.order_id = oi.order_id
+ ),
+ order_details_0 AS (
+ SELECT t2.month AS month_order,
+ COALESCE(t1.order_date, t2.date_id) AS date_id_order,
+ SUM(t1.line_total) AS line_total_sum_e1f61696
+ FROM v3_order_details t1
+ LEFT OUTER JOIN v3_date t2 ON t1.order_date = t2.date_id
+ GROUP BY t2.month, COALESCE(t1.order_date, t2.date_id)
+ )
+ SELECT order_details_0.month_order AS month_order,
+ MAX_BY(
+ order_details_0.line_total_sum_e1f61696,
+ order_details_0.date_id_order
+ ) AS daily_balance
+ FROM order_details_0
+ GROUP BY order_details_0.month_order
+ """,
+ )
+
@pytest.mark.asyncio
async def test_multiple_metrics_same_grain(self, client_with_build_v3):
"""
@@ -3763,15 +4410,27 @@ async def test_cross_fact_window_metric_with_finer_grain(
CAST(COUNT( DISTINCT order_details_0.order_id) AS DOUBLE) / NULLIF(COUNT( DISTINCT page_views_enriched_0.customer_id), 0) AS conversion_rate
FROM order_details_0 FULL OUTER JOIN page_views_enriched_0 ON order_details_0.date_id = page_views_enriched_0.date_id AND order_details_0.category = page_views_enriched_0.category AND order_details_0.week = page_views_enriched_0.week
GROUP BY 1, 2, 3
+ ),
+ order_details_week_agg AS (
+ SELECT base_metrics.category AS category,
+ base_metrics.week AS week,
+ CAST(SUM(base_metrics.order_count) AS DOUBLE) / NULLIF(SUM(base_metrics.visitor_count), 0) AS conversion_rate
+ FROM base_metrics
+ GROUP BY base_metrics.category, base_metrics.week
+ ),
+ order_details_week AS (
+ SELECT order_details_week_agg.category AS category,
+ order_details_week_agg.week AS week,
+ (order_details_week_agg.conversion_rate - LAG(order_details_week_agg.conversion_rate, 1) OVER ( PARTITION BY order_details_week_agg.category
+ ORDER BY order_details_week_agg.week) ) / NULLIF(LAG(order_details_week_agg.conversion_rate, 1) OVER ( PARTITION BY order_details_week_agg.category
+ ORDER BY order_details_week_agg.week) , 0) * 100 AS wow_conversion_rate_change
+ FROM order_details_week_agg
)
-
SELECT base_metrics.date_id AS date_id,
base_metrics.category AS category,
base_metrics.week AS week,
- (base_metrics.conversion_rate - LAG(base_metrics.conversion_rate, 1) OVER ( PARTITION BY base_metrics.category
- ORDER BY base_metrics.week) ) / NULLIF(LAG(base_metrics.conversion_rate, 1) OVER ( PARTITION BY base_metrics.category
- ORDER BY base_metrics.week) , 0) * 100 AS wow_conversion_rate_change
- FROM base_metrics
+ order_details_week.wow_conversion_rate_change AS wow_conversion_rate_change
+ FROM base_metrics LEFT OUTER JOIN order_details_week ON base_metrics.category = order_details_week.category AND base_metrics.week = order_details_week.week
""",
)
@@ -3877,15 +4536,27 @@ async def test_cross_fact_window_on_derived_metric(self, client_with_build_v3):
SUM(page_views_enriched_0.view_id_count_f41e2db4) / NULLIF(COUNT( DISTINCT page_views_enriched_0.session_id), 0) AS pages_per_session
FROM order_details_0 FULL OUTER JOIN page_views_enriched_0 ON order_details_0.date_id = page_views_enriched_0.date_id AND order_details_0.category = page_views_enriched_0.category AND order_details_0.week = page_views_enriched_0.week
GROUP BY 1, 2, 3
+ ),
+ order_details_week_agg AS (
+ SELECT base_metrics.category AS category,
+ base_metrics.week AS week,
+ SUM(base_metrics.total_revenue) / NULLIF(SUM(base_metrics.order_count), 0) / NULLIF(SUM(base_metrics.page_view_count) / NULLIF(SUM(base_metrics.session_count), 0), 0) AS efficiency_ratio
+ FROM base_metrics
+ GROUP BY base_metrics.category, base_metrics.week
+ ),
+ order_details_week AS (
+ SELECT order_details_week_agg.category AS category,
+ order_details_week_agg.week AS week,
+ (order_details_week_agg.efficiency_ratio - LAG(order_details_week_agg.efficiency_ratio, 1) OVER ( PARTITION BY order_details_week_agg.category
+ ORDER BY order_details_week_agg.week) ) / NULLIF(LAG(order_details_week_agg.efficiency_ratio, 1) OVER ( PARTITION BY order_details_week_agg.category
+ ORDER BY order_details_week_agg.week) , 0) * 100 AS wow_efficiency_ratio_change
+ FROM order_details_week_agg
)
-
SELECT base_metrics.date_id AS date_id,
base_metrics.category AS category,
base_metrics.week AS week,
- (base_metrics.efficiency_ratio - LAG(base_metrics.efficiency_ratio, 1) OVER ( PARTITION BY base_metrics.category
- ORDER BY base_metrics.week) ) / NULLIF(LAG(base_metrics.efficiency_ratio, 1) OVER ( PARTITION BY base_metrics.category
- ORDER BY base_metrics.week) , 0) * 100 AS wow_efficiency_ratio_change
- FROM base_metrics""",
+ order_details_week.wow_efficiency_ratio_change AS wow_efficiency_ratio_change
+ FROM base_metrics LEFT OUTER JOIN order_details_week ON base_metrics.category = order_details_week.category AND base_metrics.week = order_details_week.week""",
)
@pytest.mark.asyncio
@@ -3989,8 +4660,8 @@ async def test_cross_fact_window_on_base_metrics(self, client_with_build_v3):
order_details_week_agg AS (
SELECT base_metrics.category AS category,
base_metrics.week AS week,
- COUNT( DISTINCT base_metrics.order_id) AS order_count,
- COUNT( DISTINCT base_metrics.customer_id) AS visitor_count
+ SUM(base_metrics.order_count) AS order_count,
+ SUM(base_metrics.visitor_count) AS visitor_count
FROM base_metrics
GROUP BY base_metrics.category, base_metrics.week
),
diff --git a/datajunction-server/tests/construction/build_v3/preagg_matcher_test.py b/datajunction-server/tests/construction/build_v3/preagg_matcher_test.py
index 294428b82..e88dfb1ec 100644
--- a/datajunction-server/tests/construction/build_v3/preagg_matcher_test.py
+++ b/datajunction-server/tests/construction/build_v3/preagg_matcher_test.py
@@ -32,6 +32,7 @@
from datajunction_server.database.preaggregation import (
PreAggregation,
compute_expression_hash,
+ measure_identity_token,
)
from datajunction_server.database.user import User
from datajunction_server.errors import DJInvalidInputException
@@ -220,7 +221,7 @@ async def test_returns_identities_for_all_components(
parent_node: Node,
metric_node: Node,
):
- """Should return an (expression hash, aggregation) pair per component."""
+ """Should return one identity token per component."""
components = [
(metric_node, make_component("sum_revenue", "price * quantity")),
(metric_node, make_component("sum_quantity", "quantity")),
@@ -230,8 +231,8 @@ async def test_returns_identities_for_all_components(
identities = get_required_measure_identities(grain_group)
assert identities == {
- (compute_expression_hash("price * quantity"), "SUM"),
- (compute_expression_hash("quantity"), "SUM"),
+ measure_identity_token(compute_expression_hash("price * quantity"), "SUM"),
+ measure_identity_token(compute_expression_hash("quantity"), "SUM"),
}
@pytest.mark.asyncio
@@ -264,7 +265,9 @@ async def test_deduplicates_same_expression_and_aggregation(
identities = get_required_measure_identities(grain_group)
- assert identities == {(compute_expression_hash("price * quantity"), "SUM")}
+ assert identities == {
+ measure_identity_token(compute_expression_hash("price * quantity"), "SUM"),
+ }
@pytest.mark.asyncio
async def test_same_expression_different_aggregation_are_distinct(
@@ -294,8 +297,8 @@ async def test_same_expression_different_aggregation_are_distinct(
identities = get_required_measure_identities(grain_group)
assert identities == {
- (compute_expression_hash("unit_price"), "SUM"),
- (compute_expression_hash("unit_price"), "MAX"),
+ measure_identity_token(compute_expression_hash("unit_price"), "SUM"),
+ measure_identity_token(compute_expression_hash("unit_price"), "MAX"),
}
diff --git a/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py b/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py
index 0edd7753e..f8a045a70 100644
--- a/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py
+++ b/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py
@@ -86,6 +86,28 @@ async def _fake_columns(*args, **kwargs):
del client.app.dependency_overrides[get_query_service_client]
+async def _create_daily_balance_metric(client_with_build_v3):
+ """Create a semi-additive metric protected by order date."""
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": "v3.daily_balance",
+ "description": "Semi-additive balance measured by order date",
+ "query": "SELECT SUM(line_total) FROM v3.order_details",
+ "mode": "published",
+ "reaggregate": {
+ "rules": [
+ {
+ "dimension": "v3.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+
+
class TestExternalPreAggRouting:
"""Queries route to externally-registered pre-agg tables via source_column."""
@@ -198,6 +220,98 @@ async def test_external_preagg_rolls_up_additive(self, client_with_build_v3):
""",
)
+ @pytest.mark.asyncio
+ async def test_external_preagg_retaining_reaggregate_dimension_rolls_up(
+ self,
+ client_with_build_v3,
+ ):
+ """A semi-additive metric can read a fine pre-agg that still has the
+ protected dimension, then collapse in the final metrics query."""
+ await _create_daily_balance_metric(client_with_build_v3)
+ await _register_external_preagg(
+ client_with_build_v3,
+ metrics=["v3.daily_balance"],
+ dimensions=[
+ "v3.product.category",
+ "v3.date.date_id[order]",
+ ],
+ table_ref={
+ "catalog": "default",
+ "schema": "analytics",
+ "table": "daily_balance_by_category_day",
+ "valid_through_ts": 20250101,
+ },
+ measure_columns={"v3.daily_balance": "balance_sum"},
+ dimension_columns={"v3.date.date_id[order]": "order_day"},
+ table_columns={
+ "category": "string",
+ "order_day": "int",
+ "balance_sum": "double",
+ },
+ )
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ },
+ )
+ assert response.status_code == 200, response.json()
+ assert_sql_equal(
+ response.json()["sql"],
+ """
+ WITH order_details_0 AS (
+ SELECT category,
+ order_day date_id_order,
+ SUM(balance_sum) balance_sum
+ FROM default.analytics.daily_balance_by_category_day
+ GROUP BY category, order_day
+ )
+ SELECT order_details_0.category AS category,
+ MAX_BY(order_details_0.balance_sum, order_details_0.date_id_order)
+ AS daily_balance
+ FROM order_details_0
+ GROUP BY order_details_0.category
+ """,
+ )
+
+ @pytest.mark.asyncio
+ async def test_external_preagg_missing_reaggregate_dimension_is_not_used(
+ self,
+ client_with_build_v3,
+ ):
+ """A semi-additive metric must not read a pre-agg that dropped the key
+ required by the final collapse."""
+ await _create_daily_balance_metric(client_with_build_v3)
+ await _register_external_preagg(
+ client_with_build_v3,
+ metrics=["v3.daily_balance"],
+ dimensions=["v3.product.category"],
+ table_ref={
+ "catalog": "default",
+ "schema": "analytics",
+ "table": "daily_balance_by_category",
+ "valid_through_ts": 20250101,
+ },
+ measure_columns={"v3.daily_balance": "balance_sum"},
+ table_columns={"category": "string", "balance_sum": "double"},
+ )
+
+ response = await client_with_build_v3.get(
+ "/sql/metrics/v3/",
+ params={
+ "metrics": ["v3.daily_balance"],
+ "dimensions": ["v3.product.category"],
+ },
+ )
+ assert response.status_code == 200, response.json()
+
+ sql = response.json()["sql"]
+ assert "default.analytics.daily_balance_by_category" not in sql
+ assert "MAX_BY(" in sql
+ assert "date_id_order" in sql
+
@pytest.mark.asyncio
async def test_external_non_additive_not_rolled_up(self, client_with_build_v3):
"""A non-additive measure (COUNT DISTINCT) does not roll up to a coarser
diff --git a/datajunction-server/tests/construction/build_v3/reaggregate_coverage_test.py b/datajunction-server/tests/construction/build_v3/reaggregate_coverage_test.py
new file mode 100644
index 000000000..358f14d11
--- /dev/null
+++ b/datajunction-server/tests/construction/build_v3/reaggregate_coverage_test.py
@@ -0,0 +1,764 @@
+"""Focused coverage tests for build-v3 reaggregation helpers."""
+
+from types import SimpleNamespace
+
+import pytest
+
+from datajunction_server.construction.build_v3.cube_matcher import (
+ _cube_dimension_covers_reaggregate_dimension,
+ _metric_graph_has_reaggregate,
+ _reaggregate_dimensions_for_cube_metrics,
+ _reaggregate_requirements_for_cube_metrics,
+ _reaggregate_requirements_for_metrics,
+ _reaggregate_requirements_for_decomposed_metrics,
+ _reaggregate_requirements_for_metrics_if_needed,
+ build_synthetic_grain_group,
+)
+from datajunction_server.construction.build_v3 import (
+ cube_matcher as cube_matcher_module,
+)
+from datajunction_server.construction.build_v3 import builder as builder_module
+from datajunction_server.construction.build_v3 import metrics as metrics_module
+from datajunction_server.construction.build_v3.builder import setup_build_context
+from datajunction_server.construction.build_v3.measures import (
+ build_grain_group_sql,
+ build_select_ast,
+ build_window_metric_grain_groups,
+)
+from datajunction_server.construction.build_v3.metrics import (
+ _build_reaggregate_collapse_expression,
+ _dimension_ref_base,
+ _dimension_ref_role,
+ _references_component,
+ _replace_reaggregate_merge_expression,
+ _metric_parent_refs,
+ _source_dimension_alias,
+ build_window_agg_cte_from_base_metrics,
+ generate_metrics_sql,
+)
+from datajunction_server.construction.build_v3.types import (
+ BuildContext,
+ ColumnMetadata,
+ DecomposedMetricInfo,
+ GeneratedMeasuresSQL,
+ GrainGroup,
+ GrainGroupSQL,
+ ResolvedDimension,
+)
+from datajunction_server.errors import DJInvalidInputException
+from datajunction_server.models.decompose import (
+ Aggregability,
+ AggregationRule,
+ MetricComponent,
+)
+from datajunction_server.models.dialect import Dialect
+from datajunction_server.models.node import NodeType
+from datajunction_server.models.reaggregate import (
+ DimensionReaggregateRule,
+ ReaggregationFunction,
+)
+from datajunction_server.sql.parsing import ast
+from datajunction_server.sql.parsing.backends.antlr4 import parse
+from datajunction_server.sql.parsing.types import BigIntType, DoubleType, StringType
+
+
+def _metric_node(name: str, query: str = "SELECT SUM(value) FROM test.parent"):
+ return SimpleNamespace(
+ name=name,
+ type=NodeType.METRIC,
+ current=SimpleNamespace(
+ query=query,
+ columns=[SimpleNamespace(type=DoubleType())],
+ ),
+ )
+
+
+def _source_node(name: str = "test.parent"):
+ return SimpleNamespace(
+ name=name,
+ type=NodeType.SOURCE,
+ current=SimpleNamespace(
+ catalog=SimpleNamespace(name="default"),
+ schema_="analytics",
+ table="parent",
+ columns=[
+ SimpleNamespace(name="id", type=BigIntType()),
+ SimpleNamespace(name="value", type=DoubleType()),
+ SimpleNamespace(name="region", type=StringType()),
+ SimpleNamespace(name="date_id", type=BigIntType()),
+ ],
+ ),
+ )
+
+
+def _semi_additive_component(
+ name: str = "balance_sum",
+ dimension: str = "v3.date.date_id",
+ fn: ReaggregationFunction = ReaggregationFunction.LAST_VALUE,
+) -> MetricComponent:
+ return MetricComponent(
+ name=name,
+ expression="value",
+ aggregation="SUM",
+ rule=AggregationRule(
+ type=Aggregability.FULL,
+ reaggregate=DimensionReaggregateRule(
+ dimension=dimension,
+ fn=fn,
+ ),
+ ),
+ )
+
+
+def _decomposed_metric(
+ metric_name: str,
+ component: MetricComponent | None = None,
+ query: str = "SELECT SUM(value) FROM test.parent",
+) -> DecomposedMetricInfo:
+ return DecomposedMetricInfo(
+ metric_node=_metric_node(metric_name, query),
+ components=[component or _semi_additive_component()],
+ aggregability=Aggregability.FULL,
+ combiner=query,
+ derived_ast=parse(query),
+ )
+
+
+def test_reaggregate_collapse_expression_handles_min_max_and_rejects_unsupported():
+ """MIN/MAX collapse ignore the protected dimension; unsupported functions fail."""
+ value_ref = ast.Column(name=ast.Name("balance"))
+ protected_ref = ast.Column(name=ast.Name("date_id"))
+
+ assert (
+ str(
+ _build_reaggregate_collapse_expression(
+ ReaggregationFunction.MIN,
+ Dialect.SPARK,
+ value_ref,
+ protected_ref,
+ ),
+ )
+ == "MIN(balance)"
+ )
+ assert (
+ str(
+ _build_reaggregate_collapse_expression(
+ ReaggregationFunction.MAX,
+ Dialect.SPARK,
+ value_ref,
+ protected_ref,
+ ),
+ )
+ == "MAX(balance)"
+ )
+
+ with pytest.raises(DJInvalidInputException, match="Unsupported semi-additive"):
+ _build_reaggregate_collapse_expression(
+ ReaggregationFunction.SUM,
+ Dialect.SPARK,
+ value_ref,
+ protected_ref,
+ )
+
+
+def test_references_component_checks_all_columns_and_handles_no_match():
+ """Component reference detection handles misses before matches."""
+ expression = parse(
+ "SELECT SUM(other_value) + SUM(balance_sum) FROM base",
+ ).select.projection[0]
+
+ assert _references_component(expression, "balance_sum")
+ assert not _references_component(expression, "missing_component")
+
+
+def test_replace_reaggregate_merge_expression_skips_nonmatching_functions():
+ """Combiner replacement skips unrelated functions before replacing the match."""
+ combiner = parse(
+ "SELECT SUM(other_value) + SUM(balance_sum) FROM base",
+ ).select.projection[0]
+ collapse_expr = ast.Function(
+ ast.Name("MAX"),
+ args=[ast.Column(name=ast.Name("balance_sum"))],
+ )
+
+ rendered = str(
+ _replace_reaggregate_merge_expression(
+ combiner,
+ "balance_sum",
+ "SUM",
+ collapse_expr,
+ ),
+ )
+
+ assert "SUM(other_value)" in rendered
+ assert "MAX(balance_sum)" in rendered
+
+
+def test_replace_reaggregate_merge_expression_rejects_missing_component():
+ """Combiner replacement fails if the expected merge function is absent."""
+ combiner = parse("SELECT AVG(balance_sum) FROM base").select.projection[0]
+
+ with pytest.raises(DJInvalidInputException, match="could not find"):
+ _replace_reaggregate_merge_expression(
+ combiner,
+ "balance_sum",
+ "SUM",
+ ast.Function(
+ ast.Name("MAX"),
+ args=[ast.Column(name=ast.Name("balance_sum"))],
+ ),
+ )
+
+
+def test_replace_reaggregate_merge_expression_skips_parentless_nested_match():
+ """A matching child without a parent is skipped before the final error."""
+
+ class ParentlessExpression(ast.Expression):
+ def __str__(self):
+ return "parentless"
+
+ @property
+ def children(self):
+ yield ast.Function(
+ ast.Name("SUM"),
+ args=[ast.Column(name=ast.Name("balance_sum"))],
+ )
+
+ with pytest.raises(DJInvalidInputException, match="could not find"):
+ _replace_reaggregate_merge_expression(
+ ParentlessExpression(),
+ "balance_sum",
+ "SUM",
+ ast.Function(
+ ast.Name("MAX"),
+ args=[ast.Column(name=ast.Name("balance_sum"))],
+ ),
+ )
+
+
+def test_reaggregate_dimension_ref_helpers_handle_roleless_refs_without_alias():
+ """Roleless refs return None for role and do not require an alias fallback."""
+ ctx = BuildContext(session=SimpleNamespace(), metrics=[], dimensions=[])
+
+ assert _dimension_ref_base("v3.date.date_id[order]") == "v3.date.date_id"
+ assert _dimension_ref_role("v3.date.date_id") is None
+ assert _source_dimension_alias(ctx, "v3.date.date_id") is None
+
+
+def test_metric_parent_refs_adds_metric_refs_discovered_from_query():
+ """Parsed-query metric references supplement the cached parent map."""
+ balance = _metric_node("balance")
+ extra = _metric_node("extra_metric")
+ derived = _metric_node(
+ "derived",
+ "SELECT balance + extra_metric + raw_col FROM test.parent",
+ )
+ ctx = BuildContext(session=SimpleNamespace(), metrics=[], dimensions=[])
+ ctx.nodes = {
+ "balance": balance,
+ "extra_metric": extra,
+ "derived": derived,
+ }
+ ctx.parent_map = {"derived": ["balance"]}
+
+ assert _metric_parent_refs(ctx, "derived") == ["balance", "extra_metric"]
+
+
+def test_window_agg_from_base_metrics_collapses_semi_additive_derived_parent(
+ monkeypatch,
+):
+ """Cross-fact window reaggregation replaces derived metric parents recursively."""
+ monkeypatch.setattr(metrics_module, "get_column_full_name", lambda _col: "")
+ balance = _metric_node("balance")
+ extra = _metric_node("extra_metric")
+ derived = _metric_node(
+ "derived",
+ "SELECT balance + extra_metric + raw_col FROM test.parent",
+ )
+ window = _metric_node("window_metric")
+ ctx = BuildContext(
+ session=SimpleNamespace(),
+ metrics=["window_metric"],
+ dimensions=["category"],
+ )
+ ctx.nodes = {
+ "balance": balance,
+ "extra_metric": extra,
+ "derived": derived,
+ "window_metric": window,
+ }
+ ctx.parent_map = {
+ "window_metric": ["derived"],
+ "derived": ["balance"],
+ }
+ ctx.alias_registry.register("v3.date.date_id")
+
+ query = build_window_agg_cte_from_base_metrics(
+ GrainGroupSQL(
+ query=parse("SELECT category FROM base_metrics"),
+ columns=[
+ ColumnMetadata(
+ name="category",
+ semantic_name="category",
+ type="string",
+ semantic_type="dimension",
+ ),
+ ],
+ grain=["category"],
+ aggregability=Aggregability.FULL,
+ metrics=[],
+ parent_name="cross_fact",
+ is_window_grain_group=True,
+ window_metrics_served=["window_metric"],
+ ),
+ "base_metrics",
+ ctx,
+ {
+ "balance": _decomposed_metric(
+ "balance",
+ _semi_additive_component(fn=ReaggregationFunction.MAX),
+ ),
+ },
+ )
+
+ rendered = str(query)
+ assert "MAX(base_metrics.balance)" in rendered
+ assert "extra_metric" in rendered
+ assert "raw_col" in rendered
+
+
+def test_generate_metrics_sql_rejects_multiple_base_grain_groups_with_reaggregate():
+ """Live SQL does not allow protected-dimension fanout across base groups."""
+ ctx = BuildContext(session=SimpleNamespace(), metrics=["balance"], dimensions=[])
+ grain_groups = [
+ GrainGroupSQL(
+ query=parse("SELECT category, balance_sum FROM fact_a"),
+ columns=[],
+ grain=["category"],
+ aggregability=Aggregability.FULL,
+ metrics=["balance"],
+ parent_name="fact_a",
+ reaggregate_dimension_aliases={"balance_sum": "date_id"},
+ ),
+ GrainGroupSQL(
+ query=parse("SELECT category, orders FROM fact_b"),
+ columns=[],
+ grain=["category"],
+ aggregability=Aggregability.FULL,
+ metrics=["orders"],
+ parent_name="fact_b",
+ ),
+ ]
+ measures_result = GeneratedMeasuresSQL(
+ grain_groups=grain_groups,
+ dialect=Dialect.SPARK,
+ requested_dimensions=[],
+ ctx=ctx,
+ )
+
+ with pytest.raises(DJInvalidInputException, match="multiple base grain groups"):
+ generate_metrics_sql(ctx, measures_result, {})
+
+
+def test_cube_reaggregate_requirements_skip_satisfied_dimensions_and_dedupe():
+ """Cube requirement helpers skip requested protected dims and de-duplicate."""
+ metric_revision = SimpleNamespace(
+ name="balance",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "v3.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ cube = SimpleNamespace(metric_node_revisions=lambda: [metric_revision])
+
+ assert (
+ _reaggregate_requirements_for_cube_metrics(
+ cube,
+ ["balance"],
+ ["v3.date.date_id[order]"],
+ )
+ == []
+ )
+
+ duplicate_cube = SimpleNamespace(
+ metric_node_revisions=lambda: [metric_revision, metric_revision],
+ )
+ assert _reaggregate_dimensions_for_cube_metrics(
+ duplicate_cube,
+ ["balance"],
+ ["v3.product.category"],
+ ) == ["v3.date.date_id[order]"]
+
+
+def test_cube_dimension_coverage_accepts_bare_protected_parent_column():
+ """Cube coverage accepts bare parent-column protected dimensions."""
+ assert _cube_dimension_covers_reaggregate_dimension(
+ "order_date",
+ "v3.order_details.order_date",
+ )
+ assert not _cube_dimension_covers_reaggregate_dimension(
+ "order_date",
+ "v3.order_details.ship_date",
+ )
+ assert not _cube_dimension_covers_reaggregate_dimension(
+ "order_date",
+ "v3.order_details.order_date[ship]",
+ )
+ assert not _cube_dimension_covers_reaggregate_dimension(
+ "v3.date.date_id",
+ "v3.order_details.order_date",
+ )
+
+
+@pytest.mark.asyncio
+async def test_metric_graph_has_reaggregate_skips_empty_metric_list():
+ """An empty metric list does not touch the database."""
+
+ class Session:
+ async def execute(self, _stmt):
+ raise AssertionError("empty metric graph should not query")
+
+ assert not await _metric_graph_has_reaggregate(Session(), [])
+
+
+@pytest.mark.asyncio
+async def test_cube_reaggregate_requirements_skip_full_extract_without_reaggregate(
+ monkeypatch,
+):
+ """No reaggregate in the metric graph skips full component extraction."""
+
+ async def fail_full_extract(*_args, **_kwargs):
+ raise AssertionError("full metric extraction should not run")
+
+ class Result:
+ def all(self):
+ return [("v3.total_revenue", None, None)]
+
+ class Session:
+ async def execute(self, _stmt):
+ return Result()
+
+ monkeypatch.setattr(
+ cube_matcher_module,
+ "_reaggregate_requirements_for_metrics",
+ fail_full_extract,
+ )
+
+ assert (
+ await _reaggregate_requirements_for_metrics_if_needed(
+ Session(),
+ ["v3.total_revenue"],
+ ["v3.product.category"],
+ )
+ == []
+ )
+
+
+@pytest.mark.asyncio
+async def test_cube_reaggregate_requirements_checks_derived_metric_ancestors(
+ monkeypatch,
+):
+ """Derived metrics still use full extraction when an ancestor has reaggregate."""
+ expected = [("v3.balance_index", "v3.date.date_id", ReaggregationFunction.MAX)]
+
+ async def full_extract(*_args, **_kwargs):
+ return expected
+
+ class Result:
+ def __init__(self, rows):
+ self.rows = rows
+
+ def all(self):
+ return self.rows
+
+ class Session:
+ def __init__(self):
+ self.calls = 0
+
+ async def execute(self, _stmt):
+ self.calls += 1
+ if self.calls == 1:
+ return Result([("v3.balance_index", None, "v3.daily_balance")])
+ return Result([("v3.daily_balance", {"rules": []}, None)])
+
+ monkeypatch.setattr(
+ cube_matcher_module,
+ "_reaggregate_requirements_for_metrics",
+ full_extract,
+ )
+
+ assert (
+ await _reaggregate_requirements_for_metrics_if_needed(
+ Session(),
+ ["v3.balance_index"],
+ ["v3.product.category"],
+ )
+ == expected
+ )
+
+
+@pytest.mark.asyncio
+async def test_reaggregate_requirements_skip_components_without_reaggregate(
+ monkeypatch,
+):
+ """Full extraction ignores ordinary components before collecting reaggregate."""
+
+ class Extractor:
+ async def extract(self, _session):
+ return [
+ MetricComponent(
+ name="ordinary_sum",
+ expression="value",
+ aggregation="SUM",
+ rule=AggregationRule(type=Aggregability.FULL),
+ ),
+ _semi_additive_component(
+ "balance_sum",
+ "v3.date.date_id",
+ ReaggregationFunction.MAX,
+ ),
+ ], None
+
+ class FakeMetricComponentExtractor:
+ @classmethod
+ async def from_node_name(cls, _metric_name, _session):
+ return Extractor()
+
+ monkeypatch.setattr(
+ cube_matcher_module,
+ "MetricComponentExtractor",
+ FakeMetricComponentExtractor,
+ )
+
+ assert await _reaggregate_requirements_for_metrics(
+ SimpleNamespace(),
+ ["balance"],
+ ["v3.product.category"],
+ ) == [("balance", "v3.date.date_id", ReaggregationFunction.MAX)]
+
+
+def test_decomposed_reaggregate_requirements_dedupe_duplicate_components():
+ """Duplicate component rules produce one cube materialization requirement."""
+ rule = DimensionReaggregateRule(
+ dimension="v3.date.date_id",
+ fn=ReaggregationFunction.LAST_VALUE,
+ )
+ components = [
+ MetricComponent(
+ name=f"balance_sum_{idx}",
+ expression="value",
+ aggregation="SUM",
+ rule=AggregationRule(type=Aggregability.FULL, reaggregate=rule),
+ )
+ for idx in range(2)
+ ]
+ decomposed = _decomposed_metric("balance", components[0])
+ decomposed.components = components
+
+ assert _reaggregate_requirements_for_decomposed_metrics(
+ {"balance": decomposed},
+ ["balance"],
+ [],
+ ) == [("balance", "v3.date.date_id", ReaggregationFunction.LAST_VALUE)]
+
+
+def test_build_synthetic_grain_group_skips_duplicate_internal_dimension():
+ """Filter-only protected dims can already be present in ctx.dimensions."""
+ ctx = BuildContext(
+ session=SimpleNamespace(),
+ metrics=["balance"],
+ dimensions=["v3.date.date_id"],
+ dialect=Dialect.DRUID,
+ )
+ ctx.filter_dimensions = {"v3.date.date_id"}
+ ctx.nodes = {"balance": _metric_node("balance")}
+ ctx.parent_map = {"balance": ["test.parent"]}
+ cube = SimpleNamespace(
+ name="daily_balance_cube",
+ availability=SimpleNamespace(table="daily_balance_cube"),
+ materializations=[],
+ )
+
+ grain_group = build_synthetic_grain_group(
+ ctx,
+ {"balance": _decomposed_metric("balance")},
+ cube,
+ )
+
+ assert grain_group.grain == ["date_id"]
+ assert [
+ col.name for col in grain_group.columns if col.semantic_type == "dimension"
+ ] == [
+ "date_id",
+ ]
+
+
+def test_build_select_ast_skips_non_output_dimensions():
+ """Resolved dimensions outside output refs are join/group-only."""
+ ctx = BuildContext(session=SimpleNamespace(), metrics=[], dimensions=[])
+ parent = _source_node()
+ resolved_dimensions = [
+ ResolvedDimension(
+ original_ref="test.parent.region",
+ node_name="test.parent",
+ column_name="region",
+ role=None,
+ join_path=None,
+ is_local=True,
+ ),
+ ResolvedDimension(
+ original_ref="test.parent.date_id",
+ node_name="test.parent",
+ column_name="date_id",
+ role=None,
+ join_path=None,
+ is_local=True,
+ ),
+ ]
+
+ query, _ = build_select_ast(
+ ctx,
+ metric_expressions=[],
+ resolved_dimensions=resolved_dimensions,
+ parent_node=parent,
+ output_dimension_refs={"test.parent.region"},
+ )
+
+ rendered = str(query)
+ assert "t1.region" in rendered
+ assert "t1.date_id AS date_id" not in rendered
+
+
+def test_build_grain_group_sql_handles_internal_alias_already_in_grain():
+ """The internal reaggregate alias can match a requested dimension alias."""
+ ctx = BuildContext(
+ session=SimpleNamespace(),
+ metrics=["balance"],
+ dimensions=["test.parent.date_id"],
+ use_materialized=False,
+ )
+ parent = _source_node()
+ metric = _metric_node("balance")
+ component = _semi_additive_component(dimension="test.parent.date_id")
+ resolved_dimension = ResolvedDimension(
+ original_ref="test.parent.date_id",
+ node_name="test.parent",
+ column_name="date_id",
+ role=None,
+ join_path=None,
+ is_local=True,
+ )
+ grain_group = GrainGroup(
+ parent_node=parent,
+ aggregability=Aggregability.FULL,
+ grain_columns=[],
+ components=[(metric, component)],
+ reaggregate_component_dimensions={component.name: "test.parent.date_id"},
+ )
+
+ result = build_grain_group_sql(
+ ctx,
+ grain_group,
+ [resolved_dimension],
+ {"balance": 1},
+ output_dimension_refs={"test.parent.date_id"},
+ )
+
+ assert result.grain == ["date_id"]
+
+
+def test_build_window_metric_grain_groups_skips_missing_base_components():
+ """Window grain planning skips parent groups with no decomposed components."""
+ ctx = BuildContext(session=SimpleNamespace(), metrics=["window"], dimensions=[])
+ ctx.parent_map = {"window": ["missing_base"]}
+ existing_group = GrainGroupSQL(
+ query=parse("SELECT missing_base FROM test_parent_0"),
+ columns=[],
+ grain=[],
+ aggregability=Aggregability.FULL,
+ metrics=["missing_base"],
+ parent_name="test.parent",
+ )
+
+ assert (
+ build_window_metric_grain_groups(
+ ctx,
+ {"window": {"v3.date.week"}},
+ [existing_group],
+ {},
+ )
+ == []
+ )
+
+
+def test_build_window_metric_grain_groups_skips_when_parent_node_missing():
+ """Window grain planning skips groups whose source parent is not loaded."""
+ ctx = BuildContext(session=SimpleNamespace(), metrics=["window"], dimensions=[])
+ ctx.nodes = {"missing_base": _metric_node("missing_base")}
+ ctx.parent_map = {"window": ["missing_base"]}
+ existing_group = GrainGroupSQL(
+ query=parse("SELECT missing_base FROM missing_parent_0"),
+ columns=[],
+ grain=[],
+ aggregability=Aggregability.FULL,
+ metrics=["missing_base"],
+ parent_name="missing.parent",
+ )
+
+ assert (
+ build_window_metric_grain_groups(
+ ctx,
+ {"window": {"v3.date.week"}},
+ [existing_group],
+ {
+ "window": _decomposed_metric("window"),
+ "missing_base": _decomposed_metric("missing_base"),
+ },
+ )
+ == []
+ )
+
+
+@pytest.mark.asyncio
+async def test_setup_build_context_skips_duplicate_internal_reaggregate_dimension(
+ monkeypatch,
+):
+ """setup_build_context does not duplicate internal reaggregate dimensions."""
+ load_calls = []
+
+ async def fake_load_nodes(ctx):
+ load_calls.append(list(ctx.dimensions))
+ ctx.nodes = {"balance": _metric_node("balance")}
+
+ async def fake_decompose_and_group_metrics(ctx):
+ return [], {"balance": _decomposed_metric("balance")}
+
+ monkeypatch.setattr(builder_module, "load_nodes", fake_load_nodes)
+ monkeypatch.setattr(
+ builder_module,
+ "decompose_and_group_metrics",
+ fake_decompose_and_group_metrics,
+ )
+ monkeypatch.setattr(
+ builder_module,
+ "add_dimensions_from_metric_expressions",
+ lambda _ctx, _decomposed_metrics: None,
+ )
+ monkeypatch.setattr(
+ builder_module,
+ "missing_reaggregate_dimensions",
+ lambda _decomposed_metrics, _requested_dimensions: ["v3.date.date_id"],
+ )
+
+ ctx = await setup_build_context(
+ session=SimpleNamespace(),
+ metrics=["balance"],
+ dimensions=["v3.date.date_id"],
+ )
+
+ assert ctx.dimensions == ["v3.date.date_id"]
+ assert load_calls == [["v3.date.date_id"], ["v3.date.date_id"]]
diff --git a/datajunction-server/tests/construction/build_v3/serialize_target_test.py b/datajunction-server/tests/construction/build_v3/serialize_target_test.py
new file mode 100644
index 000000000..ef5bbcf76
--- /dev/null
+++ b/datajunction-server/tests/construction/build_v3/serialize_target_test.py
@@ -0,0 +1,244 @@
+"""
+Tests for the materialization-target serialize hook.
+
+Some components must be written to destinations in a representation different
+from their accumulated form. The fork keys off ``MaterializationTarget`` to
+determine if the expression needs to be serialized during measures table writing.
+"""
+
+import pytest
+
+from types import SimpleNamespace
+
+from datajunction_server.construction.build_v3.decomposition import (
+ build_component_expression,
+)
+from datajunction_server.construction.build_v3.measures import infer_component_type
+from datajunction_server.models.decompose import AggregationRule, MetricComponent
+from datajunction_server.models.materialization import MaterializationTarget
+
+
+def _component(
+ aggregation: str,
+ serialize: str | None = None,
+ serialize_targets: list[MaterializationTarget] | None = None,
+) -> MetricComponent:
+ return MetricComponent(
+ name="latency_td_abc123",
+ expression="latency_ms",
+ aggregation=aggregation,
+ merge="nflx_tdigest_agg",
+ rule=AggregationRule(),
+ serialize=serialize,
+ serialize_targets=serialize_targets or [],
+ )
+
+
+DRUID = [MaterializationTarget.DRUID]
+
+
+class TestSerializeIsApplied:
+ """When the component declares a conversion and the target asks for it."""
+
+ def test_simple_accumulate_is_wrapped(self):
+ """A bare function-name accumulate gets the conversion."""
+ component = _component(
+ "nflx_tdigest",
+ serialize="nflx_tdigest_sketch({})",
+ serialize_targets=DRUID,
+ )
+
+ expr = build_component_expression(component, MaterializationTarget.DRUID)
+
+ assert str(expr) == "nflx_tdigest_sketch(nflx_tdigest(latency_ms))"
+
+ def test_template_accumulate_is_wrapped(self):
+ """
+ A pre-expanded template accumulate gets the conversion too.
+ """
+ component = _component(
+ "nflx_tdigest(latency_ms, 200)",
+ serialize="nflx_tdigest_sketch({})",
+ serialize_targets=DRUID,
+ )
+
+ expr = build_component_expression(component, MaterializationTarget.DRUID)
+
+ assert str(expr) == "nflx_tdigest_sketch(nflx_tdigest(latency_ms, 200))"
+
+
+class TestSerializeIsSkipped:
+ """Every path that must leave the accumulated expression untouched."""
+
+ def test_no_target_means_no_conversion(self):
+ """
+ A query-time build passes no target and gets the unwrapped expression.
+ """
+ component = _component(
+ "nflx_tdigest(latency_ms, 200)",
+ serialize="nflx_tdigest_sketch({})",
+ serialize_targets=DRUID,
+ )
+
+ expr = build_component_expression(component)
+
+ assert str(expr) == "nflx_tdigest(latency_ms, 200)"
+
+ def test_different_target_means_no_conversion(self):
+ """
+ An Iceberg pre-agg keeps the in-engine representation.
+ """
+ component = _component(
+ "nflx_tdigest(latency_ms, 200)",
+ serialize="nflx_tdigest_sketch({})",
+ serialize_targets=DRUID,
+ )
+
+ expr = build_component_expression(component, MaterializationTarget.ICEBERG)
+
+ assert str(expr) == "nflx_tdigest(latency_ms, 200)"
+
+ def test_component_without_serialize_is_untouched(self):
+ """
+ A component declaring no conversion is unaffected even for Druid.
+ """
+ component = _component("SUM")
+
+ expr = build_component_expression(component, MaterializationTarget.DRUID)
+
+ assert str(expr) == "SUM(latency_ms)"
+
+ @pytest.mark.parametrize(
+ "target",
+ [None, MaterializationTarget.DRUID, MaterializationTarget.ICEBERG],
+ )
+ def test_plain_sum_is_stable_across_targets(self, target):
+ """
+ An ordinary measure renders identically whatever the target.
+ """
+ expr = build_component_expression(_component("SUM"), target)
+
+ assert str(expr) == "SUM(latency_ms)"
+
+
+class TestSerializeDeclaration:
+ """The declaration itself."""
+
+ def test_targets_default_to_empty(self):
+ """
+ A component declaring a conversion but no targets never applies it.
+
+ Defaulting to "no targets" rather than "all targets" prevents
+ half-finished declarations from silently rewriting measures tables.
+ """
+ component = _component(
+ "nflx_tdigest(latency_ms, 200)",
+ serialize="nflx_tdigest_sketch({})",
+ )
+
+ expr = build_component_expression(component, MaterializationTarget.DRUID)
+
+ assert str(expr) == "nflx_tdigest(latency_ms, 200)"
+
+
+class TestSerializedColumnType:
+ """
+ The recorded column type must match the representation actually written.
+
+ These are set in different places (SQL by ``build_component_expression``,
+ type by ``infer_component_type``). A disagreement is silent and could cause
+ Druid ingestion to miss the correct aggregator mappings.
+ """
+
+ @staticmethod
+ def _parent():
+ return SimpleNamespace(
+ current=SimpleNamespace(
+ columns=[SimpleNamespace(name="latency_ms", type="double")],
+ ),
+ )
+
+ def _sketch_component(self, serialize_type: str | None = "binary"):
+ """
+ A component that converts on the way to Druid.
+ """
+ component = _component(
+ "SUM",
+ serialize="to_druid_bytes({})",
+ serialize_targets=DRUID,
+ )
+ component.serialize_type = serialize_type
+ return component
+
+ def test_druid_target_records_the_converted_type(self):
+ """The declared conversion type wins for the target that converts."""
+ assert (
+ infer_component_type(
+ self._sketch_component(),
+ "double",
+ self._parent(),
+ MaterializationTarget.DRUID,
+ )
+ == "binary"
+ )
+
+ @pytest.mark.parametrize(
+ "target",
+ [None, MaterializationTarget.ICEBERG],
+ )
+ def test_other_targets_keep_the_accumulated_type(self, target):
+ """Query-time and Iceberg store the unconverted value, so keep its type."""
+ assert (
+ infer_component_type(
+ self._sketch_component(),
+ "double",
+ self._parent(),
+ target,
+ )
+ == "double"
+ )
+
+ def test_conversion_without_a_declared_type_falls_through(self):
+ """
+ Declaring a conversion but no type leaves inference in charge.
+ """
+ assert (
+ infer_component_type(
+ self._sketch_component(serialize_type=None),
+ "double",
+ self._parent(),
+ MaterializationTarget.DRUID,
+ )
+ == "double"
+ )
+
+ def test_ordinary_component_is_unaffected(self):
+ """A component with no conversion types exactly as it did before."""
+ assert (
+ infer_component_type(
+ _component("SUM"),
+ "double",
+ self._parent(),
+ MaterializationTarget.DRUID,
+ )
+ == "double"
+ )
+
+
+class TestSerializesFor:
+ """The single predicate both the SQL and the type side consult."""
+
+ @pytest.mark.parametrize(
+ ("serialize", "targets", "target", "expected"),
+ [
+ ("f({})", DRUID, MaterializationTarget.DRUID, True),
+ ("f({})", DRUID, MaterializationTarget.ICEBERG, False),
+ ("f({})", DRUID, None, False),
+ ("f({})", [], MaterializationTarget.DRUID, False),
+ (None, DRUID, MaterializationTarget.DRUID, False),
+ ],
+ )
+ def test_predicate(self, serialize, targets, target, expected):
+ component = _component("SUM", serialize=serialize, serialize_targets=targets)
+
+ assert component.serializes_for(target) is expected
diff --git a/datajunction-server/tests/construction/build_v3/sketch_family_e2e_test.py b/datajunction-server/tests/construction/build_v3/sketch_family_e2e_test.py
new file mode 100644
index 000000000..95e7ca8f4
--- /dev/null
+++ b/datajunction-server/tests/construction/build_v3/sketch_family_e2e_test.py
@@ -0,0 +1,429 @@
+"""
+A sketch family, end to end, through the HTTP endpoints.
+
+The rest of the suite tests this machinery in pieces -- `merge_args_test` on
+the merge call, `accumulate_type_test` on inferred types, `serialize_target_test`
+on the write-time conversion, `reaggregate_test` on the spec. Each one holds a
+`ComponentDef` or a `MetricComponent` in its hand and asserts about it. None of
+them puts a family behind the server and asks what SQL comes out.
+
+This does. One metric, one grain, followed from raw rows to the number the
+engine returns:
+
+ accumulate build_digest(line_total, 200.0) from source, per partition
+ merge merge_digest(digest, 200.0) from the pre-agg table
+ combine digest_quantiles(digest, ...)[0] Spark / Trino
+ DIGEST_QUANTILE(digest, 0.95) Druid, merge fused in
+
+OSS registers no family of its own -- they live downstream with the
+engine-specific functions -- so the fixture below supplies one, the same way
+`decompose_test.registered_family` does for the unit tests. That makes this
+file the executable form of the extension contract: a downstream deployment
+has to register *both* a decomposition and the functions it names, and if
+either half is missing the failure is a 500 out of the measures endpoint, not
+something a type checker catches.
+
+`serialize` is the one hook not asserted here. It applies when writing a
+measures table for a target rather than when reading, so no query endpoint
+renders it; `serialize_target_test` and `cube_druid_sketch_spec_test` cover it.
+"""
+
+import re
+
+import pytest
+from httpx import AsyncClient
+
+from datajunction_server.models.dialect import Dialect
+from datajunction_server.models.materialization import MaterializationTarget
+from datajunction_server.models.reaggregate import ReaggregationFunction
+from datajunction_server.sql.decompose import (
+ FAMILY_DECOMPOSITION_REGISTRY,
+ AggDecomposition,
+ ComponentDef,
+ decomposes_family,
+ make_func,
+)
+from datajunction_server.sql.functions import Function, function_registry
+from datajunction_server.sql.parsing import ast
+from datajunction_server.sql.parsing import types as ct
+
+from . import assert_sql_equal
+
+METRIC = "v3.p95_line_total"
+DIMENSIONS = ["v3.product.category"]
+COMPRESSION = 200
+PREAGG_TABLE = "analytics.preaggs.v3_digest_by_cat"
+
+
+# ---------------------------------------------------------------------------
+# A stand-in sketch family: four functions and a decomposition that uses them.
+# Shaped after a real t-digest -- an accumulate that takes a tuning argument, a
+# merge that must repeat it, and a combiner whose form differs on Druid -- so
+# the parts of the machinery that only a sketch exercises are actually hit.
+# ---------------------------------------------------------------------------
+
+
+class _DigestFunction(Function):
+ """Base class, so the fixture can find these to register and unregister."""
+
+
+class BuildDigest(_DigestFunction):
+ """build_digest(col, compression) -> digest. Phase 1."""
+
+ is_aggregation = True
+ dialects = [Dialect.SPARK, Dialect.TRINO]
+
+
+class MergeDigest(_DigestFunction):
+ """merge_digest(digest, compression) -> digest. Phase 2."""
+
+ is_aggregation = True
+ dialects = [Dialect.SPARK, Dialect.TRINO]
+
+
+class DigestQuantiles(_DigestFunction):
+ """digest_quantiles(digest, quantiles) -> array. Phase 3, non-Druid."""
+
+ is_aggregation = False
+ dialects = [Dialect.SPARK, Dialect.TRINO]
+
+
+class DigestQuantile(_DigestFunction):
+ """DIGEST_QUANTILE(digest, q) -> double. Phase 2+3 fused, Druid only."""
+
+ is_aggregation = True
+ dialects = [Dialect.DRUID]
+
+
+@BuildDigest.register
+def infer_type(col: ct.ColumnType, compression: ct.ColumnType) -> ct.BinaryType:
+ return ct.BinaryType()
+
+
+@MergeDigest.register
+def infer_type(digest: ct.ColumnType, compression: ct.ColumnType) -> ct.BinaryType:
+ return ct.BinaryType()
+
+
+@DigestQuantiles.register
+def infer_type(digest: ct.ColumnType, quantiles: ct.ColumnType) -> ct.ListType:
+ return ct.ListType(element_type=ct.DoubleType())
+
+
+@DigestQuantile.register
+def infer_type(digest: ct.ColumnType, quantile: ct.ColumnType) -> ct.DoubleType:
+ return ct.DoubleType()
+
+
+@pytest.fixture
+def digest_family():
+ """
+ Register the family and its functions, then put the registries back.
+
+ Both registries are process-global, so leaving either populated would leak
+ into every later test in the session.
+ """
+ saved: dict[str, type | None] = {}
+ for cls in _DigestFunction.__subclasses__():
+ snake_cased = re.sub(r"(? int:
+ return int(self.params.get("compression", 100))
+
+ @property
+ def compression_literal(self) -> str:
+ return f"CAST({self.compression}.0 AS DOUBLE)"
+
+ @property
+ def components(self) -> list[ComponentDef]:
+ return [
+ ComponentDef(
+ # Compression rides in the suffix so two differently tuned
+ # sketches over the same column get distinct columns.
+ suffix=f"_digest_c{self.compression}",
+ accumulate=f"build_digest({{}}, {self.compression_literal})",
+ merge="merge_digest",
+ merge_args=(self.compression_literal,),
+ serialize="digest_to_bytes({})",
+ serialize_targets=(MaterializationTarget.DRUID,),
+ serialize_type="binary",
+ ),
+ ]
+
+ def combine(self, components, func, dialect=Dialect.SPARK):
+ column, fraction = components[0].name, func.args[1]
+ if dialect == Dialect.DRUID:
+ return make_func("DIGEST_QUANTILE", column, fraction)
+ return ast.Subscript(
+ expr=make_func(
+ "digest_quantiles",
+ column,
+ make_func("array", fraction),
+ ),
+ index=ast.Number(0),
+ )
+
+ try:
+ yield _DigestDecomposition
+ finally:
+ FAMILY_DECOMPOSITION_REGISTRY.pop(ReaggregationFunction.TDIGEST, None)
+ for key, previous in saved.items():
+ if previous is None:
+ function_registry.pop(key, None)
+ else:
+ function_registry[key] = previous
+
+
+@pytest.fixture
+async def percentile_metric(client_with_build_v3: AsyncClient, digest_family):
+ """
+ A percentile metric that opts into the family.
+
+ The aggregation is a plain APPROX_PERCENTILE -- nothing about the query
+ mentions a sketch. Only `reaggregate.fn` selects the family, which is the
+ point of the family registry: the metric keeps its readable definition.
+ """
+ response = await client_with_build_v3.post(
+ "/nodes/metric/",
+ json={
+ "name": METRIC,
+ "description": "95th percentile line total",
+ "query": (
+ "SELECT APPROX_PERCENTILE(line_total, 0.95) FROM v3.order_details"
+ ),
+ "mode": "published",
+ "reaggregate": {
+ "fn": "tdigest",
+ "params": {"compression": COMPRESSION},
+ },
+ },
+ )
+ assert response.status_code in (200, 201), response.json()
+ return client_with_build_v3
+
+
+async def _measures(client: AsyncClient, dialect: str = "spark") -> dict:
+ response = await client.get(
+ "/sql/measures/v3/",
+ params={"metrics": [METRIC], "dimensions": DIMENSIONS, "dialect": dialect},
+ )
+ assert response.status_code == 200, response.json()
+ return response.json()
+
+
+async def _publish_preagg(client: AsyncClient) -> None:
+ response = await client.post(
+ "/preaggs/plan",
+ json={
+ "metrics": [METRIC],
+ "dimensions": DIMENSIONS,
+ "strategy": "full",
+ "schedule": "0 0 * * *",
+ },
+ )
+ assert response.status_code == 201, response.text
+
+ catalog, schema, table = PREAGG_TABLE.split(".")
+ response = await client.post(
+ f"/preaggs/{response.json()['preaggs'][0]['id']}/availability/",
+ json={
+ "catalog": catalog,
+ "schema": schema,
+ "table": table,
+ "valid_through_ts": 1704067200,
+ },
+ )
+ assert response.status_code == 200, response.text
+
+
+@pytest.mark.asyncio
+async def test_accumulate_renders_the_tuning_param(percentile_metric):
+ """
+ Phase 1: `params` reaches the generated SQL as a real argument.
+
+ `compression` is declared once on the metric's reaggregate spec and has to
+ survive into the accumulate call. If it were dropped the query would still
+ run -- `build_digest` has a one-argument form -- and quietly build sketches
+ at the wrong accuracy, so this is worth pinning on the SQL rather than on
+ the spec.
+ """
+ (group,) = (await _measures(percentile_metric))["grain_groups"]
+
+ assert_sql_equal(
+ group["sql"],
+ f"""
+ WITH v3_order_details AS (
+ SELECT oi.product_id,
+ oi.quantity * oi.unit_price AS line_total
+ FROM default.v3.orders o
+ JOIN default.v3.order_items oi ON o.order_id = oi.order_id
+ ),
+ v3_product AS (
+ SELECT product_id, category FROM default.v3.products
+ )
+ SELECT t2.category,
+ build_digest(t1.line_total, CAST({COMPRESSION}.0 AS DOUBLE))
+ line_total_digest_c{COMPRESSION}_e1f61696
+ FROM v3_order_details t1
+ LEFT OUTER JOIN v3_product t2 ON t1.product_id = t2.product_id
+ GROUP BY t2.category
+ """,
+ )
+
+
+@pytest.mark.asyncio
+async def test_component_identity_carries_the_compression(percentile_metric):
+ """
+ The column name encodes the tuning, and the recorded type is the sketch's.
+
+ Both matter downstream: the name keeps a sketch built at one accuracy from
+ satisfying a query that asked for another, and the type is what the
+ materialized table is created from.
+ """
+ (group,) = (await _measures(percentile_metric))["grain_groups"]
+ (component,) = group["components"]
+ (column,) = [c for c in group["columns"] if c["semantic_type"] != "dimension"]
+
+ assert component["name"] == f"line_total_digest_c{COMPRESSION}_e1f61696"
+ assert component["merge"] == "merge_digest"
+ assert component["aggregability"] == "full"
+ assert column["type"] == "binary"
+
+
+@pytest.mark.asyncio
+async def test_merge_from_preagg_repeats_the_tuning_param(percentile_metric):
+ """
+ Phase 2: reading stored sketches back, with `merge_args` applied.
+
+ The accumulate is gone and the source CTEs with it -- a single scan of the
+ pre-agg table. The compression has to appear a second time here: `merge` is
+ a bare function name (the Druid aggregator lookup and the semi-additive
+ rewrite both match on it), so the argument cannot be folded into it and
+ travels as `merge_args` instead.
+ """
+ await _publish_preagg(percentile_metric)
+ (group,) = (await _measures(percentile_metric))["grain_groups"]
+
+ assert_sql_equal(
+ group["sql"],
+ f"""
+ SELECT category,
+ merge_digest(line_total_digest_c{COMPRESSION}_e1f61696,
+ CAST({COMPRESSION}.0 AS DOUBLE))
+ line_total_digest_c{COMPRESSION}_e1f61696
+ FROM {PREAGG_TABLE}
+ GROUP BY category
+ """,
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "dialect, expected",
+ [
+ (
+ "spark",
+ f"digest_quantiles(line_total_digest_c{COMPRESSION}_e1f61696, "
+ "array(0.95))[0]",
+ ),
+ (
+ "druid",
+ f"DIGEST_QUANTILE(line_total_digest_c{COMPRESSION}_e1f61696, 0.95)",
+ ),
+ ],
+)
+async def test_combiner_takes_the_shape_the_engine_needs(
+ percentile_metric,
+ dialect,
+ expected,
+):
+ """
+ Phase 3: the combiner, and the one place the dialect legitimately matters.
+
+ Druid has no scalar reader for the sketch, so it fuses merge and combine
+ into one aggregating call; Spark and Trino read the merged column with a
+ separate extractor. That is a difference in shape, which no transpiler can
+ derive -- unlike function spelling or array index base, which the
+ transpilation layer handles and a decomposition must therefore leave alone.
+
+ The requested quantile is authored in the metric query, not the reaggregate
+ spec, so it has to survive decomposition and reappear here. That is what
+ lets p50/p95/p99 share one sketch column.
+ """
+ payload = await _measures(percentile_metric, dialect=dialect)
+ (formula,) = payload["metric_formulas"]
+
+ assert formula["combiner"] == expected
+
+
+@pytest.mark.asyncio
+async def test_metrics_layer_recomputes_from_source(percentile_metric):
+ """
+ Final metrics layer: recomputes from source, pre-agg or not.
+
+ Pinning current behaviour, not endorsing it. The pre-agg published here is
+ the same one `test_merge_from_preagg_repeats_the_tuning_param` reads, and
+ the measures layer picks it up. The metrics layer never consults the
+ matcher: `build_metrics_sql` takes the cube branch only when
+ `use_materialized and dialect == Dialect.DRUID`, and otherwise builds grain
+ groups from source.
+
+ So no sketch appears below at all -- a bare APPROX_PERCENTILE off the fact
+ -- and the combiner asserted above never reaches rendered SQL on this path.
+ Defensible if scanning source on Trino is cheap enough, but it does mean
+ the sketch columns are written for Druid's benefit alone.
+ """
+ await _publish_preagg(percentile_metric)
+
+ response = await percentile_metric.get(
+ "/sql/",
+ params={"metrics": [METRIC], "dimensions": DIMENSIONS},
+ )
+ assert response.status_code == 200, response.json()
+
+ assert_sql_equal(
+ response.json()["sql"],
+ """
+ WITH v3_DOT_order_details AS (
+ SELECT o.order_id,
+ oi.line_number,
+ o.customer_id,
+ o.order_date,
+ o.from_location_id,
+ o.to_location_id,
+ o.status,
+ oi.product_id,
+ oi.quantity,
+ oi.unit_price,
+ oi.quantity * oi.unit_price AS line_total
+ FROM v3.orders AS o
+ JOIN v3.order_items AS oi ON o.order_id = oi.order_id
+ ),
+ v3_DOT_product AS (
+ SELECT v3_DOT_src_products.product_id,
+ v3_DOT_src_products.name,
+ v3_DOT_src_products.category,
+ v3_DOT_src_products.subcategory,
+ v3_DOT_src_products.price
+ FROM v3.products AS v3_DOT_src_products
+ ),
+ v3_DOT_order_details_metrics AS (
+ SELECT v3_DOT_product.category v3_DOT_product_DOT_category,
+ APPROX_PERCENTILE(v3_DOT_order_details.line_total, 0.95)
+ v3_DOT_p95_line_total
+ FROM v3_DOT_order_details
+ LEFT JOIN v3_DOT_product
+ ON v3_DOT_order_details.product_id = v3_DOT_product.product_id
+ GROUP BY v3_DOT_product.category
+ )
+ SELECT v3_DOT_order_details_metrics.v3_DOT_product_DOT_category,
+ v3_DOT_order_details_metrics.v3_DOT_p95_line_total
+ FROM v3_DOT_order_details_metrics
+ """,
+ )
diff --git a/datajunction-server/tests/database/preaggregation_test.py b/datajunction-server/tests/database/preaggregation_test.py
index 0364ab65f..a88bac361 100644
--- a/datajunction-server/tests/database/preaggregation_test.py
+++ b/datajunction-server/tests/database/preaggregation_test.py
@@ -716,3 +716,75 @@ async def test_find_matching_no_candidates(
},
)
assert result is None
+
+
+class TestMeasureIdentityParams:
+ """Tuning parameters participate in measure identity."""
+
+ def test_existing_tokens_are_unchanged(self):
+ """A component without params produces the identical token."""
+ assert measure_identity_token("abc123", "SUM") == "abc123:SUM"
+ assert measure_identity_token("abc123", "SUM", None) == "abc123:SUM"
+ assert measure_identity_token("abc123", "SUM", {}) == "abc123:SUM"
+ assert measure_identity_token("abc123", None) == "abc123:"
+ assert measure_identity_token("abc123", " count ") == "abc123:COUNT"
+
+ def test_differing_params_differ(self):
+ """Two sketches over one column at different accuracies are distinct."""
+ low = measure_identity_token("abc123", "TDIGEST_AGG", {"compression": 100})
+ high = measure_identity_token("abc123", "TDIGEST_AGG", {"compression": 1000})
+ assert low != high
+ # ...and neither collides with the un-parameterized token.
+ assert low != measure_identity_token("abc123", "TDIGEST_AGG")
+
+ def test_params_are_order_and_spelling_insensitive(self):
+ """Order and type variations (e.g. 200 vs 200.0) do not change the identity token."""
+ assert measure_identity_token(
+ "abc123",
+ "TDIGEST_AGG",
+ {"compression": 200, "k": 12},
+ ) == measure_identity_token(
+ "abc123",
+ "TDIGEST_AGG",
+ {"k": 12, "compression": 200},
+ )
+ assert measure_identity_token(
+ "abc123",
+ "TDIGEST_AGG",
+ {"compression": 200.0},
+ ) == measure_identity_token("abc123", "TDIGEST_AGG", {"compression": 200})
+ # Fractional values are preserved.
+ assert measure_identity_token(
+ "abc123",
+ "TDIGEST_AGG",
+ {"compression": 200.5},
+ ) != measure_identity_token("abc123", "TDIGEST_AGG", {"compression": 200})
+
+ def test_get_measure_identities_carries_params(self):
+ """Stored measures round-trip their params into the identity set."""
+ plain = make_measure("latency_sum", "latency_ms")
+ sketched = make_measure(
+ "latency_tdigest",
+ "latency_ms",
+ aggregation="TDIGEST_AGG",
+ merge="TDIGEST_AGG",
+ )
+ sketched.params = {"compression": 200}
+ identities = get_measure_identities([plain, sketched])
+ assert identities == {
+ measure_identity_token(compute_expression_hash("latency_ms"), "SUM"),
+ measure_identity_token(
+ compute_expression_hash("latency_ms"),
+ "TDIGEST_AGG",
+ {"compression": 200},
+ ),
+ }
+ # Ensure the sketch identity distinguishes different compressions.
+ assert (
+ measure_identity_token(
+ compute_expression_hash("latency_ms"),
+ "TDIGEST_AGG",
+ {"compression": 1000},
+ )
+ not in identities
+ )
diff --git a/datajunction-server/tests/internal/deployment/orchestration_test.py b/datajunction-server/tests/internal/deployment/orchestration_test.py
index a163ba5ab..974895046 100644
--- a/datajunction-server/tests/internal/deployment/orchestration_test.py
+++ b/datajunction-server/tests/internal/deployment/orchestration_test.py
@@ -27,6 +27,7 @@
from datajunction_server.internal.deployment.orchestrator import (
DeploymentOrchestrator,
DeploymentPlan,
+ DeploymentTimer,
ResourceRegistry,
column_changed,
tag_needs_update,
@@ -82,6 +83,26 @@ def mock_deployment_context(current_user: User):
return context
+def test_deployment_timer_logs_unaccounted_overhead(monkeypatch, caplog):
+ """
+ Deployment timing summary includes unaccounted overhead when it is meaningful.
+ """
+ timer = DeploymentTimer()
+ timer.record("validate", 10, "1 node")
+ monkeypatch.setattr(
+ "datajunction_server.internal.deployment.orchestrator.time.perf_counter",
+ lambda: timer._start + 0.2,
+ )
+
+ with caplog.at_level(
+ "INFO",
+ logger="datajunction_server.internal.deployment.orchestrator",
+ ):
+ timer.log_summary("default", "deployment-1")
+
+ assert "(unaccounted overhead)" in caplog.text
+
+
@pytest.fixture
def sample_deployment_spec():
"""Sample deployment specification for testing"""
diff --git a/datajunction-server/tests/internal/deployment/validation_test.py b/datajunction-server/tests/internal/deployment/validation_test.py
index c252035a5..6a270b7c7 100644
--- a/datajunction-server/tests/internal/deployment/validation_test.py
+++ b/datajunction-server/tests/internal/deployment/validation_test.py
@@ -793,6 +793,97 @@ async def test_invalid_short_name_not_in_parent_columns(
assert err.debug is not None
assert "no_such_col" in err.debug["invalid_required_dimensions"]
+ @pytest.mark.asyncio
+ async def test_valid_reaggregate_dimension_full_path(
+ self,
+ session: AsyncSession,
+ parent_node: Node,
+ ):
+ """reaggregate.rules[].dimension full-path column found in dim nodes is valid."""
+ dim_node = self._make_dim_node("test.dim", ["dateint"])
+ context = self._make_context(session, parent_node)
+ spec = MetricSpec(
+ name="test.metric",
+ query="SELECT SUM(value) FROM test.parent",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "test.dim.dateint",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ validator = NodeSpecBulkValidator(context)
+ validator._all_dim_nodes = {**context.dependency_nodes, "test.dim": dim_node}
+ result = validator.validate_query_node(spec)
+
+ assert result.status == NodeStatus.VALID
+ error_codes = [e.code for e in result.errors]
+ assert ErrorCode.INVALID_COLUMN not in error_codes
+
+ @pytest.mark.asyncio
+ async def test_invalid_reaggregate_dimension_column(
+ self,
+ session: AsyncSession,
+ parent_node: Node,
+ ):
+ """reaggregate.rules[].dimension full-path column absent from dim node is invalid."""
+ dim_node = self._make_dim_node("test.dim", ["dateint"])
+ context = self._make_context(session, parent_node)
+ spec = MetricSpec(
+ name="test.metric",
+ query="SELECT SUM(value) FROM test.parent",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "test.dim.nonexistent_col",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ validator = NodeSpecBulkValidator(context)
+ validator._all_dim_nodes = {**context.dependency_nodes, "test.dim": dim_node}
+ result = validator.validate_query_node(spec)
+
+ assert result.status == NodeStatus.INVALID
+ err = next(e for e in result.errors if e.code == ErrorCode.INVALID_COLUMN)
+ assert err.debug is not None
+ assert "test.dim.nonexistent_col" in err.debug["invalid_reaggregate_dimensions"]
+
+ @pytest.mark.asyncio
+ async def test_invalid_reaggregate_function(
+ self,
+ session: AsyncSession,
+ parent_node: Node,
+ ):
+ """unsupported dimension-specific reaggregate functions are invalid."""
+ context = self._make_context(session, parent_node)
+ spec = MetricSpec(
+ name="test.metric",
+ query="SELECT SUM(value) FROM test.parent",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "id",
+ "fn": "sum",
+ },
+ ],
+ },
+ )
+ validator = NodeSpecBulkValidator(context)
+ validator._all_dim_nodes = dict(context.dependency_nodes)
+ result = validator.validate_query_node(spec)
+
+ assert result.status == NodeStatus.INVALID
+ err = next(
+ e
+ for e in result.errors
+ if e.code == ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION
+ )
+ assert err.debug == {"invalid_reaggregate_functions": ["sum"]}
+
@pytest.mark.asyncio
async def test_no_required_dimensions_is_noop(
self,
@@ -866,6 +957,64 @@ async def test_prefetch_fetches_dim_node_from_db(
col_names = {c.name for c in fetched.current.columns}
assert "region" in col_names
+ @pytest.mark.asyncio
+ async def test_prefetch_fetches_reaggregate_dim_node_from_db(
+ self,
+ session: AsyncSession,
+ parent_node: Node,
+ dim_node_in_db: Node,
+ ):
+ """_prefetch_required_dimension_nodes includes reaggregate dimensions."""
+ context = self._make_context(session, parent_node)
+ spec = MetricSpec(
+ name="test.metric",
+ query="SELECT SUM(value) FROM test.parent",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "test.external_dim.region",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ validator = NodeSpecBulkValidator(context)
+
+ await validator._prefetch_required_dimension_nodes([spec])
+
+ assert "test.external_dim" in validator._all_dim_nodes
+ fetched = validator._all_dim_nodes["test.external_dim"]
+ assert fetched.current is not None
+ assert {c.name for c in fetched.current.columns} == {"region"}
+
+ @pytest.mark.asyncio
+ async def test_prefetch_skips_short_reaggregate_dimension(
+ self,
+ session: AsyncSession,
+ parent_node: Node,
+ ):
+ """Short-form reaggregate dimensions do not trigger dimension-node prefetch."""
+ context = self._make_context(session, parent_node)
+ spec = MetricSpec(
+ name="test.metric",
+ query="SELECT SUM(value) FROM test.parent",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "value",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ validator = NodeSpecBulkValidator(context)
+
+ await validator._prefetch_required_dimension_nodes([spec])
+
+ assert set(validator._all_dim_nodes.keys()) == set(
+ context.dependency_nodes.keys(),
+ )
+
@pytest.mark.asyncio
async def test_prefetch_with_no_required_dimensions(
self,
diff --git a/datajunction-server/tests/internal/namespaces_test.py b/datajunction-server/tests/internal/namespaces_test.py
index 1c563211d..839f99086 100644
--- a/datajunction-server/tests/internal/namespaces_test.py
+++ b/datajunction-server/tests/internal/namespaces_test.py
@@ -3,6 +3,7 @@
"""
from datetime import UTC, datetime
+from types import SimpleNamespace
import pytest
from ruamel.yaml import YAML
@@ -22,6 +23,7 @@
_merge_list_with_key,
_merge_yaml_preserving_comments,
create_or_reactivate_namespace,
+ get_node_specs_for_export,
node_spec_to_yaml,
provision_namespace_boundary,
)
@@ -1108,6 +1110,81 @@ def test_no_yaml_document_start_marker(self):
"query: SELECT SUM(rev) FROM ns.transforms.t",
]
+ def test_metric_reaggregate_exports_dimension_rules(self):
+ """metric reaggregate exports the rule list without null future fields"""
+ spec = MetricSpec(
+ name="ns.metrics.daily_balance",
+ node_type=NodeType.METRIC,
+ query="SELECT SUM(balance) FROM ns.transforms.accounts",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "${prefix}v3.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ assert node_spec_to_yaml(spec).splitlines() == [
+ "name: ns.metrics.daily_balance",
+ "node_type: metric",
+ "mode: published",
+ "query: SELECT SUM(balance) FROM ns.transforms.accounts",
+ "reaggregate:",
+ " rules:",
+ " - dimension: ${prefix}v3.date.date_id[order]",
+ " fn: last_value",
+ ]
+
+ @pytest.mark.asyncio
+ async def test_metric_reaggregate_export_injects_dimension_prefix(
+ self,
+ monkeypatch,
+ ):
+ """namespace export parameterizes reaggregate rule dimensions."""
+ spec = MetricSpec(
+ name="demo.branch.metrics.daily_balance",
+ node_type=NodeType.METRIC,
+ query="SELECT SUM(balance) FROM demo.branch.transforms.accounts",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "demo.branch.dims.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+
+ async def to_spec(_session):
+ return spec
+
+ fake_node = SimpleNamespace(
+ name="demo.branch.metrics.daily_balance",
+ current=SimpleNamespace(parents=[], status="valid"),
+ to_spec=to_spec,
+ )
+
+ async def namespace_get(_session, _namespace, raise_if_not_exists=False):
+ return SimpleNamespace(parent_namespace="demo.main")
+
+ async def list_all_nodes(_session, _namespace, options=None):
+ return [fake_node]
+
+ monkeypatch.setattr(NodeNamespace, "get", namespace_get)
+ monkeypatch.setattr(NodeNamespace, "list_all_nodes", list_all_nodes)
+
+ exported = await get_node_specs_for_export(SimpleNamespace(), "demo.branch")
+
+ assert exported[0].name == "${prefix}metrics.daily_balance"
+ assert exported[0].query == (
+ "SELECT SUM(balance) FROM ${prefix}transforms.accounts"
+ )
+ assert (
+ exported[0].reaggregate.rules[0].dimension
+ == "${prefix}dims.date.date_id[order]"
+ )
+
def test_multiline_query_uses_literal_block_style(self):
"""multiline queries are serialized with |- literal block style"""
spec = MetricSpec(
diff --git a/datajunction-server/tests/internal/node_validation_test.py b/datajunction-server/tests/internal/node_validation_test.py
index 5711a6ea7..7f765c7fe 100644
--- a/datajunction-server/tests/internal/node_validation_test.py
+++ b/datajunction-server/tests/internal/node_validation_test.py
@@ -1019,6 +1019,133 @@ async def test_validate_node_data_v2_flags_invalid_required_dimensions(
), [(e.code, e.message) for e in validator.errors]
+@pytest.mark.asyncio
+async def test_validate_node_data_v2_flags_invalid_reaggregate_dimensions(
+ session: AsyncSession,
+ user: User,
+):
+ """reaggregate dimensions must resolve to columns on parent dimension nodes."""
+ from datajunction_server.errors import ErrorCode
+ from datajunction_server.internal.validation import validate_node_data_v2
+
+ source = Node(
+ name="test.v2_reagg_dim_parent",
+ type=NodeType.SOURCE,
+ created_by_id=user.id,
+ current_version="v1.0",
+ )
+ source_rev = NodeRevision(
+ name="test.v2_reagg_dim_parent",
+ display_name="reaggregate dim parent",
+ type=NodeType.SOURCE,
+ query=None,
+ status=NodeStatus.VALID,
+ version="v1.0",
+ node=source,
+ columns=[Column(name="id", type=ct.BigIntType(), order=0)],
+ created_by_id=user.id,
+ )
+ dim = Node(
+ name="test.v2_reagg_dim",
+ type=NodeType.DIMENSION,
+ created_by_id=user.id,
+ current_version="v1.0",
+ )
+ dim_rev = NodeRevision(
+ name="test.v2_reagg_dim",
+ display_name="tiny reaggregate dim",
+ type=NodeType.DIMENSION,
+ query="SELECT 1 AS id",
+ status=NodeStatus.VALID,
+ version="v1.0",
+ node=dim,
+ columns=[Column(name="id", type=ct.BigIntType(), order=0)],
+ created_by_id=user.id,
+ )
+ session.add_all([source, source_rev, dim, dim_rev])
+ await session.commit()
+
+ child = NodeRevision(
+ name="test.v2_reagg_dim_child",
+ display_name="reaggregate dim child",
+ type=NodeType.METRIC,
+ query="SELECT SUM(id) FROM test.v2_reagg_dim_parent",
+ status=NodeStatus.VALID,
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "test.v2_reagg_dim.ghost_col",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ validator = await validate_node_data_v2(child, session)
+
+ assert validator.status == NodeStatus.INVALID
+ assert any(
+ err.code == ErrorCode.INVALID_COLUMN and "reaggregate dimensions" in err.message
+ for err in validator.errors
+ ), [(e.code, e.message) for e in validator.errors]
+
+
+@pytest.mark.asyncio
+async def test_validate_node_data_v2_flags_unsupported_reaggregate_function(
+ session: AsyncSession,
+ user: User,
+):
+ """dimension-specific reaggregate only accepts supported collapse functions."""
+ from datajunction_server.errors import ErrorCode
+ from datajunction_server.internal.validation import validate_node_data_v2
+
+ source = Node(
+ name="test.v2_reagg_fn_parent",
+ type=NodeType.SOURCE,
+ created_by_id=user.id,
+ current_version="v1.0",
+ )
+ source_rev = NodeRevision(
+ name="test.v2_reagg_fn_parent",
+ display_name="reaggregate function parent",
+ type=NodeType.SOURCE,
+ query=None,
+ status=NodeStatus.VALID,
+ version="v1.0",
+ node=source,
+ columns=[
+ Column(name="id", type=ct.BigIntType(), order=0),
+ Column(name="order_date", type=ct.BigIntType(), order=1),
+ ],
+ created_by_id=user.id,
+ )
+ session.add_all([source, source_rev])
+ await session.commit()
+
+ child = NodeRevision(
+ name="test.v2_reagg_fn_child",
+ display_name="reaggregate function child",
+ type=NodeType.METRIC,
+ query="SELECT SUM(id) FROM test.v2_reagg_fn_parent",
+ status=NodeStatus.VALID,
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "order_date",
+ "fn": "sum",
+ },
+ ],
+ },
+ )
+ validator = await validate_node_data_v2(child, session)
+
+ assert validator.status == NodeStatus.INVALID
+ assert any(
+ err.code == ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION
+ and err.debug == {"invalid_reaggregate_functions": ["sum"]}
+ for err in validator.errors
+ ), [(e.code, e.message, e.debug) for e in validator.errors]
+
+
@pytest.mark.asyncio
async def test_validate_node_data_v2_cross_fact_metrics_no_shared_dims(
session: AsyncSession,
diff --git a/datajunction-server/tests/internal/nodes/derive_frozen_measures_test.py b/datajunction-server/tests/internal/nodes/derive_frozen_measures_test.py
index 292be4521..c0000b83e 100644
--- a/datajunction-server/tests/internal/nodes/derive_frozen_measures_test.py
+++ b/datajunction-server/tests/internal/nodes/derive_frozen_measures_test.py
@@ -16,7 +16,16 @@
from datajunction_server.database.measure import FrozenMeasure
from datajunction_server.database.node import Node, NodeRevision
from datajunction_server.database.user import OAuthProvider, User
-from datajunction_server.internal.nodes import derive_frozen_measures_bulk
+from datajunction_server.errors import DJInvalidInputException
+from datajunction_server.internal.nodes import (
+ _raise_if_frozen_measure_conflicts,
+ derive_frozen_measures_bulk,
+)
+from datajunction_server.models.decompose import (
+ Aggregability,
+ AggregationRule,
+ MetricComponent,
+)
from datajunction_server.models.node import NodeStatus
from datajunction_server.models.node_type import NodeType
@@ -62,6 +71,7 @@ async def _make_metric(
name: str,
query: str,
parents: list[Node],
+ reaggregate: dict[str, object] | None = None,
) -> Node:
node = Node(
name=name,
@@ -75,6 +85,7 @@ async def _make_metric(
type=NodeType.METRIC,
version="v1.0",
query=query,
+ reaggregate=reaggregate,
status=NodeStatus.VALID,
parents=parents,
created_by_id=user.id,
@@ -123,6 +134,180 @@ async def test_base_metric_populates_derived_expression_and_measure(
assert any(fm.aggregation == "SUM" for fm in metric.current.frozen_measures)
+@pytest.mark.asyncio
+async def test_frozen_measure_name_collision_allows_reaggregate_only_difference(
+ session: AsyncSession,
+ user: User,
+):
+ """Metric-level reaggregate metadata does not change measure identity."""
+ src = await _make_source(
+ session,
+ user,
+ "src_reaggregate_collision",
+ [Column(name="semi_amount", type=ct.DoubleType(), order=0)],
+ )
+ reaggregate = await _make_metric(
+ session,
+ user,
+ "m.semi_amount_eod",
+ "SELECT SUM(semi_amount) FROM src_reaggregate_collision",
+ [src],
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ additive = await _make_metric(
+ session,
+ user,
+ "m.semi_amount_total",
+ "SELECT SUM(semi_amount) FROM src_reaggregate_collision",
+ [src],
+ )
+ await session.refresh(reaggregate, ["current"])
+ await session.refresh(additive, ["current"])
+
+ await derive_frozen_measures_bulk(session, [additive.current.id])
+ await session.commit()
+
+ await session.refresh(additive.current, ["frozen_measures"])
+ assert all(fm.rule.reaggregate is None for fm in additive.current.frozen_measures)
+
+ await derive_frozen_measures_bulk(session, [reaggregate.current.id])
+ await session.commit()
+ await session.refresh(reaggregate.current, ["frozen_measures"])
+
+ assert {fm.name for fm in additive.current.frozen_measures} == {
+ fm.name for fm in reaggregate.current.frozen_measures
+ }
+
+
+@pytest.mark.asyncio
+async def test_reaggregate_rule_is_not_persisted_on_shared_frozen_measure(
+ session: AsyncSession,
+ user: User,
+):
+ """FrozenMeasure.rule stays metric-independent even for semi-additive metrics."""
+ src = await _make_source(
+ session,
+ user,
+ "src_reaggregate_storage",
+ [Column(name="semi_amount", type=ct.DoubleType(), order=0)],
+ )
+ reaggregate = await _make_metric(
+ session,
+ user,
+ "m.semi_amount_snapshot",
+ "SELECT SUM(semi_amount) FROM src_reaggregate_storage",
+ [src],
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ await session.refresh(reaggregate, ["current"])
+
+ await derive_frozen_measures_bulk(session, [reaggregate.current.id])
+ await session.commit()
+
+ await session.refresh(reaggregate.current, ["frozen_measures"])
+ assert all(
+ fm.rule.reaggregate is None for fm in reaggregate.current.frozen_measures
+ )
+
+
+def test_frozen_measure_conflict_rejects_different_measure_identity():
+ """FrozenMeasure name collisions fail when the metric-independent rule differs."""
+ frozen_measure = FrozenMeasure(
+ name="amount_sum",
+ upstream_revision_id=1,
+ expression="amount",
+ aggregation="SUM",
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+ measure = MetricComponent(
+ name="amount_sum",
+ expression="discounted_amount",
+ aggregation="SUM",
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+
+ with pytest.raises(DJInvalidInputException, match="already exists"):
+ _raise_if_frozen_measure_conflicts(frozen_measure, measure)
+
+
+def test_frozen_measure_conflict_rejects_different_tuning_params():
+ """
+ A name collision fails when only tuning parameters differ, preventing
+ a metric from silently reusing a sketch built with a different accuracy.
+ """
+ frozen_measure = FrozenMeasure(
+ name="latency_tdigest",
+ upstream_revision_id=1,
+ expression="latency_ms",
+ aggregation="TDIGEST_AGG",
+ params={"compression": 200},
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+ measure = MetricComponent(
+ name="latency_tdigest",
+ expression="latency_ms",
+ aggregation="TDIGEST_AGG",
+ params={"compression": 1000},
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+
+ with pytest.raises(DJInvalidInputException, match="tuning parameters"):
+ _raise_if_frozen_measure_conflicts(frozen_measure, measure)
+
+
+def test_frozen_measure_reuse_allows_matching_params():
+ """
+ Identical params are reusable; absent-vs-empty is not a conflict.
+ """
+ frozen_measure = FrozenMeasure(
+ name="latency_tdigest",
+ upstream_revision_id=1,
+ expression="latency_ms",
+ aggregation="TDIGEST_AGG",
+ params={"compression": 200},
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+ measure = MetricComponent(
+ name="latency_tdigest",
+ expression="latency_ms",
+ aggregation="TDIGEST_AGG",
+ params={"compression": 200},
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+ _raise_if_frozen_measure_conflicts(frozen_measure, measure)
+
+ unparameterized = FrozenMeasure(
+ name="amount_sum",
+ upstream_revision_id=1,
+ expression="amount",
+ aggregation="SUM",
+ params=None,
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+ empty_params = MetricComponent(
+ name="amount_sum",
+ expression="amount",
+ aggregation="SUM",
+ params={},
+ rule=AggregationRule(type=Aggregability.FULL),
+ )
+ _raise_if_frozen_measure_conflicts(unparameterized, empty_params)
+
+
@pytest.mark.asyncio
async def test_derived_metric_expands_parent_cache(
session: AsyncSession,
diff --git a/datajunction-server/tests/models/cube_druid_sketch_spec_test.py b/datajunction-server/tests/models/cube_druid_sketch_spec_test.py
new file mode 100644
index 000000000..09a3bb067
--- /dev/null
+++ b/datajunction-server/tests/models/cube_druid_sketch_spec_test.py
@@ -0,0 +1,196 @@
+"""
+Regression tests for sketch measures reaching Druid ingestion.
+"""
+
+from datajunction_server.api.cubes import _build_metrics_spec
+from datajunction_server.models.cube_materialization import CombineMaterialization
+from datajunction_server.models.decompose import AggregationRule, MetricComponent
+from datajunction_server.models.node_type import NodeNameVersion
+from datajunction_server.models.partition import Granularity
+from datajunction_server.models.query import ColumnMetadata
+
+TIMESTAMP_COLUMN = ColumnMetadata(name="order_date", type="int")
+
+
+def _measure(
+ name: str,
+ aggregation: str | None,
+ merge: str | None = None,
+) -> MetricComponent:
+ """A measure component as the decomposer would emit it."""
+ return MetricComponent(
+ name=name,
+ expression=name,
+ aggregation=aggregation,
+ merge=merge,
+ rule=AggregationRule(),
+ )
+
+
+def _combiner(
+ measures: list[MetricComponent],
+ measure_columns: list[ColumnMetadata],
+) -> CombineMaterialization:
+ """A combiner stage carrying real measures, ready to build a Druid spec."""
+ return CombineMaterialization(
+ node=NodeNameVersion(name="default.repairs_cube", version="v1.0"),
+ columns=[TIMESTAMP_COLUMN, *measure_columns],
+ grain=["order_date"],
+ dimensions=["order_date"],
+ measures=measures,
+ timestamp_column="order_date",
+ timestamp_format="yyyyMMdd",
+ granularity=Granularity.DAY,
+ )
+
+
+class TestCombinerSketchMetricsSpec:
+ """``CombineMaterialization.metrics_spec`` with measures that carry sketches."""
+
+ def test_hll_measure_maps_to_hll_sketch_merge(self):
+ """
+ An HLL sketch measure reaches Druid as an HLLSketchMerge aggregator.
+ """
+ combiner = _combiner(
+ measures=[
+ _measure("customer_id_hll_23002251", "hll_sketch_agg", "hll_union_agg"),
+ ],
+ measure_columns=[
+ ColumnMetadata(name="customer_id_hll_23002251", type="binary"),
+ ],
+ )
+
+ assert combiner.metrics_spec() == [
+ {
+ "fieldName": "customer_id_hll_23002251",
+ "name": "customer_id_hll_23002251",
+ "type": "HLLSketchMerge",
+ "lgK": 12,
+ "tgtHllType": "HLL_4",
+ },
+ ]
+
+ def test_hll_measure_is_neither_dropped_nor_downgraded(self):
+ """
+ The sketch measure survives as a sketch aggregator.
+ """
+ combiner = _combiner(
+ measures=[
+ _measure("customer_id_hll_23002251", "hll_sketch_agg", "hll_union_agg"),
+ ],
+ measure_columns=[
+ ColumnMetadata(name="customer_id_hll_23002251", type="binary"),
+ ],
+ )
+
+ spec = combiner.metrics_spec()
+
+ assert len(spec) == 1, "sketch measure was dropped from the metricsSpec"
+ assert spec[0]["type"] != "longSum"
+
+ def test_plain_sum_measure_maps_to_long_sum(self):
+ """A non-sketch measure still maps as before."""
+ combiner = _combiner(
+ measures=[_measure("total_repair_cost", "SUM", "SUM")],
+ measure_columns=[
+ ColumnMetadata(name="total_repair_cost", type="bigint"),
+ ],
+ )
+
+ assert combiner.metrics_spec() == [
+ {
+ "fieldName": "total_repair_cost",
+ "name": "total_repair_cost",
+ "type": "longSum",
+ },
+ ]
+
+ def test_merge_phase_takes_precedence_over_accumulate(self):
+ """
+ Pins which phase the aggregator lookup uses: ``merge``, not ``aggregation``.
+ Druid ingests already-accumulated partials, so the merge function is the
+ correct phase to key off.
+ """
+ combiner = _combiner(
+ measures=[_measure("order_count", "COUNT", "SUM")],
+ measure_columns=[ColumnMetadata(name="order_count", type="double")],
+ )
+
+ assert combiner.metrics_spec() == [
+ {
+ "fieldName": "order_count",
+ "name": "order_count",
+ "type": "doubleSum",
+ },
+ ]
+
+ def test_percentile_measure_is_dropped_silently(self):
+ """
+ A percentile measure disappears from the ingestion spec without error.
+ """
+ combiner = _combiner(
+ measures=[_measure("p95_latency", "approx_percentile")],
+ measure_columns=[ColumnMetadata(name="p95_latency", type="double")],
+ )
+
+ assert combiner.metrics_spec() == []
+
+
+class TestMetricsSpecBuildersAgree:
+ """
+ The two Druid metricsSpec builders, compared directly.
+
+ ``CombineMaterialization.metrics_spec`` and ``api.cubes._build_metrics_spec``
+ map measures for different entry points. They share ``get_druid_aggregator_spec``
+ so the aggregator type and family config agree.
+ """
+
+ def test_builders_agree_on_hll_aggregator_type(self):
+ """Both resolve an HLL sketch measure to the same aggregator type."""
+ measure = _measure(
+ "customer_id_hll_23002251",
+ "hll_sketch_agg",
+ "hll_union_agg",
+ )
+ measure_column = ColumnMetadata(
+ name="customer_id_hll_23002251",
+ type="binary",
+ )
+
+ v3_spec = _combiner([measure], [measure_column]).metrics_spec()
+ cubes_api_spec = _build_metrics_spec([measure_column], [measure], {})
+
+ assert v3_spec[0]["type"] == cubes_api_spec[0]["type"] == "HLLSketchMerge"
+
+ def test_builders_agree_on_hll_tuning_config(self):
+ """
+ Both builders attach the same HLL tuning config.
+ """
+ measure = _measure(
+ "customer_id_hll_23002251",
+ "hll_sketch_agg",
+ "hll_union_agg",
+ )
+ measure_column = ColumnMetadata(
+ name="customer_id_hll_23002251",
+ type="binary",
+ )
+
+ v3_entry = _combiner([measure], [measure_column]).metrics_spec()[0]
+ cubes_api_entry = _build_metrics_spec([measure_column], [measure], {})[0]
+
+ assert v3_entry["lgK"] == cubes_api_entry["lgK"] == 12
+ assert v3_entry["tgtHllType"] == cubes_api_entry["tgtHllType"] == "HLL_4"
+
+ def test_builders_disagree_on_unmappable_measures(self):
+ """
+ An unmappable measure is dropped by one builder and defaulted by the other.
+ """
+ measure = _measure("p95_latency", "approx_percentile")
+ measure_column = ColumnMetadata(name="p95_latency", type="double")
+
+ v3_spec = _combiner([measure], [measure_column]).metrics_spec()
+ cubes_api_spec = _build_metrics_spec([measure_column], [measure], {})
+
+ assert v3_spec == []
+ assert cubes_api_spec[0]["type"] == "longSum"
diff --git a/datajunction-server/tests/models/deployment_test.py b/datajunction-server/tests/models/deployment_test.py
index 91507b62d..a371e2f61 100644
--- a/datajunction-server/tests/models/deployment_test.py
+++ b/datajunction-server/tests/models/deployment_test.py
@@ -1706,7 +1706,7 @@ def fingerprint(spec: NodeSpec) -> SemanticFingerprint:
"source": "71dcbc388988c2bdd850670427710384687b58565ee38ca392dc220adfed868d",
"transform": "978e692880c7bcfb1bd78ece85895a1ec1558e85377b064a8dac3f3719cff2a5",
"dimension": "f7b3c87a61fdadf9997432fd9334befdf43f2488874ef555e3f7d4c4ba86e3e1",
- "metric": "a3fde7af5dbd00d194805af33fc213bca52244c7cdedf1f7363ec52d2f6d4116",
+ "metric": "f0b5356d75b1f6a39a2809581a9167b9e1298a41980ee8e83040cf6bd0996851",
"cube": "9b0a56d974d1e3769bc2db94e2cfbae7a6a4839f664eebd2a4387ef112ceea81",
}
@@ -1853,6 +1853,38 @@ def test_semantic_fingerprint_renders_prefixes_and_normalizes_sql():
assert fingerprint(explicit) != fingerprint(implicit)
+def test_metric_spec_equality_compares_rendered_reaggregate():
+ """Parameterized reaggregate dimensions should not cause perpetual deploys."""
+ parameterized = MetricSpec(
+ namespace="analytics",
+ name="daily_balance",
+ query="SELECT SUM(balance) FROM analytics.daily_balances",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "${prefix}date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ rendered = MetricSpec(
+ namespace="analytics",
+ name="daily_balance",
+ query="SELECT SUM(balance) FROM analytics.daily_balances",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "analytics.date.date_id[order]",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+
+ assert parameterized == rendered
+
+
def test_semantic_diff_and_fingerprint_share_change_rules():
original = TransformSpec(name="node", query="SELECT id AS value FROM source")
formatted = TransformSpec(
@@ -1958,6 +1990,7 @@ def test_metric_presentation_fields_preserve_semantic_fingerprint():
"query",
"columns",
"required_dimensions",
+ "reaggregate",
}
)
assert all(
diff --git a/datajunction-server/tests/models/node_test.py b/datajunction-server/tests/models/node_test.py
index 1489d3928..3a630811c 100644
--- a/datajunction-server/tests/models/node_test.py
+++ b/datajunction-server/tests/models/node_test.py
@@ -227,6 +227,30 @@ def test_extra_validation() -> None:
"bound dimensions which are only for metrics."
)
+ node = Node(name="A", type=NodeType.TRANSFORM, current_version="1")
+ node_revision = NodeRevision(
+ name=node.name,
+ type=node.type,
+ node=node,
+ version="1",
+ query="SELECT * FROM B",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "B.date_id",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+ with pytest.raises(Exception) as excinfo:
+ node_revision.extra_validation()
+
+ assert str(excinfo.value) == (
+ "Node A of type transform cannot have "
+ "reaggregate settings which are only for metrics."
+ )
+
def test_merging_availability_simple_no_partitions() -> None:
"""
diff --git a/datajunction-server/tests/models/reaggregate_test.py b/datajunction-server/tests/models/reaggregate_test.py
new file mode 100644
index 000000000..3cf21b60d
--- /dev/null
+++ b/datajunction-server/tests/models/reaggregate_test.py
@@ -0,0 +1,188 @@
+"""Tests for reaggregation models."""
+
+import pytest
+from pydantic import ValidationError
+
+from datajunction_server.models import reaggregate as reaggregate_module
+from datajunction_server.models.reaggregate import (
+ DimensionReaggregateRule,
+ ReaggregateSpec,
+ ReaggregationFunction,
+ dimension_reaggregate_rules,
+ dump_reaggregate_spec,
+ is_parameterized_reaggregate_function,
+ parse_reaggregate_spec,
+ unsupported_dimension_reaggregate_functions,
+)
+
+
+def test_dump_reaggregate_spec_from_dict():
+ """
+ Reaggregate specs passed as dictionaries are validated and serialized.
+ """
+ assert dump_reaggregate_spec(
+ {
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ },
+ ) == {
+ "fn": None,
+ "weight": None,
+ "params": None,
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ }
+
+
+def test_dump_reaggregate_spec_from_model():
+ """
+ Reaggregate spec models are serialized to JSON-compatible dictionaries.
+ """
+ assert dump_reaggregate_spec(
+ ReaggregateSpec(
+ rules=[
+ DimensionReaggregateRule(
+ dimension="default.date_dim.date",
+ fn=ReaggregationFunction.LAST_VALUE,
+ ),
+ ],
+ ),
+ ) == {
+ "fn": None,
+ "weight": None,
+ "params": None,
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ }
+
+
+def test_dump_reaggregate_spec_none():
+ """
+ Missing reaggregate specs pass through unchanged.
+ """
+ assert dump_reaggregate_spec(None) is None
+
+
+def test_parse_reaggregate_spec_from_model():
+ """
+ Parsed model inputs are returned unchanged.
+ """
+ spec = ReaggregateSpec(
+ rules=[
+ DimensionReaggregateRule(
+ dimension="default.date_dim.date",
+ fn=ReaggregationFunction.LAST_VALUE,
+ ),
+ ],
+ )
+
+ assert parse_reaggregate_spec(spec) is spec
+
+
+def test_dimension_reaggregate_rules_empty_for_none():
+ """
+ Empty reaggregate specs have no dimension-specific rules.
+ """
+ assert dimension_reaggregate_rules(None) == []
+
+
+def test_unsupported_dimension_reaggregate_functions_handles_empty_and_invalid():
+ """
+ Unsupported dimension collapse functions are reported from parsed specs.
+ """
+ assert unsupported_dimension_reaggregate_functions(None) == []
+ assert unsupported_dimension_reaggregate_functions(
+ {
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "sum",
+ },
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ },
+ ) == ["sum"]
+
+
+def test_params_rejected_for_unparameterized_function():
+ """
+ Functions without tuneable parameters reject `params`.
+ """
+ with pytest.raises(ValidationError) as excinfo:
+ ReaggregateSpec(fn=ReaggregationFunction.SUM, params={"compression": 200})
+ assert "`sum` does not accept parameters" in str(excinfo.value)
+ assert "compression" in str(excinfo.value)
+
+
+def test_params_rejected_when_no_function_given():
+ """
+ `params` without an `fn` is rejected.
+ """
+ with pytest.raises(ValidationError) as excinfo:
+ ReaggregateSpec(params={"k": 256})
+ assert "no reaggregation function does not accept parameters" in str(excinfo.value)
+
+
+def test_empty_params_allowed():
+ """
+ An absent or empty `params` is fine on any function.
+ """
+ assert ReaggregateSpec(fn=ReaggregationFunction.SUM).params is None
+ assert ReaggregateSpec(fn=ReaggregationFunction.SUM, params={}).params == {}
+
+
+def test_params_accepted_for_parameterized_function(monkeypatch):
+ """
+ Registering a function as parameterized enables `params` assignment.
+ """
+ monkeypatch.setattr(
+ reaggregate_module,
+ "PARAMETERIZED_REAGGREGATE_FUNCTIONS",
+ frozenset({ReaggregationFunction.AVG}),
+ )
+ spec = ReaggregateSpec(fn=ReaggregationFunction.AVG, params={"compression": 200})
+ assert spec.params == {"compression": 200}
+ assert dump_reaggregate_spec(spec)["params"] == {"compression": 200}
+
+
+def test_only_sketch_families_are_parameterized():
+ """
+ Only a sketch family takes tuning parameters.
+ """
+ parameterized = {
+ fn for fn in ReaggregationFunction if is_parameterized_reaggregate_function(fn)
+ }
+
+ assert parameterized == {ReaggregationFunction.TDIGEST}
+
+
+def test_params_round_trip_through_parse():
+ """
+ `params` survives dict -> model -> dict without loss.
+ """
+ monkeypatched = frozenset({ReaggregationFunction.AVG})
+ original = reaggregate_module.PARAMETERIZED_REAGGREGATE_FUNCTIONS
+ reaggregate_module.PARAMETERIZED_REAGGREGATE_FUNCTIONS = monkeypatched
+ try:
+ spec = parse_reaggregate_spec({"fn": "avg", "params": {"compression": 200}})
+ assert spec.params == {"compression": 200}
+ assert parse_reaggregate_spec(dump_reaggregate_spec(spec)).params == {
+ "compression": 200,
+ }
+ finally:
+ reaggregate_module.PARAMETERIZED_REAGGREGATE_FUNCTIONS = original
diff --git a/datajunction-server/tests/sql/decompose_test.py b/datajunction-server/tests/sql/decompose_test.py
index b2c93644b..af8607eb0 100644
--- a/datajunction-server/tests/sql/decompose_test.py
+++ b/datajunction-server/tests/sql/decompose_test.py
@@ -2,23 +2,38 @@
Tests for ``datajunction_server.sql.decompose``.
"""
+from types import SimpleNamespace
+
import pytest
import pytest_asyncio
from sqlalchemy.ext.asyncio import AsyncSession
from datajunction_server.database.node import Node, NodeRelationship, NodeRevision
+from datajunction_server.errors import DJInvalidInputException
from datajunction_server.models.cube_materialization import (
Aggregability,
AggregationRule,
MetricComponent,
)
from datajunction_server.models.engine import Dialect
+from datajunction_server.models.materialization import MaterializationTarget
from datajunction_server.models.node_type import NodeType
+from datajunction_server.models.reaggregate import (
+ DimensionReaggregateRule,
+ ReaggregateSpec,
+ ReaggregationFunction,
+)
from datajunction_server.sql import functions as dj_functions
from datajunction_server.sql.decompose import (
+ DECOMPOSITION_REGISTRY,
+ FAMILY_DECOMPOSITION_REGISTRY,
DUPLICATION_INVARIANT_AGGREGATIONS,
+ AggDecomposition,
+ ComponentDef,
MetricComponentExtractor,
+ decomposes_family,
get_decomposition,
+ make_func,
is_duplication_invariant,
is_metric_duplication_sensitive,
safe_denominator,
@@ -62,7 +77,12 @@ async def create_metric(session: AsyncSession, current_user, parent_node):
"""Fixture to create a metric node with a query."""
created_metrics: list[NodeRevision] = []
- async def _create(query: str, name: str | None = None, parent=None):
+ async def _create(
+ query: str,
+ name: str | None = None,
+ parent=None,
+ reaggregate: dict[str, object] | None = None,
+ ):
parent_to_use = parent if parent else parent_node
metric_name = name or f"test_metric_{len(created_metrics)}"
@@ -81,6 +101,7 @@ async def _create(query: str, name: str | None = None, parent=None):
name=metric_name,
type=NodeType.METRIC,
query=query,
+ reaggregate=reaggregate,
created_by_id=current_user.id,
)
session.add(metric_rev)
@@ -120,6 +141,105 @@ async def test_simple_sum(session: AsyncSession, create_metric):
)
+@pytest.mark.asyncio
+async def test_reaggregate_sum_attaches_spec(session: AsyncSession, create_metric):
+ """
+ Semi-additive declarations attach to the single extracted base measure.
+ """
+ metric_rev = await create_metric(
+ "SELECT SUM(account_balance) FROM parent_node",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+
+ extractor = MetricComponentExtractor(metric_rev.id)
+ measures, derived_sql = await extractor.extract(session)
+
+ expected_spec = DimensionReaggregateRule(
+ dimension="default.date_dim.date",
+ fn=ReaggregationFunction.LAST_VALUE,
+ )
+ assert measures == [
+ MetricComponent(
+ name="account_balance_sum_8e611a76",
+ expression="account_balance",
+ aggregation="SUM",
+ merge="SUM",
+ rule=AggregationRule(
+ type=Aggregability.FULL,
+ reaggregate=expected_spec,
+ ),
+ ),
+ ]
+ assert_sql_equal(
+ str(derived_sql),
+ "SELECT SUM(account_balance_sum_8e611a76) FROM parent_node",
+ )
+
+
+@pytest.mark.asyncio
+async def test_reaggregate_multiple_rules_not_supported(
+ session: AsyncSession,
+ create_metric,
+):
+ """
+ V1 semi-additive metrics support one dimension-specific rule.
+ """
+ metric_rev = await create_metric(
+ "SELECT SUM(account_balance) FROM parent_node",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ {
+ "dimension": "default.region_dim.region",
+ "fn": "first_value",
+ },
+ ],
+ },
+ )
+
+ extractor = MetricComponentExtractor(metric_rev.id)
+ with pytest.raises(DJInvalidInputException, match="exactly one rule"):
+ await extractor.extract(session)
+
+
+@pytest.mark.asyncio
+async def test_reaggregate_unsupported_dimension_function_not_supported(
+ session: AsyncSession,
+ create_metric,
+):
+ """
+ Dimension-specific reaggregate accepts only collapse-safe functions.
+ """
+ metric_rev = await create_metric(
+ "SELECT SUM(account_balance) FROM parent_node",
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "sum",
+ },
+ ],
+ },
+ )
+
+ extractor = MetricComponentExtractor(metric_rev.id)
+ with pytest.raises(
+ DJInvalidInputException,
+ match="unsupported dimension reaggregation function",
+ ):
+ await extractor.extract(session)
+
+
@pytest.mark.asyncio
async def test_sum_with_cast(session: AsyncSession, create_metric):
"""
@@ -854,6 +974,50 @@ async def test_unsupported_aggregation_function(session: AsyncSession, create_me
)
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("query", "match"),
+ [
+ (
+ "SELECT AVG(sales_amount) FROM parent_node",
+ "exactly one component",
+ ),
+ (
+ "SELECT COUNT(DISTINCT user_id) FROM parent_node",
+ "one non-distinct fully-aggregatable component",
+ ),
+ (
+ "SELECT MEDIAN(sales_amount) FROM parent_node",
+ "decomposable aggregation",
+ ),
+ ],
+)
+async def test_unsupported_reaggregate_shapes(
+ session: AsyncSession,
+ create_metric,
+ query: str,
+ match: str,
+):
+ """
+ V1 semi-additive metrics only support one ordinary decomposable component.
+ """
+ metric_rev = await create_metric(
+ query,
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+
+ extractor = MetricComponentExtractor(metric_rev.id)
+ with pytest.raises(DJInvalidInputException, match=match):
+ await extractor.extract(session)
+
+
@pytest.mark.asyncio
async def test_count_if(session: AsyncSession, create_metric):
"""
@@ -1539,7 +1703,12 @@ async def create_derived_metric(clean_session: AsyncSession, clean_current_user)
session = clean_session
current_user = clean_current_user
- async def _create(name: str, query: str, base_metric_nodes: list[Node]):
+ async def _create(
+ name: str,
+ query: str,
+ base_metric_nodes: list[Node],
+ reaggregate: dict[str, object] | None = None,
+ ):
metric_node = Node(
name=name,
type=NodeType.METRIC,
@@ -1555,6 +1724,7 @@ async def _create(name: str, query: str, base_metric_nodes: list[Node]):
name=name,
type=NodeType.METRIC,
query=query,
+ reaggregate=reaggregate,
created_by_id=current_user.id,
)
session.add(metric_rev)
@@ -1571,6 +1741,116 @@ async def _create(name: str, query: str, base_metric_nodes: list[Node]):
return _create
+@pytest.mark.asyncio
+async def test_derived_metric_reaggregate_not_supported_db_load(
+ clean_session: AsyncSession,
+ create_base_metric,
+ create_derived_metric,
+):
+ """
+ Semi-additive declarations are currently limited to base metrics.
+ """
+ session = clean_session
+ balance_node, _ = await create_base_metric(
+ "default.balance",
+ "SELECT SUM(account_balance) FROM parent_node",
+ )
+ _, derived_rev = await create_derived_metric(
+ "default.double_balance",
+ "SELECT default.balance * 2",
+ [balance_node],
+ reaggregate={
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ },
+ )
+
+ extractor = MetricComponentExtractor(derived_rev.id)
+ with pytest.raises(
+ DJInvalidInputException,
+ match="Derived metric `default.double_balance` declares reaggregate",
+ ):
+ await extractor.extract(session)
+
+
+def test_derived_metric_reaggregate_not_supported_cache_load():
+ """
+ Cache-based extraction enforces the same base-metric-only constraint.
+ """
+ reaggregate = {
+ "rules": [
+ {
+ "dimension": "default.date_dim.date",
+ "fn": "last_value",
+ },
+ ],
+ }
+ base_node = SimpleNamespace(
+ name="default.balance",
+ type=NodeType.METRIC,
+ current=SimpleNamespace(
+ query="SELECT SUM(account_balance) FROM parent_node",
+ reaggregate=None,
+ ),
+ )
+ derived_node = SimpleNamespace(
+ name="default.double_balance",
+ type=NodeType.METRIC,
+ current=SimpleNamespace(
+ query="SELECT default.balance * 2",
+ reaggregate=reaggregate,
+ ),
+ )
+
+ extractor = MetricComponentExtractor(0)
+ with pytest.raises(
+ DJInvalidInputException,
+ match="Derived metric `default.double_balance` declares reaggregate",
+ ):
+ extractor._build_metric_data_from_cache(
+ derived_node,
+ {
+ "default.balance": base_node,
+ "default.double_balance": derived_node,
+ },
+ {"default.double_balance": ["default.balance"]},
+ )
+
+
+def test_normalize_aliases_leaves_other_namespaces_qualified():
+ """
+ Alias normalization only strips the primary parent alias from column refs.
+ """
+ extractor = MetricComponentExtractor(0)
+ query = parse("SELECT p.value, other.value FROM parent_node p")
+
+ normalized = extractor._normalize_aliases(query)
+
+ assert_sql_equal(str(normalized), "SELECT value, other.value FROM parent_node")
+
+
+def test_substitute_metric_references_ignores_non_metric_columns():
+ """
+ Derived metric substitution leaves ordinary columns in place.
+ """
+ extractor = MetricComponentExtractor(0)
+ query = parse("SELECT default.balance + raw_value FROM parent_node")
+
+ substituted = extractor._substitute_metric_references(
+ query,
+ {"default.balance": ([], "SUM(balance)")},
+ )
+
+ assert_sql_equal(
+ str(substituted),
+ "SELECT SUM(balance) + raw_value FROM parent_node",
+ )
+
+
@pytest.mark.asyncio
async def test_extract_derived_metric_revenue_per_order(
clean_session: AsyncSession,
@@ -2404,3 +2684,437 @@ async def test_sum_abs_decomposes(session: AsyncSession, create_metric):
assert comp.rule.type == Aggregability.FULL
assert "ABS" in comp.expression
assert_sql_equal(str(derived_sql), f"SELECT SUM({comp.name}) FROM parent_node")
+
+
+class TestReaggregateParams:
+ """
+ Tests that sketch tuning parameters flow through to components.
+ """
+
+ @staticmethod
+ def _spec(params):
+ """A spec carrying params."""
+ return ReaggregateSpec.model_construct(
+ fn=ReaggregationFunction.SUM,
+ weight=None,
+ rules=[],
+ params=params,
+ )
+
+ def test_params_reach_components(self):
+ extractor = MetricComponentExtractor(1)
+ components, _ = extractor._extract_base(
+ parse("SELECT SUM(latency_ms) FROM t"),
+ self._spec({"compression": 200}),
+ )
+ assert [component.params for component in components] == [
+ {"compression": 200},
+ ]
+
+ def test_params_reach_every_component_of_a_multi_component_metric(self):
+ """AVG decomposes to a sum and a count; both are the same sketch family."""
+ extractor = MetricComponentExtractor(1)
+ components, _ = extractor._extract_base(
+ parse("SELECT AVG(latency_ms) FROM t"),
+ self._spec({"compression": 200}),
+ )
+ assert len(components) == 2
+ assert all(c.params == {"compression": 200} for c in components)
+
+ def test_params_are_copied_not_shared(self):
+ """
+ Each component receives an independent copy of its params dict.
+ """
+ extractor = MetricComponentExtractor(1)
+ params = {"compression": 200}
+ components, _ = extractor._extract_base(
+ parse("SELECT AVG(latency_ms) FROM t"),
+ self._spec(params),
+ )
+ params["compression"] = 999
+ assert all(c.params == {"compression": 200} for c in components)
+ assert components[0].params is not components[1].params
+
+ def test_no_params_leaves_components_untouched(self):
+ """Omitting params applies no configuration to components."""
+ extractor = MetricComponentExtractor(1)
+ components, _ = extractor._extract_base(
+ parse("SELECT SUM(latency_ms) FROM t"),
+ self._spec(None),
+ )
+ assert [component.params for component in components] == [None]
+
+ def test_params_without_an_aggregating_component_is_rejected(self):
+ """
+ Declaring params on a non-aggregating metric is rejected.
+ """
+ extractor = MetricComponentExtractor(1)
+ with pytest.raises(DJInvalidInputException) as excinfo:
+ extractor._extract_base(
+ parse("SELECT COUNT(DISTINCT order_id) FROM t"),
+ self._spec({"compression": 200}),
+ )
+ assert "requires an aggregating component" in str(excinfo.value)
+
+
+# =============================================================================
+# Dialect-aware combiners
+# =============================================================================
+
+
+class _DialectAwareSum(AggDecomposition):
+ """
+ A SUM whose combiner renders differently per dialect, recording the call it
+ was handed. Tests the pipeline for dialect-specific extraction logic.
+ """
+
+ seen: list[tuple["ast.Function", Dialect]] = []
+
+ @property
+ def components(self) -> list[ComponentDef]:
+ return [ComponentDef(suffix="_sum", accumulate="SUM", merge="SUM")]
+
+ def combine(self, components, func, dialect=Dialect.SPARK):
+ type(self).seen.append((func, dialect))
+
+ combiners = {
+ Dialect.DRUID: self._combine_druid,
+ Dialect.SPARK: self._combine_spark,
+ }
+ return combiners.get(dialect, self._combine_spark)(components)
+
+ def _combine_druid(self, components):
+ return make_func("druid_combiner", components[0].name)
+
+ def _combine_spark(self, components):
+ return make_func("spark_combiner", components[0].name)
+
+
+@pytest.fixture
+def dialect_aware_sum():
+ """Swap SUM's decomposition for one that branches on dialect, then restore."""
+ original = DECOMPOSITION_REGISTRY.get(dj_functions.Sum)
+ _DialectAwareSum.seen = []
+ DECOMPOSITION_REGISTRY[dj_functions.Sum] = _DialectAwareSum
+ try:
+ yield _DialectAwareSum
+ finally:
+ DECOMPOSITION_REGISTRY[dj_functions.Sum] = original
+
+
+def test_combine_renders_per_dialect():
+ """A decomposition can key its combiner off the target dialect."""
+ components = [
+ MetricComponent(
+ name="price_sum_abc",
+ expression="price",
+ aggregation="SUM",
+ merge="SUM",
+ rule=AggregationRule(),
+ ),
+ ]
+ func = parse("SELECT SUM(price) FROM parent_node").select.projection[0]
+ decomposition = _DialectAwareSum()
+
+ spark = decomposition.combine(components, func, Dialect.SPARK)
+ druid = decomposition.combine(components, func, Dialect.DRUID)
+
+ assert "spark_combiner" in str(spark)
+ assert "druid_combiner" in str(druid)
+ assert str(spark) != str(druid)
+
+
+def test_combine_defaults_to_spark():
+ """
+ Omitting the dialect yields the Spark rendering.
+ """
+ components = [
+ MetricComponent(
+ name="price_sum_abc",
+ expression="price",
+ aggregation="SUM",
+ merge="SUM",
+ rule=AggregationRule(),
+ ),
+ ]
+ func = parse("SELECT SUM(price) FROM parent_node").select.projection[0]
+
+ assert "spark_combiner" in str(_DialectAwareSum().combine(components, func))
+
+
+def test_combine_receives_the_originating_call():
+ """
+ The combiner is handed the authored ``ast.Function``, arguments included.
+ """
+ components = [
+ MetricComponent(
+ name="latency_p50",
+ expression="latency_ms",
+ aggregation="SUM",
+ merge="SUM",
+ rule=AggregationRule(),
+ ),
+ ]
+ func = parse(
+ "SELECT APPROX_PERCENTILE(latency_ms, 0.5) FROM parent_node",
+ ).select.projection[0]
+
+ decomposition = _DialectAwareSum()
+ decomposition.seen = []
+ decomposition.combine(components, func, Dialect.DRUID)
+
+ seen_func, seen_dialect = _DialectAwareSum.seen[-1]
+ assert seen_dialect == Dialect.DRUID
+ assert str(seen_func.args[1]) == "0.5"
+
+
+def test_extractor_defaults_to_spark_dialect():
+ """An extractor built without a dialect renders for Spark."""
+ assert MetricComponentExtractor(1)._dialect == Dialect.SPARK
+
+
+@pytest.mark.asyncio
+async def test_extractor_threads_dialect_into_combiner(
+ session: AsyncSession,
+ create_metric,
+ dialect_aware_sum,
+):
+ """
+ The dialect handed to the extractor reaches the combiner.
+ """
+ metric_rev = await create_metric("SELECT SUM(price) FROM parent_node")
+
+ _, spark_sql = await MetricComponentExtractor(metric_rev.id).extract(session)
+ _, druid_sql = await MetricComponentExtractor(
+ metric_rev.id,
+ dialect=Dialect.DRUID,
+ ).extract(session)
+
+ assert "spark_combiner" in str(spark_sql)
+ assert "druid_combiner" in str(druid_sql)
+
+
+@pytest.mark.asyncio
+async def test_extractor_dialect_does_not_leak_between_instances(
+ session: AsyncSession,
+ create_metric,
+ dialect_aware_sum,
+):
+ """
+ Dialect is per-extractor configuration, not shared state.
+ """
+ metric_rev = await create_metric("SELECT SUM(price) FROM parent_node")
+
+ await MetricComponentExtractor(metric_rev.id, dialect=Dialect.DRUID).extract(
+ session,
+ )
+ _, after = await MetricComponentExtractor(metric_rev.id).extract(session)
+
+ assert "spark_combiner" in str(after)
+ assert "druid_combiner" not in str(after)
+
+
+class _SerializingSum(AggDecomposition):
+ """A SUM that declares a Druid-only serialize conversion."""
+
+ @property
+ def components(self) -> list[ComponentDef]:
+ return [
+ ComponentDef(
+ suffix="_sum",
+ accumulate="SUM",
+ merge="SUM",
+ serialize="to_druid_bytes({})",
+ serialize_targets=(MaterializationTarget.DRUID,),
+ ),
+ ]
+
+ def combine(self, components, func, dialect=Dialect.SPARK):
+ return make_func("SUM", components[0].name)
+
+
+@pytest.fixture
+def serializing_sum():
+ """Swap SUM's decomposition for one declaring a serialize, then restore."""
+ original = DECOMPOSITION_REGISTRY.get(dj_functions.Sum)
+ DECOMPOSITION_REGISTRY[dj_functions.Sum] = _SerializingSum
+ try:
+ yield
+ finally:
+ DECOMPOSITION_REGISTRY[dj_functions.Sum] = original
+
+
+@pytest.mark.asyncio
+async def test_serialize_propagates_from_component_def(
+ session: AsyncSession,
+ create_metric,
+ serializing_sum,
+):
+ """
+ A ``serialize`` declared on ``ComponentDef`` reaches the extracted component.
+ """
+ metric_rev = await create_metric("SELECT SUM(price) FROM parent_node")
+
+ components, _ = await MetricComponentExtractor(metric_rev.id).extract(session)
+
+ assert len(components) == 1
+ assert components[0].serialize == "to_druid_bytes({})"
+ assert components[0].serialize_targets == [MaterializationTarget.DRUID]
+
+
+@pytest.mark.asyncio
+async def test_components_without_serialize_declare_none(
+ session: AsyncSession,
+ create_metric,
+):
+ """An ordinary decomposition leaves the conversion unset."""
+ metric_rev = await create_metric("SELECT SUM(price) FROM parent_node")
+
+ components, _ = await MetricComponentExtractor(metric_rev.id).extract(session)
+
+ assert components[0].serialize is None
+ assert components[0].serialize_targets == []
+
+
+# =============================================================================
+# Family-gated decompositions
+# =============================================================================
+
+
+@pytest.fixture
+def registered_family():
+ """
+ Register a decomposition for a reaggregation family, then remove it.
+
+ OSS ships no family implementations -- they live downstream with the
+ engine-specific functions -- so a test has to supply one to exercise the
+ selection path at all.
+ """
+
+ @decomposes_family(ReaggregationFunction.TDIGEST)
+ class _FamilyDecomposition(AggDecomposition):
+ @property
+ def components(self) -> list[ComponentDef]:
+ return [
+ ComponentDef(
+ suffix="_sketch",
+ accumulate=f"build_sketch({{}}, {self.params.get('compression', 1)})",
+ merge="merge_sketch",
+ ),
+ ]
+
+ def combine(self, components, func, dialect=Dialect.SPARK):
+ return make_func("read_sketch", components[0].name)
+
+ try:
+ yield _FamilyDecomposition
+ finally:
+ FAMILY_DECOMPOSITION_REGISTRY.pop(ReaggregationFunction.TDIGEST, None)
+
+
+def test_declared_family_overrides_the_function_registry(registered_family):
+ """
+ A metric declaring a family gets that family's decomposition.
+
+ This is what lets `APPROX_PERCENTILE(x, 0.95)` keep its spelling and still
+ decompose: the aggregation function is unchanged, the family does the work.
+ """
+ spec = ReaggregateSpec(fn=ReaggregationFunction.TDIGEST)
+
+ decomposition = get_decomposition(dj_functions.ApproxPercentile, spec)
+
+ assert isinstance(decomposition, registered_family)
+
+
+def test_family_receives_its_tuning_params(registered_family):
+ """
+ `reaggregate.params` reaches the decomposition, and thus the accumulate.
+ """
+ spec = ReaggregateSpec(
+ fn=ReaggregationFunction.TDIGEST,
+ params={"compression": 500},
+ )
+
+ decomposition = get_decomposition(dj_functions.ApproxPercentile, spec)
+
+ assert decomposition.params == {"compression": 500}
+ assert decomposition.components[0].accumulate == "build_sketch({}, 500)"
+
+
+def test_undeclared_metric_is_untouched_by_a_registered_family(registered_family):
+ """
+ A metric without `reaggregate` keeps the by-function decomposition.
+ """
+ assert get_decomposition(dj_functions.ApproxPercentile) is None
+ assert isinstance(
+ get_decomposition(dj_functions.Sum),
+ type(get_decomposition(dj_functions.Sum)),
+ )
+ assert get_decomposition(dj_functions.Sum) is not None
+
+
+def test_unregistered_family_falls_through():
+ """
+ Declaring a family nothing implements degrades rather than raising.
+ """
+ spec = ReaggregateSpec(fn=ReaggregationFunction.TDIGEST)
+
+ assert get_decomposition(dj_functions.ApproxPercentile, spec) is None
+ assert get_decomposition(dj_functions.Sum, spec) is not None
+
+
+@pytest.mark.asyncio
+async def test_family_gated_percentile_decomposes_through_the_extractor(
+ session: AsyncSession,
+ create_metric,
+ registered_family,
+):
+ """
+ A family-gated metric survives the decomposability gate.
+
+ The regression this guards: `_extract_base` aborts early if any aggregation
+ is non-decomposable, and that check used to ask `get_decomposition(fn)`
+ without the declared family. A family-gated metric is precisely one whose
+ aggregation function has no entry of its own, so every one of them aborted
+ there and fell through to native grain -- the family registry was correct
+ and simply never reached.
+
+ Testing `get_decomposition` directly cannot catch this, and neither can a
+ family test over `SUM`, which passes the gate on its own merits. It needs a
+ non-decomposable function driven through the extractor.
+ """
+ metric_rev = await create_metric(
+ "SELECT APPROX_PERCENTILE(price, 0.95) FROM parent_node",
+ reaggregate={"fn": "tdigest", "params": {"compression": 200}},
+ )
+
+ components, combiner = await MetricComponentExtractor(metric_rev.id).extract(
+ session,
+ )
+
+ assert components, "family-gated percentile produced no components"
+ assert components[0].merge == "merge_sketch"
+ assert "build_sketch" in components[0].aggregation
+ assert "read_sketch" in str(combiner)
+
+
+@pytest.mark.asyncio
+async def test_percentile_without_a_family_still_does_not_decompose(
+ session: AsyncSession,
+ create_metric,
+ registered_family,
+):
+ """
+ The gate still stops metrics that did not opt in.
+
+ The fix must not make every percentile decomposable -- only the ones that
+ declared the family. A registered family that nobody references changes
+ nothing.
+ """
+ metric_rev = await create_metric(
+ "SELECT APPROX_PERCENTILE(price, 0.95) FROM parent_node",
+ )
+
+ components, _ = await MetricComponentExtractor(metric_rev.id).extract(session)
+
+ assert components == []
diff --git a/datajunction-server/tests/sql/functions_test.py b/datajunction-server/tests/sql/functions_test.py
index 10a1290df..964d2815b 100644
--- a/datajunction-server/tests/sql/functions_test.py
+++ b/datajunction-server/tests/sql/functions_test.py
@@ -288,6 +288,33 @@ async def test_approx_percentile(session: AsyncSession):
assert not exc.errors
assert query_with_list.select.projection[0].type == ct.FloatType() # type: ignore
+ # The two-argument form -- `accuracy` is optional in Spark, and this is the
+ # spelling metric authors actually use.
+ query = parse("SELECT approx_percentile(10.0, 0.5)")
+ exc = DJException()
+ ctx = ast.CompileContext(session=session, exception=exc)
+ await query.compile(ctx)
+ assert not exc.errors
+ assert query.select.projection[0].type == ct.FloatType() # type: ignore
+
+ query = parse("SELECT approx_percentile(10.0, array(0.5, 0.9))")
+ exc = DJException()
+ ctx = ast.CompileContext(session=session, exception=exc)
+ await query.compile(ctx)
+ assert not exc.errors
+ assert query.select.projection[0].type == ct.ListType( # type: ignore
+ element_type=ct.FloatType(),
+ )
+
+ # A double-typed percentage dispatches too: DoubleType and FloatType are
+ # siblings under FloatingBase, so a FloatType-only registration missed it.
+ query = parse("SELECT approx_percentile(10.0, CAST(0.5 AS DOUBLE))")
+ exc = DJException()
+ ctx = ast.CompileContext(session=session, exception=exc)
+ await query.compile(ctx)
+ assert not exc.errors
+ assert query.select.projection[0].type == ct.FloatType() # type: ignore
+
@pytest.mark.asyncio
async def test_array(session: AsyncSession):
diff --git a/datajunction-ui/src/app/pages/AddEditNodePage/ReaggregateFields.jsx b/datajunction-ui/src/app/pages/AddEditNodePage/ReaggregateFields.jsx
new file mode 100644
index 000000000..d163361b9
--- /dev/null
+++ b/datajunction-ui/src/app/pages/AddEditNodePage/ReaggregateFields.jsx
@@ -0,0 +1,79 @@
+/**
+ * Semi-additive metric controls.
+ */
+import { ErrorMessage, Field, useField, useFormikContext } from 'formik';
+import { useContext, useEffect, useMemo, useState } from 'react';
+import DJClientContext from '../../providers/djclient';
+import { labelize } from '../../../utils/form';
+import { FormikSelect } from './FormikSelect';
+
+const SEMI_ADDITIVE_FUNCTIONS = ['last_value', 'first_value', 'min', 'max'];
+
+export const ReaggregateFields = () => {
+ const djClient = useContext(DJClientContext).DataJunctionAPI;
+ const { values } = useFormikContext();
+ const [dimensionField] = useField('reaggregate_dimension');
+ const [dimensionOptions, setDimensionOptions] = useState([]);
+
+ useEffect(() => {
+ const fetchData = async () => {
+ if (values.upstream_node) {
+ const data = await djClient.node(values.upstream_node);
+ setDimensionOptions(
+ data.columns.map(col => ({
+ value: col.name,
+ label: col.name,
+ })),
+ );
+ } else {
+ setDimensionOptions([]);
+ }
+ };
+ fetchData().catch(console.error);
+ }, [djClient, values.upstream_node]);
+
+ const selectOptions = useMemo(() => {
+ if (
+ !dimensionField.value ||
+ dimensionOptions.some(option => option.value === dimensionField.value)
+ ) {
+ return dimensionOptions;
+ }
+ return [
+ { value: dimensionField.value, label: dimensionField.value },
+ ...dimensionOptions,
+ ];
+ }, [dimensionField.value, dimensionOptions]);
+
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {SEMI_ADDITIVE_FUNCTIONS.map(func => (
+
+ ))}
+
+
+
+ );
+};
diff --git a/datajunction-ui/src/app/pages/AddEditNodePage/__tests__/AddEditNodePageFormSuccess.test.jsx b/datajunction-ui/src/app/pages/AddEditNodePage/__tests__/AddEditNodePageFormSuccess.test.jsx
index f399c4b8d..a7bbdd578 100644
--- a/datajunction-ui/src/app/pages/AddEditNodePage/__tests__/AddEditNodePageFormSuccess.test.jsx
+++ b/datajunction-ui/src/app/pages/AddEditNodePage/__tests__/AddEditNodePageFormSuccess.test.jsx
@@ -156,6 +156,99 @@ describe('AddEditNodePage submission succeeded', () => {
);
}, 60000);
+ it('for creating a semi-additive metric node', async () => {
+ const mockDjClient = initializeMockDJClient();
+ mockDjClient.DataJunctionAPI.createNode.mockReturnValue({
+ status: 200,
+ json: { name: 'default.some_test_metric' },
+ });
+
+ mockDjClient.DataJunctionAPI.tagsNode.mockReturnValue({
+ status: 200,
+ json: { message: 'Success' },
+ });
+
+ mockDjClient.DataJunctionAPI.node.mockResolvedValue({
+ columns: [{ name: 'v3.date.date_id[order]' }],
+ });
+
+ mockDjClient.DataJunctionAPI.listTags.mockReturnValue([
+ { name: 'purpose', display_name: 'Purpose' },
+ { name: 'intent', display_name: 'Intent' },
+ ]);
+
+ mockDjClient.DataJunctionAPI.listMetricMetadata.mockReturnValue(
+ mocks.metricMetadata,
+ );
+
+ mockDjClient.DataJunctionAPI.whoami.mockReturnValue({
+ id: 123,
+ username: 'test_user',
+ });
+
+ const element = testElement(mockDjClient);
+ const { getByTestId } = renderCreateMetric(element);
+
+ await userEvent.type(
+ screen.getByLabelText('Display Name *'),
+ 'Some Test Metric',
+ );
+ const selectUpstream = screen.getByTestId('select-upstream-node');
+ fireEvent.keyDown(selectUpstream.firstChild, { key: 'ArrowDown' });
+ fireEvent.click(screen.getByText('default.repair_orders'));
+
+ await waitFor(() => {
+ expect(mockDjClient.DataJunctionAPI.node).toHaveBeenCalledWith(
+ 'default.repair_orders',
+ );
+ });
+
+ const selectReaggregateDimension = getByTestId(
+ 'select-semi-additive-dimension',
+ );
+ fireEvent.keyDown(selectReaggregateDimension.firstChild, {
+ key: 'ArrowDown',
+ });
+ fireEvent.click(await screen.findByText('v3.date.date_id[order]'));
+ fireEvent.change(screen.getByLabelText('Semi-Additive Type'), {
+ target: { value: 'last_value' },
+ });
+
+ await userEvent.type(
+ screen.getByLabelText('Aggregate Expression *'),
+ 'SUM(balance)',
+ );
+ await userEvent.click(screen.getByText('Create metric'));
+
+ await waitFor(
+ () => {
+ expect(mockDjClient.DataJunctionAPI.createNode).toBeCalledWith(
+ 'metric',
+ 'default.some_test_metric',
+ 'Some Test Metric',
+ '',
+ 'SELECT SUM(balance) \n FROM default.repair_orders',
+ 'published',
+ 'default',
+ null,
+ undefined,
+ undefined,
+ undefined,
+ null,
+ {
+ rules: [
+ {
+ dimension: 'v3.date.date_id[order]',
+ fn: 'last_value',
+ },
+ ],
+ },
+ );
+ },
+ { timeout: 10000 },
+ );
+ }, 60000);
+
it('for editing a transform or dimension node', async () => {
const mockDjClient = initializeMockDJClient();
@@ -288,4 +381,159 @@ describe('AddEditNodePage submission succeeded', () => {
).toBeInTheDocument();
});
}, 1000000);
+
+ it('for editing a semi-additive metric node', async () => {
+ const mockDjClient = initializeMockDJClient();
+
+ mockDjClient.DataJunctionAPI.getNodeForEditing.mockReturnValue({
+ ...mocks.mockGetMetricNode,
+ current: {
+ ...mocks.mockGetMetricNode.current,
+ reaggregate: {
+ rules: [
+ {
+ dimension: 'v3.date.date_id[order]',
+ fn: 'LAST_VALUE',
+ },
+ ],
+ },
+ },
+ });
+ mockDjClient.DataJunctionAPI.patchNode = vi.fn();
+ mockDjClient.DataJunctionAPI.patchNode.mockReturnValue({
+ status: 201,
+ json: { name: 'default.num_repair_orders', type: 'metric' },
+ });
+
+ mockDjClient.DataJunctionAPI.tagsNode.mockReturnValue({
+ status: 200,
+ json: { message: 'Success' },
+ });
+
+ mockDjClient.DataJunctionAPI.listTags.mockReturnValue([
+ { name: 'purpose', display_name: 'Purpose' },
+ { name: 'intent', display_name: 'Intent' },
+ ]);
+
+ mockDjClient.DataJunctionAPI.whoami.mockReturnValue({
+ id: 123,
+ username: 'test_user',
+ });
+
+ const element = testElement(mockDjClient);
+ renderEditNode(element);
+
+ await waitFor(() => {
+ expect(screen.getByLabelText('Semi-Additive Type')).toHaveValue(
+ 'last_value',
+ );
+ expect(screen.getByText('v3.date.date_id[order]')).toBeInTheDocument();
+ });
+
+ await userEvent.type(screen.getByLabelText('Description'), '!!!');
+ await userEvent.click(screen.getByText('Save'));
+
+ await waitFor(() => {
+ expect(mockDjClient.DataJunctionAPI.patchNode).toBeCalledWith(
+ 'default.num_repair_orders',
+ 'Default: Num Repair Orders',
+ 'Number of repair orders!!!',
+ 'SELECT count(repair_order_id) \n FROM default.repair_orders',
+ 'published',
+ ['repair_order_id', 'country'],
+ 'neutral',
+ 'unitless',
+ 5,
+ [],
+ ['dj'],
+ { key1: 'value1', key2: 'value2' },
+ {
+ rules: [
+ {
+ dimension: 'v3.date.date_id[order]',
+ fn: 'last_value',
+ },
+ ],
+ },
+ );
+ });
+ }, 1000000);
+
+ it('for clearing semi-additive fields from a metric node', async () => {
+ const mockDjClient = initializeMockDJClient();
+
+ mockDjClient.DataJunctionAPI.getNodeForEditing.mockReturnValue({
+ ...mocks.mockGetMetricNode,
+ current: {
+ ...mocks.mockGetMetricNode.current,
+ reaggregate: {
+ rules: [
+ {
+ dimension: 'v3.date.date_id[order]',
+ fn: 'LAST_VALUE',
+ },
+ ],
+ },
+ },
+ });
+ mockDjClient.DataJunctionAPI.patchNode = vi.fn();
+ mockDjClient.DataJunctionAPI.patchNode.mockReturnValue({
+ status: 201,
+ json: { name: 'default.num_repair_orders', type: 'metric' },
+ });
+
+ mockDjClient.DataJunctionAPI.tagsNode.mockReturnValue({
+ status: 200,
+ json: { message: 'Success' },
+ });
+
+ mockDjClient.DataJunctionAPI.listTags.mockReturnValue([
+ { name: 'purpose', display_name: 'Purpose' },
+ { name: 'intent', display_name: 'Intent' },
+ ]);
+
+ mockDjClient.DataJunctionAPI.whoami.mockReturnValue({
+ id: 123,
+ username: 'test_user',
+ });
+
+ const element = testElement(mockDjClient);
+ renderEditNode(element);
+
+ await waitFor(() => {
+ expect(screen.getByLabelText('Semi-Additive Type')).toHaveValue(
+ 'last_value',
+ );
+ expect(screen.getByText('v3.date.date_id[order]')).toBeInTheDocument();
+ });
+
+ const clearReaggregateDimension = screen
+ .getByTestId('select-semi-additive-dimension')
+ .querySelector('.ReaggregateDimension__clear-indicator');
+ expect(clearReaggregateDimension).toBeInTheDocument();
+ fireEvent.mouseDown(clearReaggregateDimension, { button: 0 });
+ fireEvent.change(screen.getByLabelText('Semi-Additive Type'), {
+ target: { value: '' },
+ });
+
+ await userEvent.click(screen.getByText('Save'));
+
+ await waitFor(() => {
+ expect(mockDjClient.DataJunctionAPI.patchNode).toBeCalledWith(
+ 'default.num_repair_orders',
+ 'Default: Num Repair Orders',
+ 'Number of repair orders',
+ 'SELECT count(repair_order_id) \n FROM default.repair_orders',
+ 'published',
+ ['repair_order_id', 'country'],
+ 'neutral',
+ 'unitless',
+ 5,
+ [],
+ ['dj'],
+ { key1: 'value1', key2: 'value2' },
+ null,
+ );
+ });
+ }, 1000000);
});
diff --git a/datajunction-ui/src/app/pages/AddEditNodePage/index.jsx b/datajunction-ui/src/app/pages/AddEditNodePage/index.jsx
index 2dc0acb22..fcb0f3088 100644
--- a/datajunction-ui/src/app/pages/AddEditNodePage/index.jsx
+++ b/datajunction-ui/src/app/pages/AddEditNodePage/index.jsx
@@ -27,6 +27,7 @@ import { RequiredDimensionsSelect } from './RequiredDimensionsSelect';
import LoadingIcon from '../../icons/LoadingIcon';
import { ColumnsSelect } from './ColumnsSelect';
import { CustomMetadataField } from './CustomMetadataField';
+import { ReaggregateFields } from './ReaggregateFields';
class Action {
static Add = new Action('add');
@@ -71,6 +72,9 @@ export function AddEditNodePage({ extensions = {} }) {
mode: 'published',
owners: [],
custom_metadata: '',
+ reaggregate_dimension: '',
+ reaggregate_function: '',
+ had_reaggregate: false,
};
const validator = values => {
@@ -92,6 +96,14 @@ export function AddEditNodePage({ extensions = {} }) {
if (values.type !== 'metric' && !values.query) {
errors.query = 'Required';
}
+ if (values.type === 'metric') {
+ if (values.reaggregate_dimension && !values.reaggregate_function) {
+ errors.reaggregate_function = 'Required';
+ }
+ if (values.reaggregate_function && !values.reaggregate_dimension) {
+ errors.reaggregate_dimension = 'Required';
+ }
+ }
return errors;
};
@@ -161,8 +173,28 @@ export function AddEditNodePage({ extensions = {} }) {
return `SELECT ${aggregateExpression}`;
};
+ const buildReaggregateSpec = values => {
+ if (values.reaggregate_dimension && values.reaggregate_function) {
+ return {
+ rules: [
+ {
+ dimension: values.reaggregate_dimension,
+ fn: values.reaggregate_function,
+ },
+ ],
+ };
+ }
+ return values.had_reaggregate ? null : undefined;
+ };
+
+ const firstReaggregateRule = reaggregate => reaggregate?.rules?.[0];
+
+ const normalizeReaggregateFunction = reaggregateFunction =>
+ reaggregateFunction ? reaggregateFunction.toLowerCase() : '';
+
const createNode = async (values, setStatus) => {
- const { status, json } = await djClient.createNode(
+ const reaggregate = buildReaggregateSpec(values);
+ const createNodeArgs = [
nodeType,
values.name,
values.display_name,
@@ -180,7 +212,11 @@ export function AddEditNodePage({ extensions = {} }) {
? values.required_dimensions
: undefined,
parseCustomMetadata(values.custom_metadata),
- );
+ ];
+ if (reaggregate !== undefined) {
+ createNodeArgs.push(reaggregate);
+ }
+ const { status, json } = await djClient.createNode(...createNodeArgs);
if (status === 200 || status === 201) {
if (values.tags) {
await djClient.tagsNode(values.name, values.tags);
@@ -201,7 +237,8 @@ export function AddEditNodePage({ extensions = {} }) {
};
const patchNode = async (values, setStatus) => {
- const { status, json } = await djClient.patchNode(
+ const reaggregate = buildReaggregateSpec(values);
+ const patchNodeArgs = [
values.name,
values.display_name,
values.description,
@@ -219,7 +256,11 @@ export function AddEditNodePage({ extensions = {} }) {
: undefined,
values.owners,
parseCustomMetadata(values.custom_metadata),
- );
+ ];
+ if (reaggregate !== undefined) {
+ patchNodeArgs.push(reaggregate);
+ }
+ const { status, json } = await djClient.patchNode(...patchNodeArgs);
const tagsResponse = await djClient.tagsNode(
values.name,
values.tags.map(tag => tag),
@@ -296,6 +337,12 @@ export function AddEditNodePage({ extensions = {} }) {
required_dimensions: node.current.requiredDimensions.map(
dim => dim.name,
),
+ reaggregate_dimension:
+ firstReaggregateRule(node.current.reaggregate)?.dimension || '',
+ reaggregate_function: normalizeReaggregateFunction(
+ firstReaggregateRule(node.current.reaggregate)?.fn,
+ ),
+ had_reaggregate: Boolean(node.current.reaggregate),
upstream_node: '', // Derived metrics have no upstream node
aggregate_expression: derivedExpression,
};
@@ -310,6 +357,12 @@ export function AddEditNodePage({ extensions = {} }) {
required_dimensions: node.current.requiredDimensions.map(
dim => dim.name,
),
+ reaggregate_dimension:
+ firstReaggregateRule(node.current.reaggregate)?.dimension || '',
+ reaggregate_function: normalizeReaggregateFunction(
+ firstReaggregateRule(node.current.reaggregate)?.fn,
+ ),
+ had_reaggregate: Boolean(node.current.reaggregate),
upstream_node: nonMetricParent?.name || '',
aggregate_expression: node.current.metricMetadata?.expression,
};
@@ -360,6 +413,9 @@ export function AddEditNodePage({ extensions = {} }) {
'metric_direction',
'significant_digits',
'required_dimensions',
+ 'reaggregate_dimension',
+ 'reaggregate_function',
+ 'had_reaggregate',
'owners',
'custom_metadata',
];
@@ -495,16 +551,21 @@ export function AddEditNodePage({ extensions = {} }) {
if (action === Action.Edit) {
const data = await getExistingNodeData(name);
runValidityChecks(data, setNode, setMessage);
- updateFieldsWithNodeData(
- data,
- setFieldValue,
- setNode,
- setSelectTags,
- setSelectPrimaryKey,
- setSelectUpstreamNode,
- setSelectRequiredDims,
- setSelectOwners,
- );
+ if (
+ data.message === undefined &&
+ nodeCanBeEdited(data.type)
+ ) {
+ updateFieldsWithNodeData(
+ data,
+ setFieldValue,
+ setNode,
+ setSelectTags,
+ setSelectPrimaryKey,
+ setSelectUpstreamNode,
+ setSelectRequiredDims,
+ setSelectOwners,
+ );
+ }
}
};
fetchData().catch(console.error);
@@ -560,7 +621,10 @@ export function AddEditNodePage({ extensions = {} }) {
/>
)}
{(nodeType === 'metric' || node.type === 'metric') && (
-
+ <>
+
+
+ >
)}
diff --git a/datajunction-ui/src/app/pages/AddEditNodePage/styles.css b/datajunction-ui/src/app/pages/AddEditNodePage/styles.css
index f18307426..1eb0b846c 100644
--- a/datajunction-ui/src/app/pages/AddEditNodePage/styles.css
+++ b/datajunction-ui/src/app/pages/AddEditNodePage/styles.css
@@ -139,6 +139,16 @@
margin: 0 !important;
}
+.node-builder .ReaggregateFields {
+ gap: 12px;
+}
+
+.node-builder .ReaggregateFields > * {
+ flex: 1;
+ min-width: 0;
+ margin: 0 !important;
+}
+
/* Override legacy form-scoped element styling for inputs/selects/buttons.
* Scope to the specific wrapper classes so we DON'T accidentally restyle
* the hidden that lives inside react-select's Control — that input
diff --git a/datajunction-ui/src/app/pages/NodePage/NodeInfoTab.jsx b/datajunction-ui/src/app/pages/NodePage/NodeInfoTab.jsx
index cd18f4fb7..bf5667944 100644
--- a/datajunction-ui/src/app/pages/NodePage/NodeInfoTab.jsx
+++ b/datajunction-ui/src/app/pages/NodePage/NodeInfoTab.jsx
@@ -10,6 +10,23 @@ import { labelize } from '../../../utils/form';
SyntaxHighlighter.registerLanguage('sql', sql);
foundation.hljs['padding'] = '2rem';
+const dimensionNodePath = dimension => {
+ if (!dimension) {
+ return null;
+ }
+ const dimensionWithoutRole = dimension.replace(/\[[^\]]+\]$/, '');
+ const parts = dimensionWithoutRole.split('.');
+ if (parts.length <= 1) {
+ return null;
+ }
+ return parts.slice(0, -1).join('.');
+};
+
+const reaggregateFunctionLabel = func =>
+ func ? labelize(func.toLowerCase()) : null;
+
+const firstReaggregateRule = reaggregate => reaggregate?.rules?.[0];
+
// interface MetricInfo {
// name: string;
// current: MetricRevision;
@@ -59,6 +76,7 @@ export default function NodeInfoTab({ node }) {
expression: metric.current.metricMetadata?.expression,
incompatible_druid_functions:
metric.current.metricMetadata?.incompatibleDruidFunctions || [],
+ reaggregate: metric.current.reaggregate,
});
};
if (node.type === 'metric') {
@@ -267,6 +285,39 @@ export default function NodeInfoTab({ node }) {
{metricInfo?.metric_metadata?.significantDigits || 'None'}
+
+
Semi-Additive
+
+ {firstReaggregateRule(metricInfo?.reaggregate) ? (
+ <>
+ {reaggregateFunctionLabel(
+ firstReaggregateRule(metricInfo.reaggregate).fn,
+ )}
+ {' on '}
+ {dimensionNodePath(
+ firstReaggregateRule(metricInfo.reaggregate).dimension,
+ ) ? (
+
+ {firstReaggregateRule(metricInfo.reaggregate).dimension}
+
+ ) : (
+ firstReaggregateRule(metricInfo.reaggregate).dimension
+ )}
+ >
+ ) : (
+ 'None'
+ )}
+
+
) : (
diff --git a/datajunction-ui/src/app/pages/NodePage/__tests__/NodePage.test.jsx b/datajunction-ui/src/app/pages/NodePage/__tests__/NodePage.test.jsx
index 7fe47c5ff..66e676b95 100644
--- a/datajunction-ui/src/app/pages/NodePage/__tests__/NodePage.test.jsx
+++ b/datajunction-ui/src/app/pages/NodePage/__tests__/NodePage.test.jsx
@@ -388,6 +388,10 @@ describe('', () => {
screen.getByRole('dialog', { name: 'RequiredDimensions' }),
).toHaveTextContent('');
+ expect(
+ screen.getByRole('dialog', { name: 'Reaggregate' }),
+ ).toHaveTextContent('None');
+
expect(
screen.getByRole('dialog', { name: 'DisplayName' }),
).toHaveTextContent('Default: Num Repair Orders');
@@ -414,6 +418,50 @@ describe('', () => {
expect(container.getElementsByClassName('language-sql')).toMatchSnapshot();
}, 60000);
+ it('renders semi-additive metric information with a dimension link', async () => {
+ const djClient = mockDJClient();
+ djClient.DataJunctionAPI.node.mockReturnValue(mocks.mockMetricNode);
+ djClient.DataJunctionAPI.getMetric.mockResolvedValue({
+ ...mocks.mockMetricNodeJson,
+ current: {
+ ...mocks.mockMetricNodeJson.current,
+ reaggregate: {
+ rules: [
+ {
+ dimension: 'v3.date.date_id[order]',
+ fn: 'LAST_VALUE',
+ },
+ ],
+ },
+ },
+ });
+ const element = (
+
+
+
+ );
+ render(
+
+
+
+
+ ,
+ );
+
+ const reaggregate = await screen.findByRole('dialog', {
+ name: 'Reaggregate',
+ });
+ await waitFor(() => {
+ expect(reaggregate).toHaveTextContent(
+ 'Last Value on v3.date.date_id[order]',
+ );
+ });
+
+ expect(
+ screen.getByRole('link', { name: 'v3.date.date_id[order]' }),
+ ).toHaveAttribute('href', '/nodes/v3.date');
+ }, 60000);
+
it('hides Edit and shows the read-only badge for a node in a read-only (flat git) namespace', async () => {
const djClient = mockDJClient();
djClient.DataJunctionAPI.node.mockReturnValue(mocks.mockMetricNode);
diff --git a/datajunction-ui/src/app/services/DJService.js b/datajunction-ui/src/app/services/DJService.js
index 86e0cc1d0..ed3ac4399 100644
--- a/datajunction-ui/src/app/services/DJService.js
+++ b/datajunction-ui/src/app/services/DJService.js
@@ -520,6 +520,14 @@ export const DataJunctionAPI = {
requiredDimensions {
name
}
+ reaggregate {
+ fn
+ weight
+ rules {
+ dimension
+ fn
+ }
+ }
mode
customMetadata
}
@@ -608,6 +616,14 @@ export const DataJunctionAPI = {
requiredDimensions {
name
}
+ reaggregate {
+ fn
+ weight
+ rules {
+ dimension
+ fn
+ }
+ }
}
}
}
@@ -747,6 +763,7 @@ export const DataJunctionAPI = {
metric_unit,
required_dimensions,
custom_metadata,
+ reaggregate,
) {
const metricMetadata =
metric_direction || metric_unit
@@ -767,6 +784,7 @@ export const DataJunctionAPI = {
metric_metadata: metricMetadata,
required_dimensions: required_dimensions,
custom_metadata: custom_metadata,
+ reaggregate: reaggregate,
};
// Remove undefined fields to avoid sending them to the API
Object.keys(requestBody).forEach(
@@ -797,6 +815,7 @@ export const DataJunctionAPI = {
required_dimensions,
owners,
custom_metadata,
+ reaggregate,
) {
try {
const metricMetadata =
@@ -818,6 +837,7 @@ export const DataJunctionAPI = {
required_dimensions: required_dimensions,
owners: owners,
custom_metadata: custom_metadata,
+ reaggregate: reaggregate,
};
// Remove undefined fields to avoid sending them to the API
Object.keys(requestBody).forEach(
diff --git a/datajunction-ui/src/app/services/__tests__/DJService.test.jsx b/datajunction-ui/src/app/services/__tests__/DJService.test.jsx
index 33bf11af1..5d022bbf5 100644
--- a/datajunction-ui/src/app/services/__tests__/DJService.test.jsx
+++ b/datajunction-ui/src/app/services/__tests__/DJService.test.jsx
@@ -182,6 +182,59 @@ describe('DataJunctionAPI', () => {
});
});
+ it('calls createNode with semi-additive metadata correctly', async () => {
+ fetch.mockResponseOnce(JSON.stringify({}));
+ await DataJunctionAPI.createNode(
+ 'metric',
+ 'default.daily_balance',
+ 'Daily Balance',
+ 'Daily balance',
+ 'SELECT sum(balance) FROM default.accounts',
+ 'published',
+ 'default',
+ null,
+ undefined,
+ undefined,
+ undefined,
+ null,
+ {
+ rules: [
+ {
+ dimension: 'v3.date.date_id[order]',
+ fn: 'last_value',
+ },
+ ],
+ },
+ );
+ expect(fetch).toHaveBeenCalledWith(`${DJ_URL}/nodes/metric`, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ name: 'default.daily_balance',
+ display_name: 'Daily Balance',
+ description: 'Daily balance',
+ query: 'SELECT sum(balance) FROM default.accounts',
+ mode: 'published',
+ namespace: 'default',
+ primary_key: null,
+ metric_metadata: null,
+ required_dimensions: undefined,
+ custom_metadata: null,
+ reaggregate: {
+ rules: [
+ {
+ dimension: 'v3.date.date_id[order]',
+ fn: 'last_value',
+ },
+ ],
+ },
+ }),
+ credentials: 'include',
+ });
+ });
+
it('calls patchNode correctly', async () => {
const sampleArgs = [
'name',
@@ -233,6 +286,65 @@ describe('DataJunctionAPI', () => {
});
});
+ it('calls patchNode with semi-additive metadata correctly', async () => {
+ fetch.mockResponseOnce(JSON.stringify({}));
+ await DataJunctionAPI.patchNode(
+ 'default.daily_balance',
+ 'Daily Balance',
+ 'Daily balance',
+ 'SELECT sum(balance) FROM default.accounts',
+ 'published',
+ null,
+ 'neutral',
+ 'unitless',
+ 5,
+ [],
+ ['dj'],
+ null,
+ {
+ rules: [
+ {
+ dimension: 'v3.date.date_id[order]',
+ fn: 'last_value',
+ },
+ ],
+ },
+ );
+ expect(fetch).toHaveBeenCalledWith(
+ `${DJ_URL}/nodes/default.daily_balance`,
+ {
+ method: 'PATCH',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ display_name: 'Daily Balance',
+ description: 'Daily balance',
+ query: 'SELECT sum(balance) FROM default.accounts',
+ mode: 'published',
+ primary_key: null,
+ metric_metadata: {
+ direction: 'neutral',
+ unit: 'unitless',
+ significant_digits: 5,
+ },
+ required_dimensions: [],
+ owners: ['dj'],
+ custom_metadata: null,
+ reaggregate: {
+ rules: [
+ {
+ dimension: 'v3.date.date_id[order]',
+ fn: 'last_value',
+ },
+ ],
+ },
+ }),
+ credentials: 'include',
+ },
+ );
+ });
+
it('calls createCube correctly', async () => {
const sampleArgs = [
'default.node_name',
@@ -1265,6 +1377,7 @@ describe('DataJunctionAPI', () => {
}),
);
await DataJunctionAPI.getMetric('default.num_repair_orders');
+ const requestBody = JSON.parse(fetch.mock.calls[0][1].body);
expect(fetch).toHaveBeenCalledWith(
`${DJ_URL}/graphql`,
expect.objectContaining({
@@ -1275,6 +1388,13 @@ describe('DataJunctionAPI', () => {
},
}),
);
+ expect(requestBody.variables).toEqual({
+ name: 'default.num_repair_orders',
+ });
+ expect(requestBody.query).toContain('reaggregate');
+ expect(requestBody.query).toContain('dimension');
+ expect(requestBody.query).toContain('rules');
+ expect(requestBody.query).toContain('fn');
});
it('calls notebookExportCube correctly', async () => {
@@ -1892,7 +2012,13 @@ describe('DataJunctionAPI', () => {
);
const result = await DataJunctionAPI.getNodeForEditing('default.node1');
+ const requestBody = JSON.parse(fetch.mock.calls[0][1].body);
expect(result).toHaveProperty('name', 'default.node1');
+ expect(requestBody.variables).toEqual({ name: 'default.node1' });
+ expect(requestBody.query).toContain('reaggregate');
+ expect(requestBody.query).toContain('dimension');
+ expect(requestBody.query).toContain('rules');
+ expect(requestBody.query).toContain('fn');
});
it('returns null when getNodeForEditing finds no nodes', async () => {
diff --git a/docker-compose.yml b/docker-compose.yml
index 82020adac..845bf7a2f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -106,7 +106,13 @@ services:
stdin_open: true
volumes:
- ./datajunction-ui:/usr/src/app/
- - ./datajunction-ui/node_modules:/usr/src/app/node_modules
+ # Keep platform-specific packages inside Docker. `nocopy` prevents the
+ # host's macOS node_modules from seeding this Linux volume.
+ - type: volume
+ source: djui_node_modules
+ target: /usr/src/app/node_modules
+ volume:
+ nocopy: true
environment:
- NODE_OPTIONS=--max-old-space-size=4096
- WATCHPACK_POLLING=true
@@ -327,3 +333,4 @@ services:
volumes:
postgres_metadata:
+ djui_node_modules: