diff --git a/.gitignore b/.gitignore index dcfc4b40f..c8450e952 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,6 @@ Untitled* postgres_metadata postgres_superset node_modules + +# Local Claude Code workspace state (large; never committed) +.claude/ diff --git a/datajunction-query/uv.lock b/datajunction-query/uv.lock index 085cb0ecd..a166f9f24 100644 --- a/datajunction-query/uv.lock +++ b/datajunction-query/uv.lock @@ -1,10 +1,11 @@ version = 1 revision = 3 -requires-python = ">=3.10, <4.0" +requires-python = ">=3.11, <4.0" resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version < '3.11'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version < '3.12'", ] [[package]] @@ -23,7 +24,6 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mako" }, { name = "sqlalchemy" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/cc/aca263693b2ece99fa99a09b6d092acb89973eb2bb575faef1777e04f8b4/alembic-1.18.1.tar.gz", hash = "sha256:83ac6b81359596816fb3b893099841a0862f2117b2963258e965d70dc62fb866", size = 2044319, upload-time = "2026-01-14T18:53:14.907Z" } @@ -54,7 +54,6 @@ name = "anyio" version = "4.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "idna" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] @@ -76,23 +75,11 @@ wheels = [ name = "astroid" version = "4.0.3" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] sdist = { url = "https://files.pythonhosted.org/packages/a1/ca/c17d0f83016532a1ad87d1de96837164c99d47a3b6bbba28bd597c25b37a/astroid-4.0.3.tar.gz", hash = "sha256:08d1de40d251cc3dc4a7a12726721d475ac189e4e583d596ece7422bc176bda3", size = 406224, upload-time = "2026-01-03T22:14:26.096Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ce/66/686ac4fc6ef48f5bacde625adac698f41d5316a9753c2b20bb0931c9d4e2/astroid-4.0.3-py3-none-any.whl", hash = "sha256:864a0a34af1bd70e1049ba1e61cee843a7252c826d97825fcee9b2fcbd9e1b14", size = 276443, upload-time = "2026-01-03T22:14:24.412Z" }, ] -[[package]] -name = "backports-asyncio-runner" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/ff/70dca7d7cb1cbc0edb2c6cc0c38b65cba36cccc491eca64cabd5fe7f8670/backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162", size = 69893, upload-time = "2025-07-02T02:27:15.685Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, -] - [[package]] name = "boto3" version = "1.42.34" @@ -127,10 +114,8 @@ version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "os_name == 'nt'" }, - { name = "importlib-metadata", marker = "python_full_version < '3.10.2'" }, { name = "packaging" }, { name = "pyproject-hooks" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/18/94eaffda7b329535d91f00fe605ab1f1e5cd68b2074d03f255c7d250687d/build-1.4.0.tar.gz", hash = "sha256:f1b91b925aa322be454f8330c6fb48b465da993d1e7e7e6fa35027ec49f3c936", size = 50054, upload-time = "2026-01-08T16:41:47.696Z" } wheels = [ @@ -164,18 +149,6 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/d7/516d984057745a6cd96575eea814fe1edd6646ee6efd552fb7b0921dec83/cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", size = 184283, upload-time = "2025-09-08T23:22:08.01Z" }, - { url = "https://files.pythonhosted.org/packages/9e/84/ad6a0b408daa859246f57c03efd28e5dd1b33c21737c2db84cae8c237aa5/cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", size = 180504, upload-time = "2025-09-08T23:22:10.637Z" }, - { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" }, - { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" }, - { url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" }, - { url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" }, - { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" }, - { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" }, - { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" }, - { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" }, - { url = "https://files.pythonhosted.org/packages/e2/cc/027d7fb82e58c48ea717149b03bcadcbdc293553edb283af792bd4bcbb3f/cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", size = 172184, upload-time = "2025-09-08T23:22:23.328Z" }, - { url = "https://files.pythonhosted.org/packages/33/fa/072dd15ae27fbb4e06b437eb6e944e75b068deb09e2a2826039e49ee2045/cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", size = 182790, upload-time = "2025-09-08T23:22:24.752Z" }, { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, @@ -252,22 +225,6 @@ version = "3.4.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/b8/6d51fc1d52cbd52cd4ccedd5b5b2f0f6a11bbf6765c782298b0f3e808541/charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d", size = 209709, upload-time = "2025-10-14T04:40:11.385Z" }, - { url = "https://files.pythonhosted.org/packages/5c/af/1f9d7f7faafe2ddfb6f72a2e07a548a629c61ad510fe60f9630309908fef/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8", size = 148814, upload-time = "2025-10-14T04:40:13.135Z" }, - { url = "https://files.pythonhosted.org/packages/79/3d/f2e3ac2bbc056ca0c204298ea4e3d9db9b4afe437812638759db2c976b5f/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad", size = 144467, upload-time = "2025-10-14T04:40:14.728Z" }, - { url = "https://files.pythonhosted.org/packages/ec/85/1bf997003815e60d57de7bd972c57dc6950446a3e4ccac43bc3070721856/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8", size = 162280, upload-time = "2025-10-14T04:40:16.14Z" }, - { url = "https://files.pythonhosted.org/packages/3e/8e/6aa1952f56b192f54921c436b87f2aaf7c7a7c3d0d1a765547d64fd83c13/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d", size = 159454, upload-time = "2025-10-14T04:40:17.567Z" }, - { url = "https://files.pythonhosted.org/packages/36/3b/60cbd1f8e93aa25d1c669c649b7a655b0b5fb4c571858910ea9332678558/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313", size = 153609, upload-time = "2025-10-14T04:40:19.08Z" }, - { url = "https://files.pythonhosted.org/packages/64/91/6a13396948b8fd3c4b4fd5bc74d045f5637d78c9675585e8e9fbe5636554/charset_normalizer-3.4.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e", size = 151849, upload-time = "2025-10-14T04:40:20.607Z" }, - { url = "https://files.pythonhosted.org/packages/b7/7a/59482e28b9981d105691e968c544cc0df3b7d6133152fb3dcdc8f135da7a/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93", size = 151586, upload-time = "2025-10-14T04:40:21.719Z" }, - { url = "https://files.pythonhosted.org/packages/92/59/f64ef6a1c4bdd2baf892b04cd78792ed8684fbc48d4c2afe467d96b4df57/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0", size = 145290, upload-time = "2025-10-14T04:40:23.069Z" }, - { url = "https://files.pythonhosted.org/packages/6b/63/3bf9f279ddfa641ffa1962b0db6a57a9c294361cc2f5fcac997049a00e9c/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84", size = 163663, upload-time = "2025-10-14T04:40:24.17Z" }, - { url = "https://files.pythonhosted.org/packages/ed/09/c9e38fc8fa9e0849b172b581fd9803bdf6e694041127933934184e19f8c3/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e", size = 151964, upload-time = "2025-10-14T04:40:25.368Z" }, - { url = "https://files.pythonhosted.org/packages/d2/d1/d28b747e512d0da79d8b6a1ac18b7ab2ecfd81b2944c4c710e166d8dd09c/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db", size = 161064, upload-time = "2025-10-14T04:40:26.806Z" }, - { url = "https://files.pythonhosted.org/packages/bb/9a/31d62b611d901c3b9e5500c36aab0ff5eb442043fb3a1c254200d3d397d9/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6", size = 155015, upload-time = "2025-10-14T04:40:28.284Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f3/107e008fa2bff0c8b9319584174418e5e5285fef32f79d8ee6a430d0039c/charset_normalizer-3.4.4-cp310-cp310-win32.whl", hash = "sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f", size = 99792, upload-time = "2025-10-14T04:40:29.613Z" }, - { url = "https://files.pythonhosted.org/packages/eb/66/e396e8a408843337d7315bab30dbf106c38966f1819f123257f5520f8a96/charset_normalizer-3.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d", size = 107198, upload-time = "2025-10-14T04:40:30.644Z" }, - { url = "https://files.pythonhosted.org/packages/b5/58/01b4f815bf0312704c267f2ccb6e5d42bcc7752340cd487bc9f8c3710597/charset_normalizer-3.4.4-cp310-cp310-win_arm64.whl", hash = "sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69", size = 100262, upload-time = "2025-10-14T04:40:32.108Z" }, { url = "https://files.pythonhosted.org/packages/ed/27/c6491ff4954e58a10f69ad90aca8a1b6fe9c5d3c6f380907af3c37435b59/charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8", size = 206988, upload-time = "2025-10-14T04:40:33.79Z" }, { url = "https://files.pythonhosted.org/packages/94/59/2e87300fe67ab820b5428580a53cad894272dbb97f38a7a814a2a1ac1011/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0", size = 147324, upload-time = "2025-10-14T04:40:34.961Z" }, { url = "https://files.pythonhosted.org/packages/07/fb/0cf61dc84b2b088391830f6274cb57c82e4da8bbc2efeac8c025edb88772/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3", size = 142742, upload-time = "2025-10-14T04:40:36.105Z" }, @@ -371,18 +328,6 @@ version = "7.13.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/23/f9/e92df5e07f3fc8d4c7f9a0f146ef75446bf870351cd37b788cf5897f8079/coverage-7.13.1.tar.gz", hash = "sha256:b7593fe7eb5feaa3fbb461ac79aac9f9fc0387a5ca8080b0c6fe2ca27b091afd", size = 825862, upload-time = "2025-12-28T15:42:56.969Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/9a/3742e58fd04b233df95c012ee9f3dfe04708a5e1d32613bd2d47d4e1be0d/coverage-7.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e1fa280b3ad78eea5be86f94f461c04943d942697e0dac889fa18fff8f5f9147", size = 218633, upload-time = "2025-12-28T15:40:10.165Z" }, - { url = "https://files.pythonhosted.org/packages/7e/45/7e6bdc94d89cd7c8017ce735cf50478ddfe765d4fbf0c24d71d30ea33d7a/coverage-7.13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c3d8c679607220979434f494b139dfb00131ebf70bb406553d69c1ff01a5c33d", size = 219147, upload-time = "2025-12-28T15:40:12.069Z" }, - { url = "https://files.pythonhosted.org/packages/f7/38/0d6a258625fd7f10773fe94097dc16937a5f0e3e0cdf3adef67d3ac6baef/coverage-7.13.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:339dc63b3eba969067b00f41f15ad161bf2946613156fb131266d8debc8e44d0", size = 245894, upload-time = "2025-12-28T15:40:13.556Z" }, - { url = "https://files.pythonhosted.org/packages/27/58/409d15ea487986994cbd4d06376e9860e9b157cfbfd402b1236770ab8dd2/coverage-7.13.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db622b999ffe49cb891f2fff3b340cdc2f9797d01a0a202a0973ba2562501d90", size = 247721, upload-time = "2025-12-28T15:40:15.37Z" }, - { url = "https://files.pythonhosted.org/packages/da/bf/6e8056a83fd7a96c93341f1ffe10df636dd89f26d5e7b9ca511ce3bcf0df/coverage-7.13.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1443ba9acbb593fa7c1c29e011d7c9761545fe35e7652e85ce7f51a16f7e08d", size = 249585, upload-time = "2025-12-28T15:40:17.226Z" }, - { url = "https://files.pythonhosted.org/packages/f4/15/e1daff723f9f5959acb63cbe35b11203a9df77ee4b95b45fffd38b318390/coverage-7.13.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c832ec92c4499ac463186af72f9ed4d8daec15499b16f0a879b0d1c8e5cf4a3b", size = 246597, upload-time = "2025-12-28T15:40:19.028Z" }, - { url = "https://files.pythonhosted.org/packages/74/a6/1efd31c5433743a6ddbc9d37ac30c196bb07c7eab3d74fbb99b924c93174/coverage-7.13.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:562ec27dfa3f311e0db1ba243ec6e5f6ab96b1edfcfc6cf86f28038bc4961ce6", size = 247626, upload-time = "2025-12-28T15:40:20.846Z" }, - { url = "https://files.pythonhosted.org/packages/6d/9f/1609267dd3e749f57fdd66ca6752567d1c13b58a20a809dc409b263d0b5f/coverage-7.13.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4de84e71173d4dada2897e5a0e1b7877e5eefbfe0d6a44edee6ce31d9b8ec09e", size = 245629, upload-time = "2025-12-28T15:40:22.397Z" }, - { url = "https://files.pythonhosted.org/packages/e2/f6/6815a220d5ec2466383d7cc36131b9fa6ecbe95c50ec52a631ba733f306a/coverage-7.13.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:a5a68357f686f8c4d527a2dc04f52e669c2fc1cbde38f6f7eb6a0e58cbd17cae", size = 245901, upload-time = "2025-12-28T15:40:23.836Z" }, - { url = "https://files.pythonhosted.org/packages/ac/58/40576554cd12e0872faf6d2c0eb3bc85f71d78427946ddd19ad65201e2c0/coverage-7.13.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:77cc258aeb29a3417062758975521eae60af6f79e930d6993555eeac6a8eac29", size = 246505, upload-time = "2025-12-28T15:40:25.421Z" }, - { url = "https://files.pythonhosted.org/packages/3b/77/9233a90253fba576b0eee81707b5781d0e21d97478e5377b226c5b096c0f/coverage-7.13.1-cp310-cp310-win32.whl", hash = "sha256:bb4f8c3c9a9f34423dba193f241f617b08ffc63e27f67159f60ae6baf2dcfe0f", size = 221257, upload-time = "2025-12-28T15:40:27.217Z" }, - { url = "https://files.pythonhosted.org/packages/e0/43/e842ff30c1a0a623ec80db89befb84a3a7aad7bfe44a6ea77d5a3e61fedd/coverage-7.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:c8e2706ceb622bc63bac98ebb10ef5da80ed70fbd8a7999a5076de3afaef0fb1", size = 222191, upload-time = "2025-12-28T15:40:28.916Z" }, { url = "https://files.pythonhosted.org/packages/b4/9b/77baf488516e9ced25fc215a6f75d803493fc3f6a1a1227ac35697910c2a/coverage-7.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a55d509a1dc5a5b708b5dad3b5334e07a16ad4c2185e27b40e4dba796ab7f88", size = 218755, upload-time = "2025-12-28T15:40:30.812Z" }, { url = "https://files.pythonhosted.org/packages/d7/cd/7ab01154e6eb79ee2fab76bf4d89e94c6648116557307ee4ebbb85e5c1bf/coverage-7.13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4d010d080c4888371033baab27e47c9df7d6fb28d0b7b7adf85a4a49be9298b3", size = 219257, upload-time = "2025-12-28T15:40:32.333Z" }, { url = "https://files.pythonhosted.org/packages/01/d5/b11ef7863ffbbdb509da0023fad1e9eda1c0eaea61a6d2ea5b17d4ac706e/coverage-7.13.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d938b4a840fb1523b9dfbbb454f652967f18e197569c32266d4d13f37244c3d9", size = 249657, upload-time = "2025-12-28T15:40:34.1Z" }, @@ -475,7 +420,6 @@ version = "46.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9f/33/c00162f49c0e2fe8064a62cb92b93e50c74a72bc370ab92f86112b33ff62/cryptography-46.0.3.tar.gz", hash = "sha256:a8b17438104fed022ce745b362294d9ce35b4c2e45c1d958ad4a4b019285f4a1", size = 749258, upload-time = "2025-10-15T23:18:31.74Z" } wheels = [ @@ -524,8 +468,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/6e/1c8331ddf91ca4730ab3086a0f1be19c65510a33b5a441cb334e7a2d2560/cryptography-46.0.3-cp38-abi3-win32.whl", hash = "sha256:6276eb85ef938dc035d59b87c8a7dc559a232f954962520137529d77b18ff1df", size = 3036695, upload-time = "2025-10-15T23:18:08.672Z" }, { url = "https://files.pythonhosted.org/packages/90/45/b0d691df20633eff80955a0fc7695ff9051ffce8b69741444bd9ed7bd0db/cryptography-46.0.3-cp38-abi3-win_amd64.whl", hash = "sha256:416260257577718c05135c55958b674000baef9a1c7d9e8f306ec60d71db850f", size = 3501720, upload-time = "2025-10-15T23:18:10.632Z" }, { url = "https://files.pythonhosted.org/packages/e8/cb/2da4cc83f5edb9c3257d09e1e7ab7b23f049c7962cae8d842bbef0a9cec9/cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372", size = 2918740, upload-time = "2025-10-15T23:18:12.277Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cd/1a8633802d766a0fa46f382a77e096d7e209e0817892929655fe0586ae32/cryptography-46.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a23582810fedb8c0bc47524558fb6c56aac3fc252cb306072fd2815da2a47c32", size = 3689163, upload-time = "2025-10-15T23:18:13.821Z" }, - { url = "https://files.pythonhosted.org/packages/4c/59/6b26512964ace6480c3e54681a9859c974172fb141c38df11eadd8416947/cryptography-46.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e7aec276d68421f9574040c26e2a7c3771060bc0cff408bae1dcb19d3ab1e63c", size = 3429474, upload-time = "2025-10-15T23:18:15.477Z" }, { url = "https://files.pythonhosted.org/packages/06/8a/e60e46adab4362a682cf142c7dcb5bf79b782ab2199b0dcb81f55970807f/cryptography-46.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7ce938a99998ed3c8aa7e7272dca1a610401ede816d36d0693907d863b10d9ea", size = 3698132, upload-time = "2025-10-15T23:18:17.056Z" }, { url = "https://files.pythonhosted.org/packages/da/38/f59940ec4ee91e93d3311f7532671a5cef5570eb04a144bf203b58552d11/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:191bb60a7be5e6f54e30ba16fdfae78ad3a342a0599eb4193ba88e3f3d6e185b", size = 4243992, upload-time = "2025-10-15T23:18:18.695Z" }, { url = "https://files.pythonhosted.org/packages/b0/0c/35b3d92ddebfdfda76bb485738306545817253d0a3ded0bfe80ef8e67aa5/cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c70cc23f12726be8f8bc72e41d5065d77e4515efae3690326764ea1b07845cfb", size = 4409944, upload-time = "2025-10-15T23:18:20.597Z" }, @@ -544,6 +486,7 @@ dependencies = [ { name = "duckdb" }, { name = "duckdb-engine" }, { name = "fastapi" }, + { name = "google-cloud-bigquery" }, { name = "importlib-metadata" }, { name = "msgpack" }, { name = "psycopg", extra = ["pool"] }, @@ -592,12 +535,13 @@ requires-dist = [ { name = "accept-types", specifier = "==0.4.1" }, { name = "alembic", specifier = ">=1.7.7" }, { name = "cachelib", specifier = ">=0.4.0" }, - { name = "duckdb", specifier = "==0.8.1" }, + { name = "duckdb", specifier = ">=1.0.0" }, { name = "duckdb-engine" }, { name = "fastapi", specifier = ">=0.79.0" }, + { name = "google-cloud-bigquery", specifier = ">=3.11.0" }, { name = "importlib-metadata" }, { name = "msgpack", specifier = ">=1.0.3" }, - { name = "psycopg", extras = ["async", "pool"], specifier = ">=3.2.1" }, + { name = "psycopg", extras = ["pool"], specifier = ">=3.2.1" }, { name = "pytest-asyncio", specifier = ">=0.24.0" }, { name = "pytest-integration", specifier = ">=0.2.3" }, { name = "python-dotenv", specifier = ">=0.21.0" }, @@ -621,7 +565,7 @@ test = [ { name = "httpx", specifier = ">=0.24.1" }, { name = "pip-tools", specifier = ">=6.4.0" }, { name = "pre-commit", specifier = ">=3.2.2" }, - { name = "psycopg", extras = ["async", "pool"], specifier = ">=3.2.1" }, + { name = "psycopg", extras = ["pool"], specifier = ">=3.2.1" }, { name = "pydruid", specifier = ">=0.6.4" }, { name = "pyfakefs", specifier = ">=4.5.1" }, { name = "pylint", specifier = ">=2.15.3" }, @@ -669,30 +613,38 @@ wheels = [ [[package]] name = "duckdb" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/69/49e495d5e75d60410144efa50134713b1695a588e36dd8af72a78eef9f03/duckdb-0.8.1.tar.gz", hash = "sha256:a54d37f4abc2afc4f92314aaa56ecf215a411f40af4bffe1e86bd25e62aceee9", size = 10421102, upload-time = "2023-06-13T12:29:04.295Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/c1/3d4cef11f02af5c25d1d4d6dc8f15967ac3e4a2a806adce84cb695440b37/duckdb-0.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:14781d21580ee72aba1f5dcae7734674c9b6c078dd60470a08b2b420d15b996d", size = 27317784, upload-time = "2023-06-13T10:35:42.348Z" }, - { url = "https://files.pythonhosted.org/packages/65/05/cd7088defc381df9ac63fa379f4f7ce7f7bc100679eabbdd6013c8dc3d9a/duckdb-0.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f13bf7ab0e56ddd2014ef762ae4ee5ea4df5a69545ce1191b8d7df8118ba3167", size = 14664958, upload-time = "2023-06-13T10:35:47.034Z" }, - { url = "https://files.pythonhosted.org/packages/ec/9e/c8724532dc5b57c1336b5b45250c6a101a4d338560d16336c9f9fcc1764c/duckdb-0.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4032042d8363e55365bbca3faafc6dc336ed2aad088f10ae1a534ebc5bcc181", size = 12632512, upload-time = "2023-06-13T10:35:51.942Z" }, - { url = "https://files.pythonhosted.org/packages/c0/77/6ca11ed3e339d7fe8acd38ad75eca886ce2403f9fddf9f4f503d42d7093e/duckdb-0.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a71bd8f0b0ca77c27fa89b99349ef22599ffefe1e7684ae2e1aa2904a08684", size = 14798677, upload-time = "2023-06-13T15:46:23.659Z" }, - { url = "https://files.pythonhosted.org/packages/df/c6/e30f5a865d23f94d903f9731bb1fc9f113e329f0862564fd5363cd55d039/duckdb-0.8.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24568d6e48f3dbbf4a933109e323507a46b9399ed24c5d4388c4987ddc694fd0", size = 17713728, upload-time = "2023-06-13T13:11:28.127Z" }, - { url = "https://files.pythonhosted.org/packages/0c/a3/4e52ef89606292b26864bcc3be3e36e1345ba4bb8a6df5b2fa36dfc01fd7/duckdb-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:297226c0dadaa07f7c5ae7cbdb9adba9567db7b16693dbd1b406b739ce0d7924", size = 15880589, upload-time = "2023-06-13T12:42:16.21Z" }, - { url = "https://files.pythonhosted.org/packages/80/4d/140619aa73d3de8ae5ea903dba34257ce129c01e77f0db1f18ffaf9dfe71/duckdb-0.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5792cf777ece2c0591194006b4d3e531f720186102492872cb32ddb9363919cf", size = 18267381, upload-time = "2023-06-13T13:11:31.764Z" }, - { url = "https://files.pythonhosted.org/packages/7a/73/edfe5e3f845eda8cf3db91af7550e9dbfc9e57ce0b2990e4c693fcfeb364/duckdb-0.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:12803f9f41582b68921d6b21f95ba7a51e1d8f36832b7d8006186f58c3d1b344", size = 16476321, upload-time = "2023-06-13T12:42:20.412Z" }, - { url = "https://files.pythonhosted.org/packages/9e/13/ab717b2924313c51e631e1fde8fb2cd5d3daa87a051fbd046d05bc7a22da/duckdb-0.8.1-cp310-cp310-win32.whl", hash = "sha256:d0953d5a2355ddc49095e7aef1392b7f59c5be5cec8cdc98b9d9dc1f01e7ce2b", size = 8754102, upload-time = "2023-06-13T16:38:56.889Z" }, - { url = "https://files.pythonhosted.org/packages/6d/14/12d9c847a39c1d15dd3bf853d5e1d9f333dd3f97e1876750808d2e3bfe6e/duckdb-0.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:6e6583c98a7d6637e83bcadfbd86e1f183917ea539f23b6b41178f32f813a5eb", size = 9815141, upload-time = "2023-06-13T16:39:00.067Z" }, - { url = "https://files.pythonhosted.org/packages/83/35/acc4056703190e3203bfa7cc06bb000afaf24e8b0e63614901bebf20ae2b/duckdb-0.8.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fad7ed0d4415f633d955ac24717fa13a500012b600751d4edb050b75fb940c25", size = 27317508, upload-time = "2023-06-13T10:37:54.974Z" }, - { url = "https://files.pythonhosted.org/packages/75/da/81565f1cd3210442fc130389da0627e2e6b72db2dd173ab03cd277f64916/duckdb-0.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:81ae602f34d38d9c48dd60f94b89f28df3ef346830978441b83c5b4eae131d08", size = 14664994, upload-time = "2023-06-13T10:37:58.539Z" }, - { url = "https://files.pythonhosted.org/packages/af/af/7fc3aca20d65d9851cdf7e25b5c0ca41954c0155c4f4d907c1cfd3382411/duckdb-0.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d75cfe563aaa058d3b4ccaaa371c6271e00e3070df5de72361fd161b2fe6780", size = 12632544, upload-time = "2023-06-13T10:38:01.202Z" }, - { url = "https://files.pythonhosted.org/packages/b2/80/207a50e4dcfa2c7e04b4bb7b4ccd2dc27a19e7ef9f74677f1de5be4e85d7/duckdb-0.8.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8dbb55e7a3336f2462e5e916fc128c47fe1c03b6208d6bd413ac11ed95132aa0", size = 14799218, upload-time = "2023-06-13T14:14:23.605Z" }, - { url = "https://files.pythonhosted.org/packages/d4/5f/bafa326bb094b4b9400810a3654a21ada3ec78648ac7028d24c62b09ff8e/duckdb-0.8.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6df53efd63b6fdf04657385a791a4e3c4fb94bfd5db181c4843e2c46b04fef5", size = 17712863, upload-time = "2023-06-13T13:17:04.758Z" }, - { url = "https://files.pythonhosted.org/packages/79/22/f2b229ec92d559a30bba996e85feba966fe287c6fe837b9b388ddf921c1e/duckdb-0.8.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b188b80b70d1159b17c9baaf541c1799c1ce8b2af4add179a9eed8e2616be96", size = 15880481, upload-time = "2023-06-13T12:29:21.683Z" }, - { url = "https://files.pythonhosted.org/packages/2e/66/f66d99eea100eaa656862f396eead4fb01b0d487aef4c7acfac993393c47/duckdb-0.8.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5ad481ee353f31250b45d64b4a104e53b21415577943aa8f84d0af266dc9af85", size = 18266623, upload-time = "2023-06-13T13:17:10.66Z" }, - { url = "https://files.pythonhosted.org/packages/2f/5c/935f12ce8fb1f1bf546ef939254213d361d1faf55f1f310c9d34fbdc0d4a/duckdb-0.8.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d1d1b1729993611b1892509d21c21628917625cdbe824a61ce891baadf684b32", size = 16477164, upload-time = "2023-06-13T12:29:24.937Z" }, - { url = "https://files.pythonhosted.org/packages/76/5f/36ff597cacb235884d3c19cb182facb86a1502e08a24a114e21f1fc77551/duckdb-0.8.1-cp311-cp311-win32.whl", hash = "sha256:2d8f9cc301e8455a4f89aa1088b8a2d628f0c1f158d4cf9bc78971ed88d82eea", size = 8753987, upload-time = "2023-06-13T16:52:25.976Z" }, - { url = "https://files.pythonhosted.org/packages/ee/af/e3fdfa4776124e797d8303f85f3bd97e17f7a1aa1a90189b545369d267d1/duckdb-0.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:07457a43605223f62d93d2a5a66b3f97731f79bbbe81fdd5b79954306122f612", size = 9816167, upload-time = "2023-06-13T16:52:29.185Z" }, +version = "1.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/c2/b62ec24d57bb8df4e24b0b58f7f8facb32f5fdb9f1895aed9e9fcdded168/duckdb-1.5.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b543841b0ae18a9c982345cfa3987e9c065d3a4b0f067daa473d92d1e65f528", size = 32708371, upload-time = "2026-07-22T10:53:41.642Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ce/769171ba45f0b73632dc3bc3108d891e81dd6c6bbfba630a34a75b4dcc0f/duckdb-1.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a925d06c2a4c3b64553d6cc1aced5028d376d4479bed689a7d47e9b1dccd80a", size = 17343979, upload-time = "2026-07-22T10:53:44.951Z" }, + { url = "https://files.pythonhosted.org/packages/46/59/a8e3384ee916e00d5dcf985194c1511d61978540778a1e96fa47f9fb3e0d/duckdb-1.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0c42757cb34722144bd4dfb94b6f336339e7b2468f6813fa7fa9a319ba07bab4", size = 15493704, upload-time = "2026-07-22T10:53:47.912Z" }, + { url = "https://files.pythonhosted.org/packages/6f/1d/9840179c2607b90523a2884a129c4d4e6dbdc1178ba62a976c1043beba88/duckdb-1.5.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e72f9e1a4f90a5c8483ad4d540e495bf0834ba61c360b52499a573d7ed62a3f", size = 19366574, upload-time = "2026-07-22T10:53:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/b5/55/f9641a4eebcc2f4df631287d6c3b9ed2eea3b92644f93acbad825e3972b6/duckdb-1.5.5-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b6f86ed85d4ef5e0211eaebf75d057bd8bb520bba438a95dd0f4e42234bbfe", size = 21477952, upload-time = "2026-07-22T10:53:55.575Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3a/07c3556e37a5c97b95917b029c8fdde4a25fbd76a660bacdac195cf20dcb/duckdb-1.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:9f4287f97ccf0c1f3d471e7115be2b067cbf99627e2d34bffd462dd64703cddc", size = 13156986, upload-time = "2026-07-22T10:53:58.823Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ff/07b48eef2078ca033847e9caa46cc7633b714c5f91ad1ce091c8ca89d792/duckdb-1.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:179633a3fc6296c75d57c69c1e239fa9e5cdcb670fd1dbff88a02663f932905c", size = 14001317, upload-time = "2026-07-22T10:54:01.724Z" }, + { url = "https://files.pythonhosted.org/packages/d6/40/2e05d324400fdaa5656c9f48d6298da421cb034d85e509fa0e6e325cf04b/duckdb-1.5.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d4dd65f8941a604b947e0b9b4b4f7165988e29a23ec0b69b4038520956d9933e", size = 32753858, upload-time = "2026-07-22T10:54:05.514Z" }, + { url = "https://files.pythonhosted.org/packages/79/15/5ceb58ffb5bb8a62b3fd7abb39c41467cdf94850ece02e6d88664dfc75ce/duckdb-1.5.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33db46679b071f108d57139493dee2d37e1f5efcf5c5c039c2969eed11a6c8a7", size = 17368293, upload-time = "2026-07-22T10:54:09.139Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5c/bf02da0b354fe83cca4f95a4fbf762181af466f7d551ab2a093f7698882a/duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae", size = 15509131, upload-time = "2026-07-22T10:54:12.228Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a9/5f1f09da421d8e930e0b063d11c1b3f90363f40ede74438cd188afdd13a2/duckdb-1.5.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f316eae2323d9a851883fdf2dee91c1f9efe251ab33e14a2272f82a913422ed6", size = 19391959, upload-time = "2026-07-22T10:54:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/4f/98/6549769f158126fa64fd6c1ac2eb59a18282146c939867a3eb31b7c1db07/duckdb-1.5.5-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a6d2d11859d82a936ebdcb30ce3d8a1cbb3e990bff05c12abb9b54c44fa7bd1", size = 21510909, upload-time = "2026-07-22T10:54:19.681Z" }, + { url = "https://files.pythonhosted.org/packages/af/b7/5753b41d3124838f868f9f523362812d9fc45409e9e4dd70dcbb0a25826e/duckdb-1.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:ddfbdb096c11d51ee22492397d342c90a82e62c5d09961477895934d0a25372f", size = 13168544, upload-time = "2026-07-22T10:54:22.789Z" }, + { url = "https://files.pythonhosted.org/packages/5c/28/44b679c7d46245f8398feae7edac959d1b83d4eb143e25b3fce0630b78bd/duckdb-1.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:2725d2b9ace3a4e75d72fc5a239f6a44b502c580edadb8fb2676db772c5f9282", size = 13988684, upload-time = "2026-07-22T10:54:26.003Z" }, + { url = "https://files.pythonhosted.org/packages/47/37/4a38116e7700720fd152c666292214fd3abdf916496991296d8d1f66efbf/duckdb-1.5.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd98829b67788609017e65c761bd42a5dd0f9129441bed8bda4d6881ccf819f0", size = 32754294, upload-time = "2026-07-22T10:54:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/66/42/7d392f1ba1eee0eaf4ab4c8c7a604bfe3536cd63f979cf5c98798664f807/duckdb-1.5.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feead93c56679b79592d437c62975d39cb67adedffa7592c763baf8160ac7366", size = 17368211, upload-time = "2026-07-22T10:54:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a5/0a6f4fa60562faa615e55e15bd1953a2f2b17a8edd8105e5cda215e43457/duckdb-1.5.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49c963d9469373d7aba8d750d9ea565ab823e94166efed953f184dd9b169b98c", size = 15509136, upload-time = "2026-07-22T10:54:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/e4/cb/023c89f51978545b9fab318581bba0c457a58e7530d2d933e54ae7d8647c/duckdb-1.5.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a736217825461732b5442d05a220f3da2e23a0dae114efbf08c9bf171b53098a", size = 19392147, upload-time = "2026-07-22T10:54:39.551Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c5/41bef391fb8b23dbc133c9f2ba016e7a7a8124513d2cc1b430f1897d87e4/duckdb-1.5.5-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:078e6a60dd8eedde5832f45422ca5c4a6b8c837aeabd8a56ca0b7d933f588053", size = 21511060, upload-time = "2026-07-22T10:54:42.788Z" }, + { url = "https://files.pythonhosted.org/packages/07/9f/c44dfc1f924ac29b3252dc1b91393c01d009dbfe9f8ed33f10b986151bd1/duckdb-1.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:6826504277dba513c0c5d71d828456c94d729c9d2482f94b2e289f90a9167e28", size = 13168028, upload-time = "2026-07-22T10:54:46.127Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/591384b2cd59abddd6f5dc175e60374f9abae6064429f0c4402854c10f44/duckdb-1.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:baa9c5702002fabb559ded2a39008f9f421fcbc7237d388b8213eff1e08858de", size = 13989955, upload-time = "2026-07-22T10:54:49.262Z" }, + { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" }, + { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" }, + { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" }, + { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" }, + { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, ] [[package]] @@ -709,18 +661,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/a2/e90242f53f7ae41554419b1695b4820b364df87c8350aa420b60b20cab92/duckdb_engine-0.17.0-py3-none-any.whl", hash = "sha256:3aa72085e536b43faab635f487baf77ddc5750069c16a2f8d9c6c3cb6083e979", size = 49676, upload-time = "2025-03-29T09:49:15.564Z" }, ] -[[package]] -name = "exceptiongroup" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, -] - [[package]] name = "fastapi" version = "0.128.0" @@ -757,19 +697,137 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/2e/b41d8a1a917d6581fc27a35d05561037b048e47df50f27f8ac9c7e27a710/freezegun-1.5.5-py3-none-any.whl", hash = "sha256:cd557f4a75cf074e84bc374249b9dd491eaeacd61376b9eb3c423282211619d2", size = 19266, upload-time = "2025-08-09T10:39:06.636Z" }, ] +[[package]] +name = "google-api-core" +version = "2.30.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, + { name = "grpcio-status" }, +] + +[[package]] +name = "google-auth" +version = "2.57.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/41/64/55f316b729f92a552d26e00aa3b1542b2e149d0a5efe2842afff0cac7af7/google_auth-2.57.0.tar.gz", hash = "sha256:9b4f96d6a1feb5f7201231f47cfb3de08d8f176f8a61f9e461555116e95a8789", size = 370794, upload-time = "2026-08-25T19:18:26.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/f3/8508a702c094af5f6e89773f4dfdeee74913df0f41a02c21b5e7dc3d75cd/google_auth-2.57.0-py3-none-any.whl", hash = "sha256:180dafe015cfb62193bea26b677500fab5b9fd51a1e825ebf3ad9b182047ae59", size = 259728, upload-time = "2026-08-24T21:55:08.449Z" }, +] + +[package.optional-dependencies] +pyopenssl = [ + { name = "cryptography" }, +] + +[[package]] +name = "google-cloud-bigquery" +version = "3.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth", extra = ["pyopenssl"] }, + { name = "google-cloud-core" }, + { name = "google-resumable-media" }, + { name = "packaging" }, + { name = "python-dateutil" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/fb/86d5bfdfbd9d810f6a49eb3e9cc591a2b46d33262338acc244cc79d032c1/google_cloud_bigquery-3.44.0.tar.gz", hash = "sha256:30651ae469b419f450b9c96581fd4942e2e060490df1ac0314bf379f16883215", size = 527575, upload-time = "2026-08-25T19:18:36.317Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/84/af193ca97ce72b56fd05f8ad15776bf7446d99cb0ba1cd2f4880dbc52543/google_cloud_bigquery-3.44.0-py3-none-any.whl", hash = "sha256:ac2f0a6ab61a3c742ba4674dc220fb98461c13e1def96fe7a980ef7d5e0c0285", size = 267691, upload-time = "2026-08-24T21:55:19.249Z" }, +] + +[[package]] +name = "google-cloud-core" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f9/54/790548b190cff9cf07225c811d2668eacd8f2cbf04114475cbe3e5738752/google_cloud_core-2.7.0.tar.gz", hash = "sha256:874aaf89765db87a9b911b7a2ca7c5068554868eed9e75c7766affe342a2913d", size = 36603, upload-time = "2026-08-25T19:18:43.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/64/904dbc9bee128e7b87eeb60da67c8fa4d1a8acdc9a45dd2fedca67ef184d/google_cloud_core-2.7.0-py3-none-any.whl", hash = "sha256:c18a250904cfdda021eb3ae8b8238c9f9ca272a4cbbfb5cba946b3fe3022eed1", size = 31046, upload-time = "2026-08-24T21:55:28.36Z" }, +] + +[[package]] +name = "google-crc32c" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/ef/21ccfaab3d5078d41efe8612e0ed0bfc9ce22475de074162a91a25f7980d/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8", size = 31298, upload-time = "2025-12-16T00:20:32.241Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b8/f8413d3f4b676136e965e764ceedec904fe38ae8de0cdc52a12d8eb1096e/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7", size = 30872, upload-time = "2025-12-16T00:33:58.785Z" }, + { url = "https://files.pythonhosted.org/packages/f6/fd/33aa4ec62b290477181c55bb1c9302c9698c58c0ce9a6ab4874abc8b0d60/google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15", size = 33243, upload-time = "2025-12-16T00:40:21.46Z" }, + { url = "https://files.pythonhosted.org/packages/71/03/4820b3bd99c9653d1a5210cb32f9ba4da9681619b4d35b6a052432df4773/google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a", size = 33608, upload-time = "2025-12-16T00:40:22.204Z" }, + { url = "https://files.pythonhosted.org/packages/7c/43/acf61476a11437bf9733fb2f70599b1ced11ec7ed9ea760fdd9a77d0c619/google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2", size = 34439, upload-time = "2025-12-16T00:35:20.458Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, + { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, + { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, + { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, + { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, + { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" }, + { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, + { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, + { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, + { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, + { url = "https://files.pythonhosted.org/packages/52/c5/c171e4d8c44fec1422d801a6d2e5d7ddabd733eeda505c79730ee9607f07/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93", size = 28615, upload-time = "2025-12-16T00:40:29.298Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, +] + +[[package]] +name = "google-resumable-media" +version = "2.10.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/4b/44e128cd7b0e72cd26da4a3052cd8a9ef1b5a890be7c611558067ea0b354/google_resumable_media-2.10.2.tar.gz", hash = "sha256:1de441703cd298d75a419bfdc0066e9fc7b0a1de630df96eea8ce8f5c759358c", size = 2164918, upload-time = "2026-08-25T19:19:11.814Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/30/3e976681b5319715ab022363f58f6f8e10f546a1777756fe3ebd918857e4/google_resumable_media-2.10.2-py3-none-any.whl", hash = "sha256:e3cedc827a4ea41e216582d74346f1fb9fceb625a8c3c53912f2ca1d663334d7", size = 81531, upload-time = "2026-08-25T19:18:07.193Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.75.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/90/fb8f1c84537fbf210c1f53a53ae473a805f6599c5a40b93c1bbadd211f7a/googleapis_common_protos-1.75.2.tar.gz", hash = "sha256:8829a3d1e4508c5b7b9a6b9525f7fccff611f8531644579a76466c29295d4bb2", size = 154083, upload-time = "2026-08-25T19:19:13.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/5b/1c9e55363c3b1890a98cae813de5b4ea327845756cd8fb7ee690140c7eac/googleapis_common_protos-1.75.2-py3-none-any.whl", hash = "sha256:6b83302f554ea93a0f48409c7fc2050f954bcbcddb7e3a9c76d4a823cb22920e", size = 307002, upload-time = "2026-08-25T19:18:08.927Z" }, +] + [[package]] name = "greenlet" version = "3.3.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/8a/99/1cd3411c56a410994669062bd73dd58270c00cc074cac15f385a1fd91f8a/greenlet-3.3.1.tar.gz", hash = "sha256:41848f3230b58c08bb43dee542e74a2a2e34d3c59dc3076cec9151aeeedcae98", size = 184690, upload-time = "2026-01-23T15:31:02.076Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/65/5b235b40581ad75ab97dcd8b4218022ae8e3ab77c13c919f1a1dfe9171fd/greenlet-3.3.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:04bee4775f40ecefcdaa9d115ab44736cd4b9c5fba733575bfe9379419582e13", size = 273723, upload-time = "2026-01-23T15:30:37.521Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ad/eb4729b85cba2d29499e0a04ca6fbdd8f540afd7be142fd571eea43d712f/greenlet-3.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e1457f4fed12a50e427988a07f0f9df53cf0ee8da23fab16e6732c2ec909d4", size = 574874, upload-time = "2026-01-23T16:00:54.551Z" }, - { url = "https://files.pythonhosted.org/packages/87/32/57cad7fe4c8b82fdaa098c89498ef85ad92dfbb09d5eb713adedfc2ae1f5/greenlet-3.3.1-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:070472cd156f0656f86f92e954591644e158fd65aa415ffbe2d44ca77656a8f5", size = 586309, upload-time = "2026-01-23T16:05:25.18Z" }, - { url = "https://files.pythonhosted.org/packages/87/eb/8a1ec2da4d55824f160594a75a9d8354a5fe0a300fb1c48e7944265217e1/greenlet-3.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a300354f27dd86bae5fbf7002e6dd2b3255cd372e9242c933faf5e859b703fe", size = 586985, upload-time = "2026-01-23T15:32:47.968Z" }, - { url = "https://files.pythonhosted.org/packages/15/1c/0621dd4321dd8c351372ee8f9308136acb628600658a49be1b7504208738/greenlet-3.3.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e84b51cbebf9ae573b5fbd15df88887815e3253fc000a7d0ff95170e8f7e9729", size = 1547271, upload-time = "2026-01-23T16:04:18.977Z" }, - { url = "https://files.pythonhosted.org/packages/9d/53/24047f8924c83bea7a59c8678d9571209c6bfe5f4c17c94a78c06024e9f2/greenlet-3.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0093bd1a06d899892427217f0ff2a3c8f306182b8c754336d32e2d587c131b4", size = 1613427, upload-time = "2026-01-23T15:33:44.428Z" }, - { url = "https://files.pythonhosted.org/packages/ff/07/ac9bf1ec008916d1a3373cae212884c1dcff4a4ba0d41127ce81a8deb4e9/greenlet-3.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:7932f5f57609b6a3b82cc11877709aa7a98e3308983ed93552a1c377069b20c8", size = 226100, upload-time = "2026-01-23T15:30:56.957Z" }, { url = "https://files.pythonhosted.org/packages/ec/e8/2e1462c8fdbe0f210feb5ac7ad2d9029af8be3bf45bd9fa39765f821642f/greenlet-3.3.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5fd23b9bc6d37b563211c6abbb1b3cab27db385a4449af5c32e932f93017080c", size = 274974, upload-time = "2026-01-23T15:31:02.891Z" }, { url = "https://files.pythonhosted.org/packages/7e/a8/530a401419a6b302af59f67aaf0b9ba1015855ea7e56c036b5928793c5bd/greenlet-3.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f51496a0bfbaa9d74d36a52d2580d1ef5ed4fdfcff0a73730abfbbbe1403dd", size = 577175, upload-time = "2026-01-23T16:00:56.213Z" }, { url = "https://files.pythonhosted.org/packages/8e/89/7e812bb9c05e1aaef9b597ac1d0962b9021d2c6269354966451e885c4e6b/greenlet-3.3.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb0feb07fe6e6a74615ee62a880007d976cf739b6669cce95daa7373d4fc69c5", size = 590401, upload-time = "2026-01-23T16:05:26.365Z" }, @@ -811,6 +869,71 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/2b/98c7f93e6db9977aaee07eb1e51ca63bd5f779b900d362791d3252e60558/greenlet-3.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:301860987846c24cb8964bdec0e31a96ad4a2a801b41b4ef40963c1b44f33451", size = 233181, upload-time = "2026-01-23T15:33:00.29Z" }, ] +[[package]] +name = "grpcio" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/f6/3b781cd07a715ea5f5125ae264226e7fc4d87603d6d3955022cabfdc5da2/grpcio-1.83.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8ff0b8767ddd62704e0d9571c1890af08d84a3a689ebba1807e62519d0b3277f", size = 6338720, upload-time = "2026-07-23T15:19:13.177Z" }, + { url = "https://files.pythonhosted.org/packages/21/cc/d14833d15d5984e366f1b027fa78bd038c9b028c66880bffb0f5a4d25ee2/grpcio-1.83.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:4772402f43517b4824980be4b3b2274a81eec0004a70009473c31b340d43e223", size = 12178773, upload-time = "2026-07-23T15:19:15.401Z" }, + { url = "https://files.pythonhosted.org/packages/6b/98/8acbb416544e7871132d8e42a07ed70c802d70e6a16c6009e505a34d32a4/grpcio-1.83.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f4cee5fc86e84a0cf7ad1574b454c3320e087c07f55b7df5dc0ac6a873fb90c0", size = 6921203, upload-time = "2026-07-23T15:19:17.824Z" }, + { url = "https://files.pythonhosted.org/packages/45/9c/0fdbfaf4fc54e5c88f6bce4008a065092fe7fbc4460eb5617ae8b20fd505/grpcio-1.83.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f5e822a7e7d03282f6ad225e710493c48b9057a353358344a5f7c42b2b37618d", size = 7648508, upload-time = "2026-07-23T15:19:19.685Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ea/107b9dbb2ed3ad14dd774fd3dde7d29ff9938a6c198654becb2c3a0e9a6a/grpcio-1.83.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f5f410d7c2903eabb34789dfd6342eef04af1ad459943936b7e09a9f5bd417b9", size = 7079466, upload-time = "2026-07-23T15:19:21.478Z" }, + { url = "https://files.pythonhosted.org/packages/3b/06/9fa9941089e6fae83b060b6ce61c1e81053e52decae43197245f45e07d36/grpcio-1.83.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee94a4016fdf8699fb1fd8a38652475ff677f1c72074cee44deeeb9a7e95e745", size = 7605583, upload-time = "2026-07-23T15:19:23.74Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/f10fb56062dc2771c630827a82d9ad0ecd05cad572ea3b08d49f6631680a/grpcio-1.83.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c6444666317338e903093c7c756e6cc88eee59f798cb8dd41e87725bf54e1617", size = 8637810, upload-time = "2026-07-23T15:19:25.536Z" }, + { url = "https://files.pythonhosted.org/packages/99/55/f84927258f6a1b6ea6dea661fdc6de859b35e560c96f3012d15ccd39f85e/grpcio-1.83.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa074041231f03959cb097dd5517b0677b8ea49215bae01d5710a7b69dd59969", size = 8008021, upload-time = "2026-07-23T15:19:27.863Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6b/cdf72161397ccd29d4ca2192f641524536c9cf54ad948c9dd0e0e01138fa/grpcio-1.83.0-cp311-cp311-win32.whl", hash = "sha256:cb056f6e171c42639a50460b2929c82241fda51f71cf3dcdd68090fe45095a45", size = 4404376, upload-time = "2026-07-23T15:19:30.137Z" }, + { url = "https://files.pythonhosted.org/packages/df/ed/e0ffeb4c848699c194dc9fb6a29ab29bcb2b6aac8c416bf18c51bfe8242c/grpcio-1.83.0-cp311-cp311-win_amd64.whl", hash = "sha256:7416952ca770477990257206276999056f8316d79196f2f25942393e58a20b49", size = 5164469, upload-time = "2026-07-23T15:19:31.941Z" }, + { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" }, + { url = "https://files.pythonhosted.org/packages/39/33/b5b50fc2c6fbe350e04814047bb2d409feec7b36ef8b170254c050e06bc0/grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da", size = 12160538, upload-time = "2026-07-23T15:19:35.958Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" }, + { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" }, + { url = "https://files.pythonhosted.org/packages/17/5b/77af31228f55f55a2a5112bb0077ad0a1c4d23dbb0c2853a62475bbdcc14/grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc", size = 4394004, upload-time = "2026-07-23T15:19:50.618Z" }, + { url = "https://files.pythonhosted.org/packages/c0/da/f706e39550e7a3732ce2b9c5926107a93d74a802775b19b642a6df27dc96/grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df", size = 5158525, upload-time = "2026-07-23T15:19:52.246Z" }, + { url = "https://files.pythonhosted.org/packages/56/eb/135daaa713f32d33b8f99b4153b3f8dc3b2a124996ac15581bf9ebdad3c3/grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0", size = 6304480, upload-time = "2026-07-23T15:19:53.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a1/121806ce69f23138dabe06aa595b0e5f1ae051a37e4c1954eed7d692c800/grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1", size = 12154419, upload-time = "2026-07-23T15:19:56.3Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e8/d0389e09cd6b4c4d3089b92967ae4e3ffd64795bd349bf2f85cd6656d3da/grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867", size = 6873200, upload-time = "2026-07-23T15:19:58.701Z" }, + { url = "https://files.pythonhosted.org/packages/f8/51/f464c1d211fa50d5adbabe1b2e519948d99c13757052bfc9ea7afa28e284/grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881", size = 7618811, upload-time = "2026-07-23T15:20:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c0/539fe0832f2dd6500a28f5263071623fb34e8d4867aec632ccf81bd21156/grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f", size = 7042310, upload-time = "2026-07-23T15:20:02.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ca/ccf617d37ffa72567fa8e005ec7090c99da922799be2fb9847c8b21ca18c/grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61", size = 7575412, upload-time = "2026-07-23T15:20:04.712Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b9/fd8d5245f823a8e0fd35d90e20ea3aa4acd47f8d5318fa8df307df52dec6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45", size = 8604248, upload-time = "2026-07-23T15:20:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/14/1e/f37632fc11db72dfa4bba86c3a43e54358e53030df111ecae5e91a733ad6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c", size = 7977458, upload-time = "2026-07-23T15:20:09.109Z" }, + { url = "https://files.pythonhosted.org/packages/93/b6/d70b69ae5c0cfc341b9ba474980e4ed99cbf05c0e4a14e9eee8cb73db0a5/grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf", size = 4393993, upload-time = "2026-07-23T15:20:11.017Z" }, + { url = "https://files.pythonhosted.org/packages/0f/13/45d4cccb555cf4c476226979bf3d2fd0b0254216f7564c3a053e35117efc/grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735", size = 5159650, upload-time = "2026-07-23T15:20:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ab/d3874931d123a95e83a3ebf8aa04537988fb62425cedb8bf3cefc5ad41b2/grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c", size = 12166617, upload-time = "2026-07-23T15:20:17.435Z" }, + { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" }, + { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" }, + { url = "https://files.pythonhosted.org/packages/83/fa/c3fda157287f64bc65acee6c5aa90c41acf9e0d3a8e69a265eecff6d00a1/grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33", size = 4471765, upload-time = "2026-07-23T15:20:33.195Z" }, + { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" }, +] + +[[package]] +name = "grpcio-status" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/fd/848dd7e009de85f8ca59999d1cc618ff8ebf7ea5636d083a47455d212d24/grpcio_status-1.83.0.tar.gz", hash = "sha256:837219c6de9afdccb6f6f72b34bc71e151a2011ef04040e3faaca746a57e54ae", size = 13965, upload-time = "2026-07-23T15:24:26.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/00/73204406228cf989bea6b0fd9fe4702fab49a8a152a0c6f90856dadb6ac7/grpcio_status-1.83.0-py3-none-any.whl", hash = "sha256:f6a838a7c5fb84ae98833ec0ef81ed438c26e11e54b2ddb8e92ad328c861de69", size = 14636, upload-time = "2026-07-23T15:23:49.044Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -839,13 +962,6 @@ version = "0.7.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b5/46/120a669232c7bdedb9d52d4aeae7e6c7dfe151e99dc70802e2fc7a5e1993/httptools-0.7.1.tar.gz", hash = "sha256:abd72556974f8e7c74a259655924a717a2365b236c882c3f6f8a45fe94703ac9", size = 258961, upload-time = "2025-10-10T03:55:08.559Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/e5/c07e0bcf4ec8db8164e9f6738c048b2e66aabf30e7506f440c4cc6953f60/httptools-0.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:11d01b0ff1fe02c4c32d60af61a4d613b74fad069e47e06e9067758c01e9ac78", size = 204531, upload-time = "2025-10-10T03:54:20.887Z" }, - { url = "https://files.pythonhosted.org/packages/7e/4f/35e3a63f863a659f92ffd92bef131f3e81cf849af26e6435b49bd9f6f751/httptools-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:84d86c1e5afdc479a6fdabf570be0d3eb791df0ae727e8dbc0259ed1249998d4", size = 109408, upload-time = "2025-10-10T03:54:22.455Z" }, - { url = "https://files.pythonhosted.org/packages/f5/71/b0a9193641d9e2471ac541d3b1b869538a5fb6419d52fd2669fa9c79e4b8/httptools-0.7.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c8c751014e13d88d2be5f5f14fc8b89612fcfa92a9cc480f2bc1598357a23a05", size = 440889, upload-time = "2025-10-10T03:54:23.753Z" }, - { url = "https://files.pythonhosted.org/packages/eb/d9/2e34811397b76718750fea44658cb0205b84566e895192115252e008b152/httptools-0.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:654968cb6b6c77e37b832a9be3d3ecabb243bbe7a0b8f65fbc5b6b04c8fcabed", size = 440460, upload-time = "2025-10-10T03:54:25.313Z" }, - { url = "https://files.pythonhosted.org/packages/01/3f/a04626ebeacc489866bb4d82362c0657b2262bef381d68310134be7f40bb/httptools-0.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b580968316348b474b020edf3988eecd5d6eec4634ee6561e72ae3a2a0e00a8a", size = 425267, upload-time = "2025-10-10T03:54:26.81Z" }, - { url = "https://files.pythonhosted.org/packages/a5/99/adcd4f66614db627b587627c8ad6f4c55f18881549bab10ecf180562e7b9/httptools-0.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d496e2f5245319da9d764296e86c5bb6fcf0cf7a8806d3d000717a889c8c0b7b", size = 424429, upload-time = "2025-10-10T03:54:28.174Z" }, - { url = "https://files.pythonhosted.org/packages/d5/72/ec8fc904a8fd30ba022dfa85f3bbc64c3c7cd75b669e24242c0658e22f3c/httptools-0.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cbf8317bfccf0fed3b5680c559d3459cccf1abe9039bfa159e62e391c7270568", size = 86173, upload-time = "2025-10-10T03:54:29.5Z" }, { url = "https://files.pythonhosted.org/packages/9c/08/17e07e8d89ab8f343c134616d72eebfe03798835058e2ab579dcc8353c06/httptools-0.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:474d3b7ab469fefcca3697a10d11a32ee2b9573250206ba1e50d5980910da657", size = 206521, upload-time = "2025-10-10T03:54:31.002Z" }, { url = "https://files.pythonhosted.org/packages/aa/06/c9c1b41ff52f16aee526fd10fbda99fa4787938aa776858ddc4a1ea825ec/httptools-0.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3c3b7366bb6c7b96bd72d0dbe7f7d5eead261361f013be5f6d9590465ea1c70", size = 110375, upload-time = "2025-10-10T03:54:31.941Z" }, { url = "https://files.pythonhosted.org/packages/cc/cc/10935db22fda0ee34c76f047590ca0a8bd9de531406a3ccb10a90e12ea21/httptools-0.7.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:379b479408b8747f47f3b253326183d7c009a3936518cdb70db58cffd369d9df", size = 456621, upload-time = "2025-10-10T03:54:33.176Z" }, @@ -978,17 +1094,6 @@ version = "3.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631, upload-time = "2025-09-27T18:36:05.558Z" }, - { url = "https://files.pythonhosted.org/packages/98/1b/fbd8eed11021cabd9226c37342fa6ca4e8a98d8188a8d9b66740494960e4/markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", size = 12057, upload-time = "2025-09-27T18:36:07.165Z" }, - { url = "https://files.pythonhosted.org/packages/40/01/e560d658dc0bb8ab762670ece35281dec7b6c1b33f5fbc09ebb57a185519/markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", size = 22050, upload-time = "2025-09-27T18:36:08.005Z" }, - { url = "https://files.pythonhosted.org/packages/af/cd/ce6e848bbf2c32314c9b237839119c5a564a59725b53157c856e90937b7a/markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", size = 20681, upload-time = "2025-09-27T18:36:08.881Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2a/b5c12c809f1c3045c4d580b035a743d12fcde53cf685dbc44660826308da/markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", size = 20705, upload-time = "2025-09-27T18:36:10.131Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e3/9427a68c82728d0a88c50f890d0fc072a1484de2f3ac1ad0bfc1a7214fd5/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", size = 21524, upload-time = "2025-09-27T18:36:11.324Z" }, - { url = "https://files.pythonhosted.org/packages/bc/36/23578f29e9e582a4d0278e009b38081dbe363c5e7165113fad546918a232/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", size = 20282, upload-time = "2025-09-27T18:36:12.573Z" }, - { url = "https://files.pythonhosted.org/packages/56/21/dca11354e756ebd03e036bd8ad58d6d7168c80ce1fe5e75218e4945cbab7/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", size = 20745, upload-time = "2025-09-27T18:36:13.504Z" }, - { url = "https://files.pythonhosted.org/packages/87/99/faba9369a7ad6e4d10b6a5fbf71fa2a188fe4a593b15f0963b73859a1bbd/markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", size = 14571, upload-time = "2025-09-27T18:36:14.779Z" }, - { url = "https://files.pythonhosted.org/packages/d6/25/55dc3ab959917602c96985cb1253efaa4ff42f71194bddeb61eb7278b8be/markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", size = 15056, upload-time = "2025-09-27T18:36:16.125Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9e/0a02226640c255d1da0b8d12e24ac2aa6734da68bff14c05dd53b94a0fc3/markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", size = 13932, upload-time = "2025-09-27T18:36:17.311Z" }, { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, @@ -1081,14 +1186,6 @@ version = "1.1.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/4d/f2/bfb55a6236ed8725a96b0aa3acbd0ec17588e6a2c3b62a93eb513ed8783f/msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e", size = 173581, upload-time = "2025-10-08T09:15:56.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/a2/3b68a9e769db68668b25c6108444a35f9bd163bb848c0650d516761a59c0/msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2", size = 81318, upload-time = "2025-10-08T09:14:38.722Z" }, - { url = "https://files.pythonhosted.org/packages/5b/e1/2b720cc341325c00be44e1ed59e7cfeae2678329fbf5aa68f5bda57fe728/msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87", size = 83786, upload-time = "2025-10-08T09:14:40.082Z" }, - { url = "https://files.pythonhosted.org/packages/71/e5/c2241de64bfceac456b140737812a2ab310b10538a7b34a1d393b748e095/msgpack-1.1.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b696e83c9f1532b4af884045ba7f3aa741a63b2bc22617293a2c6a7c645f251", size = 398240, upload-time = "2025-10-08T09:14:41.151Z" }, - { url = "https://files.pythonhosted.org/packages/b7/09/2a06956383c0fdebaef5aa9246e2356776f12ea6f2a44bd1368abf0e46c4/msgpack-1.1.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:365c0bbe981a27d8932da71af63ef86acc59ed5c01ad929e09a0b88c6294e28a", size = 406070, upload-time = "2025-10-08T09:14:42.821Z" }, - { url = "https://files.pythonhosted.org/packages/0e/74/2957703f0e1ef20637d6aead4fbb314330c26f39aa046b348c7edcf6ca6b/msgpack-1.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41d1a5d875680166d3ac5c38573896453bbbea7092936d2e107214daf43b1d4f", size = 393403, upload-time = "2025-10-08T09:14:44.38Z" }, - { url = "https://files.pythonhosted.org/packages/a5/09/3bfc12aa90f77b37322fc33e7a8a7c29ba7c8edeadfa27664451801b9860/msgpack-1.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:354e81bcdebaab427c3df4281187edc765d5d76bfb3a7c125af9da7a27e8458f", size = 398947, upload-time = "2025-10-08T09:14:45.56Z" }, - { url = "https://files.pythonhosted.org/packages/4b/4f/05fcebd3b4977cb3d840f7ef6b77c51f8582086de5e642f3fefee35c86fc/msgpack-1.1.2-cp310-cp310-win32.whl", hash = "sha256:e64c8d2f5e5d5fda7b842f55dec6133260ea8f53c4257d64494c534f306bf7a9", size = 64769, upload-time = "2025-10-08T09:14:47.334Z" }, - { url = "https://files.pythonhosted.org/packages/d0/3e/b4547e3a34210956382eed1c85935fff7e0f9b98be3106b3745d7dec9c5e/msgpack-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa", size = 71293, upload-time = "2025-10-08T09:14:48.665Z" }, { url = "https://files.pythonhosted.org/packages/2c/97/560d11202bcd537abca693fd85d81cebe2107ba17301de42b01ac1677b69/msgpack-1.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2e86a607e558d22985d856948c12a3fa7b42efad264dca8a3ebbcfa2735d786c", size = 82271, upload-time = "2025-10-08T09:14:49.967Z" }, { url = "https://files.pythonhosted.org/packages/83/04/28a41024ccbd67467380b6fb440ae916c1e4f25e2cd4c63abe6835ac566e/msgpack-1.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:283ae72fc89da59aa004ba147e8fc2f766647b1251500182fac0350d8af299c0", size = 84914, upload-time = "2025-10-08T09:14:50.958Z" }, { url = "https://files.pythonhosted.org/packages/71/46/b817349db6886d79e57a966346cf0902a426375aadc1e8e7a86a75e22f19/msgpack-1.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61c8aa3bd513d87c72ed0b37b53dd5c5a0f58f2ff9f26e1555d3bd7948fb7296", size = 416962, upload-time = "2025-10-08T09:14:51.997Z" }, @@ -1173,7 +1270,6 @@ dependencies = [ { name = "pip" }, { name = "pyproject-hooks" }, { name = "setuptools" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "wheel" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/79/d149fb40bc425ad9defcb8ff73c65088bbc36a84b1825e035397d1c40624/pip_tools-7.5.2.tar.gz", hash = "sha256:2d64d72da6a044da1110257d333960563d7a4743637e8617dd2610ae7b82d60f", size = 164815, upload-time = "2025-11-12T22:46:12.627Z" } @@ -1215,6 +1311,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/19/fd3ef348460c80af7bb4669ea7926651d1f95c23ff2df18b9d24bab4f3fa/pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77", size = 226437, upload-time = "2025-12-16T21:14:32.409Z" }, ] +[[package]] +name = "proto-plus" +version = "1.28.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/a6/4fbadcc2044034449b3f8f0ce82dcf3005d53f37c136642103fd4836a31c/proto_plus-1.28.4.tar.gz", hash = "sha256:5ff7ecad828e032a491fcb86947801768e32237f99dd049b649965b892ae9a63", size = 58679, upload-time = "2026-08-25T19:19:15.102Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/5d/0f04b85dafdc3250ced7f2592efc17dce7f40712e941e9632202481e600d/proto_plus-1.28.4-py3-none-any.whl", hash = "sha256:4b01341272f8a348db3f003b6143109f83ab43091019d5181b3fcdf500ab32aa", size = 50797, upload-time = "2026-08-25T19:18:12.338Z" }, +] + +[[package]] +name = "protobuf" +version = "7.36.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/e7/0553e21d25ca4d9f573135775348a372c3ec34a93a71d5f297c3bac38341/protobuf-7.36.0.tar.gz", hash = "sha256:e8e09cb0d794c6687926fa558a8a6e72aa10edb997d5ca61da0765f12a3e00ea", size = 510034, upload-time = "2026-08-20T16:34:01.071Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/ae/58e3ca96cb2e118cc546b677359b3c6659f79a140935c08dec94c7998585/protobuf-7.36.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:9103532dffd80c6fab7e50c65a31007680a06eb57537d437bb1b35812c138a37", size = 453256, upload-time = "2026-08-20T16:33:53.945Z" }, + { url = "https://files.pythonhosted.org/packages/f0/15/5162230af4912697f0fe406f6800f80760945babcff0e2c2fe6c84ef2d5d/protobuf-7.36.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:bf94a5917c71058262de683669bc0a797a7669d3de71f0b36d058e3194f47b44", size = 341436, upload-time = "2026-08-20T16:33:55.134Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/1670b2bfc9a45e807e520c3e9be36524db9ccc7dc05ea17af7681cabdc61/protobuf-7.36.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:3297e60abdff301e5f74393d87f6cc59dacab5f024a89548a6e8de1d26576b16", size = 354440, upload-time = "2026-08-20T16:33:56.077Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f8/bd5804695ba400e423c33fd4d9f58c28d86633d5ba1945c36ff3967d98cb/protobuf-7.36.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:70f5ec8eb0da81a44360c0dc0beac99a0d78071d21956a7076bae8bd2051841b", size = 340439, upload-time = "2026-08-20T16:33:56.992Z" }, + { url = "https://files.pythonhosted.org/packages/ef/9f/acd02338235a3e7d03168c4303478347b7624fc8189ff4e7f0d2654bbe86/protobuf-7.36.0-cp310-abi3-win32.whl", hash = "sha256:7326fd717bdc419162a735938d89d4032332bcc3408804012b24ff3a37086071", size = 440216, upload-time = "2026-08-20T16:33:57.99Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4e/12cb93270967a2affff5b3f720694700d4d87712a67afd05c8cb3f6fa52c/protobuf-7.36.0-cp310-abi3-win_amd64.whl", hash = "sha256:1781cc1de61249b750848029bca452c0a8b7e990080316b9bbc2518b2117b488", size = 453731, upload-time = "2026-08-20T16:33:58.951Z" }, + { url = "https://files.pythonhosted.org/packages/01/c3/629999e78d46c1115c11886d51c6bd68c17ce4a944f1ea3e153a91316a33/protobuf-7.36.0-py3-none-any.whl", hash = "sha256:53374d53fc29a67f7dbbf0ade47d7526a0f0137bf0f9c90e48d8a60790ef748c", size = 177024, upload-time = "2026-08-20T16:34:00.053Z" }, +] + [[package]] name = "psycopg" version = "3.3.2" @@ -1245,6 +1368,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/c3/26b8a0908a9db249de3b4169692e1c7c19048a9bc41a4d3209cee7dbb758/psycopg_pool-3.3.0-py3-none-any.whl", hash = "sha256:2e44329155c410b5e8666372db44276a8b1ebd8c90f1c3026ebba40d4bc81063", size = 39995, upload-time = "2025-12-01T11:34:29.761Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -1278,19 +1422,6 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" }, - { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" }, - { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" }, - { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" }, - { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" }, - { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" }, - { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" }, - { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" }, - { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" }, - { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" }, - { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" }, - { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" }, { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, @@ -1369,14 +1500,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, - { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" }, - { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" }, - { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" }, - { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" }, - { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" }, - { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" }, - { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" }, { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, @@ -1434,7 +1557,6 @@ dependencies = [ { name = "isort" }, { name = "mccabe" }, { name = "platformdirs" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "tomlkit" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/d2/b081da1a8930d00e3fc06352a1d449aaf815d4982319fab5d8cdb2e9ab35/pylint-4.0.4.tar.gz", hash = "sha256:d9b71674e19b1c36d79265b5887bf8e55278cbe236c9e95d22dc82cf044fdbd2", size = 1571735, upload-time = "2025-11-30T13:29:04.315Z" } @@ -1470,12 +1592,10 @@ version = "9.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } wheels = [ @@ -1487,7 +1607,6 @@ name = "pytest-asyncio" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, { name = "pytest" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] @@ -1566,9 +1685,6 @@ name = "pywin32" version = "311" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/40/44efbb0dfbd33aca6a6483191dae0716070ed99e2ecb0c53683f400a0b4f/pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3", size = 8760432, upload-time = "2025-07-14T20:13:05.9Z" }, - { url = "https://files.pythonhosted.org/packages/5e/bf/360243b1e953bd254a82f12653974be395ba880e7ec23e3731d9f73921cc/pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b", size = 9590103, upload-time = "2025-07-14T20:13:07.698Z" }, - { url = "https://files.pythonhosted.org/packages/57/38/d290720e6f138086fb3d5ffe0b6caa019a791dd57866940c82e4eeaf2012/pywin32-311-cp310-cp310-win_arm64.whl", hash = "sha256:0502d1facf1fed4839a9a51ccbcc63d952cf318f78ffc00a7e78528ac27d7a2b", size = 8778557, upload-time = "2025-07-14T20:13:11.11Z" }, { url = "https://files.pythonhosted.org/packages/7c/af/449a6a91e5d6db51420875c54f6aff7c97a86a3b13a0b4f1a5c13b988de3/pywin32-311-cp311-cp311-win32.whl", hash = "sha256:184eb5e436dea364dcd3d2316d577d625c0351bf237c4e9a5fabbcfa5a58b151", size = 8697031, upload-time = "2025-07-14T20:13:13.266Z" }, { url = "https://files.pythonhosted.org/packages/51/8f/9bb81dd5bb77d22243d33c8397f09377056d5c687aa6d4042bea7fbf8364/pywin32-311-cp311-cp311-win_amd64.whl", hash = "sha256:3ce80b34b22b17ccbd937a6e78e7225d80c52f5ab9940fe0506a1a16f3dab503", size = 9508308, upload-time = "2025-07-14T20:13:15.147Z" }, { url = "https://files.pythonhosted.org/packages/44/7b/9c2ab54f74a138c491aba1b1cd0795ba61f144c711daea84a88b63dc0f6c/pywin32-311-cp311-cp311-win_arm64.whl", hash = "sha256:a733f1388e1a842abb67ffa8e7aad0e70ac519e09b0f6a784e65a136ec7cefd2", size = 8703930, upload-time = "2025-07-14T20:13:16.945Z" }, @@ -1589,15 +1705,6 @@ version = "6.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, - { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, - { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, - { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, - { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, - { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, - { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, @@ -1742,11 +1849,6 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/4aff125021a9c5e0183f2f55dd7d04b7256a0e1e10db50d537a7415d9c55/snowflake_connector_python-4.0.0.tar.gz", hash = "sha256:4b10a865c4a5e1fa60c365c7fe41e0433605e6e5edc824e8730a9038f330b3a6", size = 813937, upload-time = "2025-10-09T10:11:34.631Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/75/f845ca5079a6b911023fa945dbf1bac0ed1c2f5967108b14440c740cb410/snowflake_connector_python-4.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2c3e0f6d103fe67c975550ed424f579d3e7ae503d56467e5549f3a1a1e0e8f24", size = 1030251, upload-time = "2025-10-09T10:11:36.37Z" }, - { url = "https://files.pythonhosted.org/packages/fd/80/3a7e36a9e53beeb27c0599d2703f33bb812be931b469b154b08df0eeeaf5/snowflake_connector_python-4.0.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:e8d5b66f283967c700fff2303ac5e52d1a3cf41990a634f121ac8b1f1cd9af10", size = 1043041, upload-time = "2025-10-09T10:11:37.719Z" }, - { url = "https://files.pythonhosted.org/packages/6e/3b/bda95c4de593743c021a9968d70087674189c60a8317185de1b0f32d17c8/snowflake_connector_python-4.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ad5d0f1ebcb2c6b7a7859ee3d4e02203087e40faae539a336bbcb45a3660777", size = 2666209, upload-time = "2025-10-09T10:11:17.54Z" }, - { url = "https://files.pythonhosted.org/packages/60/e6/30c4015e2712bf8bf83b54ddadeee0494b68ae6d0f6d49d9373f463305d4/snowflake_connector_python-4.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4106a66e770e564b3037457b7b01b15ca28aee61afb88560b664aa8af439b533", size = 2693962, upload-time = "2025-10-09T10:11:20.735Z" }, - { url = "https://files.pythonhosted.org/packages/1b/e0/5f494b3353216e629f8b9d7269024518e0db9f2992df471de6ef43b60f7b/snowflake_connector_python-4.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:7789df78f7c7abfb351f2709258d05a94652cfe3c2c617fb15f15a11fc1b7b25", size = 1177353, upload-time = "2025-10-09T10:11:51.07Z" }, { url = "https://files.pythonhosted.org/packages/0c/86/0dceb37f50cd28ee61af1f0396eccd17a563d56d66067e0842ff8dfafe6d/snowflake_connector_python-4.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ca2503f705627f7e045da6254d97c37210a3b0a18b43d0f1b29616d0c7aaa01", size = 1030446, upload-time = "2025-10-09T10:11:39.321Z" }, { url = "https://files.pythonhosted.org/packages/7b/80/1ac8ae9494b2ee5bcc0e4cefe5aada86d0e61d21208a0107e99c5bec92ec/snowflake_connector_python-4.0.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:fd0d2d2c5cfd15f041e8522f5f8bdad0be4de7d805dd1646377fccd6bd404fa8", size = 1043031, upload-time = "2025-10-09T10:11:40.638Z" }, { url = "https://files.pythonhosted.org/packages/fe/db/fbd1dbe2d6ca2b8f99337e39d596a800e6c54451a93a9167927a15f60534/snowflake_connector_python-4.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebbdeec0d65c2e3f648c8b05839001c062984959417902717f7fc6eed983211d", size = 2677427, upload-time = "2025-10-09T10:11:22.537Z" }, @@ -1783,13 +1885,6 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/06/aa/9ce0f3e7a9829ead5c8ce549392f33a12c4555a6c0609bb27d882e9c7ddf/sqlalchemy-2.0.46.tar.gz", hash = "sha256:cf36851ee7219c170bb0793dbc3da3e80c582e04a5437bc601bfe8c85c9216d7", size = 9865393, upload-time = "2026-01-21T18:03:45.119Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/40/26/66ba59328dc25e523bfcb0f8db48bdebe2035e0159d600e1f01c0fc93967/sqlalchemy-2.0.46-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:895296687ad06dc9b11a024cf68e8d9d3943aa0b4964278d2553b86f1b267735", size = 2155051, upload-time = "2026-01-21T18:27:28.965Z" }, - { url = "https://files.pythonhosted.org/packages/21/cd/9336732941df972fbbfa394db9caa8bb0cf9fe03656ec728d12e9cbd6edc/sqlalchemy-2.0.46-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab65cb2885a9f80f979b85aa4e9c9165a31381ca322cbde7c638fe6eefd1ec39", size = 3234666, upload-time = "2026-01-21T18:32:28.72Z" }, - { url = "https://files.pythonhosted.org/packages/38/62/865ae8b739930ec433cd4123760bee7f8dafdc10abefd725a025604fb0de/sqlalchemy-2.0.46-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:52fe29b3817bd191cc20bad564237c808967972c97fa683c04b28ec8979ae36f", size = 3232917, upload-time = "2026-01-21T18:44:54.064Z" }, - { url = "https://files.pythonhosted.org/packages/24/38/805904b911857f2b5e00fdea44e9570df62110f834378706939825579296/sqlalchemy-2.0.46-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:09168817d6c19954d3b7655da6ba87fcb3a62bb575fb396a81a8b6a9fadfe8b5", size = 3185790, upload-time = "2026-01-21T18:32:30.581Z" }, - { url = "https://files.pythonhosted.org/packages/69/4f/3260bb53aabd2d274856337456ea52f6a7eccf6cce208e558f870cec766b/sqlalchemy-2.0.46-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:be6c0466b4c25b44c5d82b0426b5501de3c424d7a3220e86cd32f319ba56798e", size = 3207206, upload-time = "2026-01-21T18:44:55.93Z" }, - { url = "https://files.pythonhosted.org/packages/ce/b3/67c432d7f9d88bb1a61909b67e29f6354d59186c168fb5d381cf438d3b73/sqlalchemy-2.0.46-cp310-cp310-win32.whl", hash = "sha256:1bc3f601f0a818d27bfe139f6766487d9c88502062a2cd3a7ee6c342e81d5047", size = 2115296, upload-time = "2026-01-21T18:33:12.498Z" }, - { url = "https://files.pythonhosted.org/packages/4a/8c/25fb284f570f9d48e6c240f0269a50cec9cf009a7e08be4c0aaaf0654972/sqlalchemy-2.0.46-cp310-cp310-win_amd64.whl", hash = "sha256:e0c05aff5c6b1bb5fb46a87e0f9d2f733f83ef6cbbbcd5c642b6c01678268061", size = 2138540, upload-time = "2026-01-21T18:33:14.22Z" }, { url = "https://files.pythonhosted.org/packages/69/ac/b42ad16800d0885105b59380ad69aad0cce5a65276e269ce2729a2343b6a/sqlalchemy-2.0.46-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:261c4b1f101b4a411154f1da2b76497d73abbfc42740029205d4d01fa1052684", size = 2154851, upload-time = "2026-01-21T18:27:30.54Z" }, { url = "https://files.pythonhosted.org/packages/a0/60/d8710068cb79f64d002ebed62a7263c00c8fd95f4ebd4b5be8f7ca93f2bc/sqlalchemy-2.0.46-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:181903fe8c1b9082995325f1b2e84ac078b1189e2819380c2303a5f90e114a62", size = 3311241, upload-time = "2026-01-21T18:32:33.45Z" }, { url = "https://files.pythonhosted.org/packages/2b/0f/20c71487c7219ab3aa7421c7c62d93824c97c1460f2e8bb72404b0192d13/sqlalchemy-2.0.46-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:590be24e20e2424a4c3c1b0835e9405fa3d0af5823a1a9fc02e5dff56471515f", size = 3310741, upload-time = "2026-01-21T18:44:57.887Z" }, @@ -2021,7 +2116,6 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c3/d1/8f3c683c9561a4e6689dd3b1d345c815f10f86acd044ee1fb9a4dcd0b8c5/uvicorn-0.40.0.tar.gz", hash = "sha256:839676675e87e73694518b5574fd0f24c9d97b46bea16df7b8c05ea1a51071ea", size = 81761, upload-time = "2025-12-21T14:16:22.45Z" } wheels = [ @@ -2045,12 +2139,6 @@ version = "0.22.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/14/ecceb239b65adaaf7fde510aa8bd534075695d1e5f8dadfa32b5723d9cfb/uvloop-0.22.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c", size = 1343335, upload-time = "2025-10-16T22:16:11.43Z" }, - { url = "https://files.pythonhosted.org/packages/ba/ae/6f6f9af7f590b319c94532b9567409ba11f4fa71af1148cab1bf48a07048/uvloop-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792", size = 742903, upload-time = "2025-10-16T22:16:12.979Z" }, - { url = "https://files.pythonhosted.org/packages/09/bd/3667151ad0702282a1f4d5d29288fce8a13c8b6858bf0978c219cd52b231/uvloop-0.22.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac33ed96229b7790eb729702751c0e93ac5bc3bcf52ae9eccbff30da09194b86", size = 3648499, upload-time = "2025-10-16T22:16:14.451Z" }, - { url = "https://files.pythonhosted.org/packages/b3/f6/21657bb3beb5f8c57ce8be3b83f653dd7933c2fd00545ed1b092d464799a/uvloop-0.22.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:481c990a7abe2c6f4fc3d98781cc9426ebd7f03a9aaa7eb03d3bfc68ac2a46bd", size = 3700133, upload-time = "2025-10-16T22:16:16.272Z" }, - { url = "https://files.pythonhosted.org/packages/09/e0/604f61d004ded805f24974c87ddd8374ef675644f476f01f1df90e4cdf72/uvloop-0.22.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a592b043a47ad17911add5fbd087c76716d7c9ccc1d64ec9249ceafd735f03c2", size = 3512681, upload-time = "2025-10-16T22:16:18.07Z" }, - { url = "https://files.pythonhosted.org/packages/bb/ce/8491fd370b0230deb5eac69c7aae35b3be527e25a911c0acdffb922dc1cd/uvloop-0.22.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1489cf791aa7b6e8c8be1c5a080bae3a672791fcb4e9e12249b05862a2ca9cec", size = 3615261, upload-time = "2025-10-16T22:16:19.596Z" }, { url = "https://files.pythonhosted.org/packages/c7/d5/69900f7883235562f1f50d8184bb7dd84a2fb61e9ec63f3782546fdbd057/uvloop-0.22.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c60ebcd36f7b240b30788554b6f0782454826a0ed765d8430652621b5de674b9", size = 1352420, upload-time = "2025-10-16T22:16:21.187Z" }, { url = "https://files.pythonhosted.org/packages/a8/73/c4e271b3bce59724e291465cc936c37758886a4868787da0278b3b56b905/uvloop-0.22.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b7f102bf3cb1995cfeaee9321105e8f5da76fdb104cdad8986f85461a1b7b77", size = 748677, upload-time = "2025-10-16T22:16:22.558Z" }, { url = "https://files.pythonhosted.org/packages/86/94/9fb7fad2f824d25f8ecac0d70b94d0d48107ad5ece03769a9c543444f78a/uvloop-0.22.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21", size = 3753819, upload-time = "2025-10-16T22:16:23.903Z" }, @@ -2091,7 +2179,6 @@ dependencies = [ { name = "distlib" }, { name = "filelock" }, { name = "platformdirs" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/aa/a3/4d310fa5f00863544e1d0f4de93bddec248499ccf97d4791bc3122c9d4f3/virtualenv-20.36.1.tar.gz", hash = "sha256:8befb5c81842c641f8ee658481e42641c68b5eab3521d8e092d18320902466ba", size = 6032239, upload-time = "2026-01-09T18:21:01.296Z" } wheels = [ @@ -2107,18 +2194,6 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/1a/206e8cf2dd86fddf939165a57b4df61607a1e0add2785f170a3f616b7d9f/watchfiles-1.1.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:eef58232d32daf2ac67f42dea51a2c80f0d03379075d44a587051e63cc2e368c", size = 407318, upload-time = "2025-10-14T15:04:18.753Z" }, - { url = "https://files.pythonhosted.org/packages/b3/0f/abaf5262b9c496b5dad4ed3c0e799cbecb1f8ea512ecb6ddd46646a9fca3/watchfiles-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:03fa0f5237118a0c5e496185cafa92878568b652a2e9a9382a5151b1a0380a43", size = 394478, upload-time = "2025-10-14T15:04:20.297Z" }, - { url = "https://files.pythonhosted.org/packages/b1/04/9cc0ba88697b34b755371f5ace8d3a4d9a15719c07bdc7bd13d7d8c6a341/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ca65483439f9c791897f7db49202301deb6e15fe9f8fe2fed555bf986d10c31", size = 449894, upload-time = "2025-10-14T15:04:21.527Z" }, - { url = "https://files.pythonhosted.org/packages/d2/9c/eda4615863cd8621e89aed4df680d8c3ec3da6a4cf1da113c17decd87c7f/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f0ab1c1af0cb38e3f598244c17919fb1a84d1629cc08355b0074b6d7f53138ac", size = 459065, upload-time = "2025-10-14T15:04:22.795Z" }, - { url = "https://files.pythonhosted.org/packages/84/13/f28b3f340157d03cbc8197629bc109d1098764abe1e60874622a0be5c112/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bc570d6c01c206c46deb6e935a260be44f186a2f05179f52f7fcd2be086a94d", size = 488377, upload-time = "2025-10-14T15:04:24.138Z" }, - { url = "https://files.pythonhosted.org/packages/86/93/cfa597fa9389e122488f7ffdbd6db505b3b915ca7435ecd7542e855898c2/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e84087b432b6ac94778de547e08611266f1f8ffad28c0ee4c82e028b0fc5966d", size = 595837, upload-time = "2025-10-14T15:04:25.057Z" }, - { url = "https://files.pythonhosted.org/packages/57/1e/68c1ed5652b48d89fc24d6af905d88ee4f82fa8bc491e2666004e307ded1/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:620bae625f4cb18427b1bb1a2d9426dc0dd5a5ba74c7c2cdb9de405f7b129863", size = 473456, upload-time = "2025-10-14T15:04:26.497Z" }, - { url = "https://files.pythonhosted.org/packages/d5/dc/1a680b7458ffa3b14bb64878112aefc8f2e4f73c5af763cbf0bd43100658/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:544364b2b51a9b0c7000a4b4b02f90e9423d97fbbf7e06689236443ebcad81ab", size = 455614, upload-time = "2025-10-14T15:04:27.539Z" }, - { url = "https://files.pythonhosted.org/packages/61/a5/3d782a666512e01eaa6541a72ebac1d3aae191ff4a31274a66b8dd85760c/watchfiles-1.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bbe1ef33d45bc71cf21364df962af171f96ecaeca06bd9e3d0b583efb12aec82", size = 630690, upload-time = "2025-10-14T15:04:28.495Z" }, - { url = "https://files.pythonhosted.org/packages/9b/73/bb5f38590e34687b2a9c47a244aa4dd50c56a825969c92c9c5fc7387cea1/watchfiles-1.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1a0bb430adb19ef49389e1ad368450193a90038b5b752f4ac089ec6942c4dff4", size = 622459, upload-time = "2025-10-14T15:04:29.491Z" }, - { url = "https://files.pythonhosted.org/packages/f1/ac/c9bb0ec696e07a20bd58af5399aeadaef195fb2c73d26baf55180fe4a942/watchfiles-1.1.1-cp310-cp310-win32.whl", hash = "sha256:3f6d37644155fb5beca5378feb8c1708d5783145f2a0f1c4d5a061a210254844", size = 272663, upload-time = "2025-10-14T15:04:30.435Z" }, - { url = "https://files.pythonhosted.org/packages/11/a0/a60c5a7c2ec59fa062d9a9c61d02e3b6abd94d32aac2d8344c4bdd033326/watchfiles-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:a36d8efe0f290835fd0f33da35042a1bb5dc0e83cbc092dcf69bce442579e88e", size = 287453, upload-time = "2025-10-14T15:04:31.53Z" }, { url = "https://files.pythonhosted.org/packages/1f/f8/2c5f479fb531ce2f0564eda479faecf253d886b1ab3630a39b7bf7362d46/watchfiles-1.1.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f57b396167a2565a4e8b5e56a5a1c537571733992b226f4f1197d79e94cf0ae5", size = 406529, upload-time = "2025-10-14T15:04:32.899Z" }, { url = "https://files.pythonhosted.org/packages/fe/cd/f515660b1f32f65df671ddf6f85bfaca621aee177712874dc30a97397977/watchfiles-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:421e29339983e1bebc281fab40d812742268ad057db4aee8c4d2bce0af43b741", size = 394384, upload-time = "2025-10-14T15:04:33.761Z" }, { url = "https://files.pythonhosted.org/packages/7b/c3/28b7dc99733eab43fca2d10f55c86e03bd6ab11ca31b802abac26b23d161/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e43d39a741e972bab5d8100b5cdacf69db64e34eb19b6e9af162bccf63c5cc6", size = 448789, upload-time = "2025-10-14T15:04:34.679Z" }, @@ -2191,10 +2266,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/55/2af26693fd15165c4ff7857e38330e1b61ab8c37d15dc79118cdba115b7a/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c91ed27800188c2ae96d16e3149f199d62f86c7af5f5f4d2c61a3ed8cd3666c", size = 455072, upload-time = "2025-10-14T15:05:48.928Z" }, { url = "https://files.pythonhosted.org/packages/66/1d/d0d200b10c9311ec25d2273f8aad8c3ef7cc7ea11808022501811208a750/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:311ff15a0bae3714ffb603e6ba6dbfba4065ab60865d15a6ec544133bdb21099", size = 629104, upload-time = "2025-10-14T15:05:49.908Z" }, { url = "https://files.pythonhosted.org/packages/e3/bd/fa9bb053192491b3867ba07d2343d9f2252e00811567d30ae8d0f78136fe/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a916a2932da8f8ab582f242c065f5c81bed3462849ca79ee357dd9551b0e9b01", size = 622112, upload-time = "2025-10-14T15:05:50.941Z" }, - { url = "https://files.pythonhosted.org/packages/ba/4c/a888c91e2e326872fa4705095d64acd8aa2fb9c1f7b9bd0588f33850516c/watchfiles-1.1.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:17ef139237dfced9da49fb7f2232c86ca9421f666d78c264c7ffca6601d154c3", size = 409611, upload-time = "2025-10-14T15:06:05.809Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c7/5420d1943c8e3ce1a21c0a9330bcf7edafb6aa65d26b21dbb3267c9e8112/watchfiles-1.1.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:672b8adf25b1a0d35c96b5888b7b18699d27d4194bac8beeae75be4b7a3fc9b2", size = 396889, upload-time = "2025-10-14T15:06:07.035Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e5/0072cef3804ce8d3aaddbfe7788aadff6b3d3f98a286fdbee9fd74ca59a7/watchfiles-1.1.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77a13aea58bc2b90173bc69f2a90de8e282648939a00a602e1dc4ee23e26b66d", size = 451616, upload-time = "2025-10-14T15:06:08.072Z" }, - { url = "https://files.pythonhosted.org/packages/83/4e/b87b71cbdfad81ad7e83358b3e447fedd281b880a03d64a760fe0a11fc2e/watchfiles-1.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b495de0bb386df6a12b18335a0285dda90260f51bdb505503c02bcd1ce27a8b", size = 458413, upload-time = "2025-10-14T15:06:09.209Z" }, { url = "https://files.pythonhosted.org/packages/d3/8e/e500f8b0b77be4ff753ac94dc06b33d8f0d839377fee1b78e8c8d8f031bf/watchfiles-1.1.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:db476ab59b6765134de1d4fe96a1a9c96ddf091683599be0f26147ea1b2e4b88", size = 408250, upload-time = "2025-10-14T15:06:10.264Z" }, { url = "https://files.pythonhosted.org/packages/bd/95/615e72cd27b85b61eec764a5ca51bd94d40b5adea5ff47567d9ebc4d275a/watchfiles-1.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:89eef07eee5e9d1fda06e38822ad167a044153457e6fd997f8a858ab7564a336", size = 396117, upload-time = "2025-10-14T15:06:11.28Z" }, { url = "https://files.pythonhosted.org/packages/c9/81/e7fe958ce8a7fb5c73cc9fb07f5aeaf755e6aa72498c57d760af760c91f8/watchfiles-1.1.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce19e06cbda693e9e7686358af9cd6f5d61312ab8b00488bc36f5aabbaf77e24", size = 450493, upload-time = "2025-10-14T15:06:12.321Z" }, @@ -2207,15 +2278,6 @@ version = "16.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/74/221f58decd852f4b59cc3354cccaf87e8ef695fede361d03dc9a7396573b/websockets-16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a", size = 177343, upload-time = "2026-01-10T09:22:21.28Z" }, - { url = "https://files.pythonhosted.org/packages/19/0f/22ef6107ee52ab7f0b710d55d36f5a5d3ef19e8a205541a6d7ffa7994e5a/websockets-16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0", size = 175021, upload-time = "2026-01-10T09:22:22.696Z" }, - { url = "https://files.pythonhosted.org/packages/10/40/904a4cb30d9b61c0e278899bf36342e9b0208eb3c470324a9ecbaac2a30f/websockets-16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957", size = 175320, upload-time = "2026-01-10T09:22:23.94Z" }, - { url = "https://files.pythonhosted.org/packages/9d/2f/4b3ca7e106bc608744b1cdae041e005e446124bebb037b18799c2d356864/websockets-16.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72", size = 183815, upload-time = "2026-01-10T09:22:25.469Z" }, - { url = "https://files.pythonhosted.org/packages/86/26/d40eaa2a46d4302becec8d15b0fc5e45bdde05191e7628405a19cf491ccd/websockets-16.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde", size = 185054, upload-time = "2026-01-10T09:22:27.101Z" }, - { url = "https://files.pythonhosted.org/packages/b0/ba/6500a0efc94f7373ee8fefa8c271acdfd4dca8bd49a90d4be7ccabfc397e/websockets-16.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3", size = 184565, upload-time = "2026-01-10T09:22:28.293Z" }, - { url = "https://files.pythonhosted.org/packages/04/b4/96bf2cee7c8d8102389374a2616200574f5f01128d1082f44102140344cc/websockets-16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3", size = 183848, upload-time = "2026-01-10T09:22:30.394Z" }, - { url = "https://files.pythonhosted.org/packages/02/8e/81f40fb00fd125357814e8c3025738fc4ffc3da4b6b4a4472a82ba304b41/websockets-16.0-cp310-cp310-win32.whl", hash = "sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9", size = 178249, upload-time = "2026-01-10T09:22:32.083Z" }, - { url = "https://files.pythonhosted.org/packages/b4/5f/7e40efe8df57db9b91c88a43690ac66f7b7aa73a11aa6a66b927e44f26fa/websockets-16.0-cp310-cp310-win_amd64.whl", hash = "sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35", size = 178685, upload-time = "2026-01-10T09:22:33.345Z" }, { url = "https://files.pythonhosted.org/packages/f2/db/de907251b4ff46ae804ad0409809504153b3f30984daf82a1d84a9875830/websockets-16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8", size = 177340, upload-time = "2026-01-10T09:22:34.539Z" }, { url = "https://files.pythonhosted.org/packages/f3/fa/abe89019d8d8815c8781e90d697dec52523fb8ebe308bf11664e8de1877e/websockets-16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad", size = 175022, upload-time = "2026-01-10T09:22:36.332Z" }, { url = "https://files.pythonhosted.org/packages/58/5d/88ea17ed1ded2079358b40d31d48abe90a73c9e5819dbcde1606e991e2ad/websockets-16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d", size = 175319, upload-time = "2026-01-10T09:22:37.602Z" }, @@ -2287,18 +2349,6 @@ version = "2.0.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/49/2a/6de8a50cb435b7f42c46126cf1a54b2aab81784e74c8595c8e025e8f36d3/wrapt-2.0.1.tar.gz", hash = "sha256:9c9c635e78497cacb81e84f8b11b23e0aacac7a136e73b8e5b2109a1d9fc468f", size = 82040, upload-time = "2025-11-07T00:45:33.312Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/61/0d/12d8c803ed2ce4e5e7d5b9f5f602721f9dfef82c95959f3ce97fa584bb5c/wrapt-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:64b103acdaa53b7caf409e8d45d39a8442fe6dcfec6ba3f3d141e0cc2b5b4dbd", size = 77481, upload-time = "2025-11-07T00:43:11.103Z" }, - { url = "https://files.pythonhosted.org/packages/05/3e/4364ebe221ebf2a44d9fc8695a19324692f7dd2795e64bd59090856ebf12/wrapt-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:91bcc576260a274b169c3098e9a3519fb01f2989f6d3d386ef9cbf8653de1374", size = 60692, upload-time = "2025-11-07T00:43:13.697Z" }, - { url = "https://files.pythonhosted.org/packages/1f/ff/ae2a210022b521f86a8ddcdd6058d137c051003812b0388a5e9a03d3fe10/wrapt-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ab594f346517010050126fcd822697b25a7031d815bb4fbc238ccbe568216489", size = 61574, upload-time = "2025-11-07T00:43:14.967Z" }, - { url = "https://files.pythonhosted.org/packages/c6/93/5cf92edd99617095592af919cb81d4bff61c5dbbb70d3c92099425a8ec34/wrapt-2.0.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:36982b26f190f4d737f04a492a68accbfc6fa042c3f42326fdfbb6c5b7a20a31", size = 113688, upload-time = "2025-11-07T00:43:18.275Z" }, - { url = "https://files.pythonhosted.org/packages/a0/0a/e38fc0cee1f146c9fb266d8ef96ca39fb14a9eef165383004019aa53f88a/wrapt-2.0.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23097ed8bc4c93b7bf36fa2113c6c733c976316ce0ee2c816f64ca06102034ef", size = 115698, upload-time = "2025-11-07T00:43:19.407Z" }, - { url = "https://files.pythonhosted.org/packages/b0/85/bef44ea018b3925fb0bcbe9112715f665e4d5309bd945191da814c314fd1/wrapt-2.0.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8bacfe6e001749a3b64db47bcf0341da757c95959f592823a93931a422395013", size = 112096, upload-time = "2025-11-07T00:43:16.5Z" }, - { url = "https://files.pythonhosted.org/packages/7c/0b/733a2376e413117e497aa1a5b1b78e8f3a28c0e9537d26569f67d724c7c5/wrapt-2.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8ec3303e8a81932171f455f792f8df500fc1a09f20069e5c16bd7049ab4e8e38", size = 114878, upload-time = "2025-11-07T00:43:20.81Z" }, - { url = "https://files.pythonhosted.org/packages/da/03/d81dcb21bbf678fcda656495792b059f9d56677d119ca022169a12542bd0/wrapt-2.0.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:3f373a4ab5dbc528a94334f9fe444395b23c2f5332adab9ff4ea82f5a9e33bc1", size = 111298, upload-time = "2025-11-07T00:43:22.229Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d5/5e623040e8056e1108b787020d56b9be93dbbf083bf2324d42cde80f3a19/wrapt-2.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f49027b0b9503bf6c8cdc297ca55006b80c2f5dd36cecc72c6835ab6e10e8a25", size = 113361, upload-time = "2025-11-07T00:43:24.301Z" }, - { url = "https://files.pythonhosted.org/packages/a1/f3/de535ccecede6960e28c7b722e5744846258111d6c9f071aa7578ea37ad3/wrapt-2.0.1-cp310-cp310-win32.whl", hash = "sha256:8330b42d769965e96e01fa14034b28a2a7600fbf7e8f0cc90ebb36d492c993e4", size = 58035, upload-time = "2025-11-07T00:43:28.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/15/39d3ca5428a70032c2ec8b1f1c9d24c32e497e7ed81aed887a4998905fcc/wrapt-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:1218573502a8235bb8a7ecaed12736213b22dcde9feab115fa2989d42b5ded45", size = 60383, upload-time = "2025-11-07T00:43:25.804Z" }, - { url = "https://files.pythonhosted.org/packages/43/c2/dfd23754b7f7a4dce07e08f4309c4e10a40046a83e9ae1800f2e6b18d7c1/wrapt-2.0.1-cp310-cp310-win_arm64.whl", hash = "sha256:eda8e4ecd662d48c28bb86be9e837c13e45c58b8300e43ba3c9b4fa9900302f7", size = 58894, upload-time = "2025-11-07T00:43:27.074Z" }, { url = "https://files.pythonhosted.org/packages/98/60/553997acf3939079dab022e37b67b1904b5b0cc235503226898ba573b10c/wrapt-2.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e17283f533a0d24d6e5429a7d11f250a58d28b4ae5186f8f47853e3e70d2590", size = 77480, upload-time = "2025-11-07T00:43:30.573Z" }, { url = "https://files.pythonhosted.org/packages/2d/50/e5b3d30895d77c52105c6d5cbf94d5b38e2a3dd4a53d22d246670da98f7c/wrapt-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:85df8d92158cb8f3965aecc27cf821461bb5f40b450b03facc5d9f0d4d6ddec6", size = 60690, upload-time = "2025-11-07T00:43:31.594Z" }, { url = "https://files.pythonhosted.org/packages/f0/40/660b2898703e5cbbb43db10cdefcc294274458c3ca4c68637c2b99371507/wrapt-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c1be685ac7700c966b8610ccc63c3187a72e33cab53526a27b2a285a662cd4f7", size = 61578, upload-time = "2025-11-07T00:43:32.918Z" }, diff --git a/datajunction-server/datajunction_server/alembic/versions/2026_08_24_0000-rg0001reaggregate_add_reaggregate_to_noderevision.py b/datajunction-server/datajunction_server/alembic/versions/2026_08_24_0000-rg0001reaggregate_add_reaggregate_to_noderevision.py new file mode 100644 index 000000000..8a04b3fd2 --- /dev/null +++ b/datajunction-server/datajunction_server/alembic/versions/2026_08_24_0000-rg0001reaggregate_add_reaggregate_to_noderevision.py @@ -0,0 +1,27 @@ +""" +Add reaggregate column to noderevision + +Revision ID: rg0001reaggregate +Revises: cm0003dropowner +Create Date: 2026-08-24 00:00:00.000000+00:00 +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "rg0001reaggregate" +down_revision = "cm0003dropowner" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "noderevision", + sa.Column("reaggregate", sa.JSON(), nullable=True), + ) + + +def downgrade(): + op.drop_column("noderevision", "reaggregate") diff --git a/datajunction-server/datajunction_server/alembic/versions/2026_09_15_0000-fm0001params_add_params_to_frozen_measures.py b/datajunction-server/datajunction_server/alembic/versions/2026_09_15_0000-fm0001params_add_params_to_frozen_measures.py new file mode 100644 index 000000000..c00473a55 --- /dev/null +++ b/datajunction-server/datajunction_server/alembic/versions/2026_09_15_0000-fm0001params_add_params_to_frozen_measures.py @@ -0,0 +1,29 @@ +""" +Add params column to frozen_measures + +Adds a JSON column to persist tuning parameters (e.g., accuracy) for sketch-backed measures. + +Revision ID: fm0001params +Revises: rg0001reaggregate +Create Date: 2026-09-15 00:00:00.000000+00:00 +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "fm0001params" +down_revision = "rg0001reaggregate" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "frozen_measures", + sa.Column("params", sa.JSON(), nullable=True), + ) + + +def downgrade(): + op.drop_column("frozen_measures", "params") diff --git a/datajunction-server/datajunction_server/api/cubes.py b/datajunction-server/datajunction_server/api/cubes.py index 58e92f6a2..55616b86f 100644 --- a/datajunction-server/datajunction_server/api/cubes.py +++ b/datajunction-server/datajunction_server/api/cubes.py @@ -15,6 +15,9 @@ build_combiner_sql_from_preaggs, ) from datajunction_server.construction.build_v3.cte import strip_role_suffix +from datajunction_server.construction.build_v3.cube_matcher import ( + validate_cube_reaggregate_materialization, +) from datajunction_server.construction.dimensions import build_dimensions_from_cube_query from datajunction_server.database.materialization import Materialization from datajunction_server.database.node import Node @@ -30,7 +33,6 @@ AccessDenialMode, get_access_checker, ) -from datajunction_server.models.access import ResourceAction from datajunction_server.internal.materializations import ( build_cube_materialization, stop_cube_materialization_workflows, @@ -40,6 +42,7 @@ get_single_cube_revision_metadata, ) from datajunction_server.internal.views import CubeViewNames, _build_view_body +from datajunction_server.models.access import ResourceAction from datajunction_server.models.cube import ( CubeRevisionMetadata, DimensionValue, @@ -59,11 +62,10 @@ ) from datajunction_server.models.dialect import Dialect from datajunction_server.models.materialization import ( - DRUID_AGG_MAPPING, - DRUID_SKETCH_TYPES, Granularity, MaterializationJobTypeEnum, MaterializationStrategy, + get_druid_aggregator_spec, ) from datajunction_server.models.metric import TranslatedSQL from datajunction_server.models.node_type import NodeNameVersion @@ -175,32 +177,56 @@ def _build_metrics_spec( if internal_name: component = component_by_name.get(internal_name) - druid_type = "longSum" # Default fallback - - if component: - # Use merge function for pre-aggregated data, fall back to aggregation - agg_func = component.merge or component.aggregation - if agg_func: - key = (col.type, agg_func.lower()) - if key in DRUID_AGG_MAPPING: - druid_type = DRUID_AGG_MAPPING[key] - - metric_spec = { + metric_spec = ( + get_druid_aggregator_spec( + column_name=col.name, + column_type=col.type, + aggregation=component.aggregation, + merge=component.merge, + params=component.params, + ) + if component + else None + ) + # Unmappable measures fall back to longSum here, since we're loading + # pre-aggregated data; the materialization config omits them instead. + metric_spec = metric_spec or { "fieldName": col.name, "name": col.name, - "type": druid_type, + "type": "longSum", } - # HLL sketches need additional configuration - if druid_type in DRUID_SKETCH_TYPES: - metric_spec["lgK"] = 12 # Log2 of K, controls precision (4-21) - metric_spec["tgtHllType"] = "HLL_4" # HLL_4, HLL_6, or HLL_8 - metrics.append(metric_spec) return metrics +async def _validate_cube_reaggregate_materialization( + session: AsyncSession, + cube: Node, +) -> None: + """ + Validate materialization safety using full metric decomposition. + """ + if not cube.current: # pragma: no cover + return + + from datajunction_server.construction.build_v3.builder import setup_build_context + + ctx = await setup_build_context( + session=session, + metrics=cube.current.cube_node_metrics, + dimensions=cube.current.cube_node_dimensions, + filters=cube.current.cube_filters or None, + dialect=Dialect.SPARK, + use_materialized=False, + ) + validate_cube_reaggregate_materialization( + cube.current, + decomposed_metrics=ctx.decomposed_metrics, + ) + + @router.get("/cubes", name="Get all Cubes") async def get_all_cubes( *, @@ -288,6 +314,7 @@ async def cube_materialization_info( message=f"Cube node `{name}` does not exist.", http_status_code=404, ) + await _validate_cube_reaggregate_materialization(session, node) temporal_partitions = node.current.temporal_partition_columns() # type: ignore if len(temporal_partitions) != 1: raise DJInvalidInputException( @@ -520,6 +547,7 @@ async def materialize_cube( message=f"Cube '{name}' has no current revision", http_status_code=HTTPStatus.NOT_FOUND, ) + await _validate_cube_reaggregate_materialization(session, node) cube_tps = cube_revision.temporal_partition_columns() diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py b/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py index 536f54abd..802cb6ce1 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/metricmetadata.py @@ -1,6 +1,7 @@ """Metric metadata scalars""" import strawberry +from strawberry.scalars import JSON from datajunction_server.models.cube_materialization import ( Aggregability as Aggregability_, @@ -15,9 +16,15 @@ MetricComponent as MetricComponent_, ) from datajunction_server.models.node import MetricDirection as MetricDirection_ +from datajunction_server.models.reaggregate import ( + DimensionReaggregateRule as DimensionReaggregateRule_, + ReaggregateSpec as ReaggregateSpec_, + ReaggregationFunction as ReaggregationFunction_, +) MetricDirection = strawberry.enum(MetricDirection_) Aggregability = strawberry.enum(Aggregability_) +ReaggregationFunction = strawberry.enum(ReaggregationFunction_) @strawberry.type @@ -32,12 +39,48 @@ class Unit: abbreviation: str | None +@strawberry.experimental.pydantic.type( + model=DimensionReaggregateRule_, + all_fields=True, +) +class DimensionReaggregateRule: ... + + +@strawberry.experimental.pydantic.type(model=ReaggregateSpec_) +class ReaggregateSpec: + """ + Metric reaggregation declaration. + + Fields are listed explicitly rather than via `all_fields` because `params` + is an open dict, which has no automatic GraphQL mapping. + """ + + fn: strawberry.auto + weight: strawberry.auto + rules: strawberry.auto + params: JSON | None = None + + @strawberry.experimental.pydantic.type(model=AggregationRule_, all_fields=True) class AggregationRule: ... -@strawberry.experimental.pydantic.type(model=MetricComponent_, all_fields=True) -class MetricComponent: ... +@strawberry.experimental.pydantic.type(model=MetricComponent_) +class MetricComponent: + """ + A single measure with accumulate/merge phases. + + Fields are listed explicitly rather than via `all_fields` because `params` + is an open dict, which has no automatic GraphQL mapping. + """ + + name: strawberry.auto + expression: strawberry.auto + aggregation: strawberry.auto + merge: strawberry.auto + rule: strawberry.auto + grain_alias: strawberry.auto + params: JSON | None = None @strawberry.experimental.pydantic.type(model=DecomposedMetric_, all_fields=True) diff --git a/datajunction-server/datajunction_server/api/graphql/scalars/node.py b/datajunction-server/datajunction_server/api/graphql/scalars/node.py index e92b7ffd3..24be9d652 100644 --- a/datajunction-server/datajunction_server/api/graphql/scalars/node.py +++ b/datajunction-server/datajunction_server/api/graphql/scalars/node.py @@ -31,7 +31,9 @@ ) from datajunction_server.api.graphql.scalars.metricmetadata import ( DecomposedMetric, + DimensionReaggregateRule, MetricMetadata, + ReaggregateSpec, ) from datajunction_server.api.graphql.scalars.user import User from datajunction_server.api.graphql.utils import extract_fields @@ -45,6 +47,7 @@ from datajunction_server.models.node import NodeMode as NodeMode_ from datajunction_server.models.node import NodeStatus as NodeStatus_ from datajunction_server.models.node import NodeType as NodeType_ +from datajunction_server.models.reaggregate import parse_reaggregate_spec from datajunction_server.sql.parsing.backends.antlr4 import ast, parse NodeType = strawberry.enum(NodeType_) @@ -410,6 +413,29 @@ def materializations( # Only metrics will have these fields required_dimensions: list[Column] | None = None + @strawberry.field + def reaggregate(self, root: DBNodeRevision) -> ReaggregateSpec | None: + """ + Metric reaggregation declaration. + """ + if root.type != NodeType.METRIC: + return None + spec = parse_reaggregate_spec(root.reaggregate) + if not spec: + return None + return ReaggregateSpec( + fn=spec.fn, # type: ignore + weight=spec.weight, + params=spec.params, + rules=[ + DimensionReaggregateRule( + dimension=rule.dimension, + fn=rule.fn, # type: ignore + ) + for rule in spec.rules + ], + ) + @strawberry.field def primary_key(self, root: DBNodeRevision) -> list[str]: """ diff --git a/datajunction-server/datajunction_server/api/graphql/schema.graphql b/datajunction-server/datajunction_server/api/graphql/schema.graphql index 6d442f339..e4f8b97e0 100644 --- a/datajunction-server/datajunction_server/api/graphql/schema.graphql +++ b/datajunction-server/datajunction_server/api/graphql/schema.graphql @@ -7,6 +7,7 @@ enum Aggregability { type AggregationRule { type: Aggregability! level: [String!] + reaggregate: DimensionReaggregateRule } type Attribute { @@ -149,6 +150,11 @@ type DimensionLink { defaultValue: String } +type DimensionReaggregateRule { + dimension: String! + fn: ReaggregationFunction! +} + type Engine { name: String! version: String! @@ -296,6 +302,7 @@ type MetricComponent { merge: String rule: AggregationRule! grainAlias: String + params: JSON } enum MetricDirection { @@ -397,6 +404,7 @@ type NodeRevision { dimensionLinks: [DimensionLink!]! availability: AvailabilityState materializations: [MaterializationConfig!] + reaggregate: ReaggregateSpec primaryKey: [String!]! metricMetadata: MetricMetadata isDerivedMetric: Boolean! @@ -716,6 +724,25 @@ type Query { listNamespaces: [Namespace!]! } +type ReaggregateSpec { + fn: ReaggregationFunction + weight: String + rules: [DimensionReaggregateRule!]! + params: JSON +} + +enum ReaggregationFunction { + AUTO + NONE + SUM + AVG + WEIGHTED_AVG + LAST_VALUE + FIRST_VALUE + MIN + MAX +} + type SemanticEntity { name: String! diff --git a/datajunction-server/datajunction_server/construction/build_v3/builder.py b/datajunction-server/datajunction_server/construction/build_v3/builder.py index 9b1b5fa7a..747d7b181 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/builder.py +++ b/datajunction-server/datajunction_server/construction/build_v3/builder.py @@ -21,6 +21,7 @@ ) from datajunction_server.construction.build_v3.decomposition import ( decompose_and_group_metrics, + missing_reaggregate_dimensions, ) from datajunction_server.construction.build_v3.dimensions import parse_dimension_ref from datajunction_server.construction.build_v3.filters import ( @@ -320,6 +321,14 @@ async def setup_build_context( # Add dimensions referenced in metric expressions (e.g., LAG ORDER BY) add_dimensions_from_metric_expressions(ctx, ctx.decomposed_metrics) + output_dimensions_after_expression_scan = list(ctx.dimensions) + internal_reaggregate_dimensions = missing_reaggregate_dimensions( + ctx.decomposed_metrics.values(), + output_dimensions_after_expression_scan, + ) + for dimension in internal_reaggregate_dimensions: + if dimension not in ctx.dimensions: + ctx.dimensions.append(dimension) # A second load_nodes pass is needed when either: # 1. metric expressions introduced dimension nodes not yet in ctx.nodes, OR @@ -333,8 +342,15 @@ async def setup_build_context( } missing_dim_nodes = dim_roots_after - ctx.nodes.keys() internally_added_roots = dim_roots_after - dim_roots_before_load - if missing_dim_nodes or internally_added_roots: - await load_nodes(ctx) + try: + if ( + missing_dim_nodes + or internally_added_roots + or internal_reaggregate_dimensions + ): + await load_nodes(ctx) + finally: + ctx.dimensions = output_dimensions_after_expression_scan # Classify filters into dimension filters (WHERE) and metric filters (HAVING) # This MUST happen AFTER all nodes are loaded so we can correctly identify diff --git a/datajunction-server/datajunction_server/construction/build_v3/combiners.py b/datajunction-server/datajunction_server/construction/build_v3/combiners.py index c39857745..5ca723234 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/combiners.py +++ b/datajunction-server/datajunction_server/construction/build_v3/combiners.py @@ -600,6 +600,7 @@ async def build_combiner_sql_from_preaggs( measure_identity_token( compute_expression_hash(m.expression), m.aggregation, + m.params, ) for m in gg.components if m.expression diff --git a/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py b/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py index 07fead205..f0955bd04 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py +++ b/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py @@ -12,9 +12,13 @@ from sqlalchemy import and_, select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import joinedload, load_only, noload, selectinload +from sqlalchemy.orm import aliased, joinedload, load_only, noload, selectinload -from datajunction_server.construction.build_v3.decomposition import is_derived_metric +from datajunction_server.construction.build_v3.decomposition import ( + _reaggregate_dimension_requested, + is_derived_metric, + missing_reaggregate_dimensions, +) from datajunction_server.construction.build_v3.dimensions import parse_dimension_ref from datajunction_server.construction.build_v3.filters import ( parse_and_resolve_filters, @@ -36,14 +40,19 @@ ) from datajunction_server.database.catalog import Catalog from datajunction_server.database.column import Column -from datajunction_server.database.node import Node, NodeRevision +from datajunction_server.database.node import Node, NodeRelationship, NodeRevision from datajunction_server.database.partition import Partition from datajunction_server.errors import DJInvalidInputException from datajunction_server.instrumentation.provider import timed from datajunction_server.models.decompose import Aggregability from datajunction_server.models.dialect import Dialect from datajunction_server.models.node_type import NodeType +from datajunction_server.models.reaggregate import ( + ReaggregationFunction, + dimension_reaggregate_rules, +) from datajunction_server.naming import amenable_name +from datajunction_server.sql.decompose import MetricComponentExtractor from datajunction_server.sql.parsing import ast logger = logging.getLogger(__name__) @@ -53,6 +62,327 @@ # parsed: coverage can't be proven, so cube matching must fail SAFE (reject the # cube and fall back) rather than fail open and emit invalid SQL. _FILTER_COVERAGE_UNKNOWN = object() +ReaggregateRequirement = tuple[str, str, ReaggregationFunction] + + +def _split_dimension_ref(ref: str) -> tuple[str, str | None]: + """ + Return a dimension ref without role suffix and the role suffix, if present. + """ + if "[" not in ref: + return ref, None + base, role = ref.rsplit("[", 1) + return base, role.rstrip("]") + + +def _cube_dimension_covers_reaggregate_dimension( + protected_dimension: str, + cube_dimension: str, +) -> bool: + """ + Return whether a cube dimension materializes a protected dimension. + """ + if _reaggregate_dimension_requested(protected_dimension, [cube_dimension]): + return True + + protected_base, protected_role = _split_dimension_ref(protected_dimension) + cube_base, cube_role = _split_dimension_ref(cube_dimension) + if protected_role != cube_role: + return False + + # Deployment validation allows a metric to author the protected dimension as + # a bare parent column (the same form required_dimensions accepts). Cubes + # expose dimensions as fully-qualified element names, so compare the short + # column name for that narrow compatibility case. + if "." not in protected_base: + return protected_base == cube_base.rsplit(".", 1)[-1] + return False + + +def _missing_reaggregate_requirements( + requirements: list[ReaggregateRequirement], + cube_dims: set[str], +) -> list[ReaggregateRequirement]: + """ + Reaggregate requirements not covered by a cube's materialized dimensions. + """ + return [ + requirement + for requirement in requirements + if not any( + _cube_dimension_covers_reaggregate_dimension( + requirement[1], + cube_dim, + ) + for cube_dim in cube_dims + ) + ] + + +async def _metric_graph_has_reaggregate( + session: AsyncSession, + metrics: list[str], +) -> bool: + """ + Cheaply test whether requested metrics or metric ancestors use reaggregate. + """ + seen: set[str] = set() + pending = set(metrics) + ParentNode = aliased(Node) + + while pending: + batch = sorted(pending - seen) + if not batch: + break + seen.update(batch) + result = await session.execute( + select( + Node.name, + NodeRevision.reaggregate, + ParentNode.name.label("parent_name"), + ) + .select_from(Node) + .join( + NodeRevision, + and_( + Node.id == NodeRevision.node_id, + Node.current_version == NodeRevision.version, + ), + ) + .outerjoin(NodeRelationship, NodeRelationship.child_id == NodeRevision.id) + .outerjoin( + ParentNode, + and_( + NodeRelationship.parent_id == ParentNode.id, + ParentNode.type == NodeType.METRIC, + ), + ) + .where(Node.name.in_(batch)) + .where(Node.type == NodeType.METRIC), + ) + for _, reaggregate, parent_name in result.all(): + if reaggregate: + return True + if parent_name and parent_name not in seen: + pending.add(parent_name) + + return False + + +async def _reaggregate_requirements_for_metrics_if_needed( + session: AsyncSession, + metrics: list[str], + dimensions: list[str], +) -> list[ReaggregateRequirement]: + """ + Run full metric decomposition only when reaggregate is possible. + """ + if not await _metric_graph_has_reaggregate(session, metrics): + return [] + return await _reaggregate_requirements_for_metrics(session, metrics, dimensions) + + +def _reaggregate_requirements_for_cube_metrics( + cube: NodeRevision, + metrics: list[str], + requested_dimensions: list[str], +) -> list[ReaggregateRequirement]: + """ + Reaggregation requirements that are absent from the output grain. + """ + requirements: list[ReaggregateRequirement] = [] + metric_names = set(metrics) + for metric_revision in cube.metric_node_revisions(): + if ( + not metric_revision + or metric_revision.name not in metric_names + or not metric_revision.reaggregate + ): + continue + for rule in dimension_reaggregate_rules(metric_revision.reaggregate): + if not _reaggregate_dimension_requested( + rule.dimension, + requested_dimensions, + ): + requirements.append( + ( + metric_revision.name, + rule.dimension, + rule.fn, + ), + ) + return requirements + + +async def _reaggregate_requirements_for_metrics( + session: AsyncSession, + metrics: list[str], + requested_dimensions: list[str], +) -> list[ReaggregateRequirement]: + """ + Reaggregation requirements from full metric decomposition. + + Direct cube metadata only sees the metrics declared on the cube. Decomposition + also catches derived metrics whose base components carry reaggregate rules. + """ + requirements: list[ReaggregateRequirement] = [] + for metric_name in metrics: + extractor = await MetricComponentExtractor.from_node_name( + metric_name, + session, + ) + components, _ = await extractor.extract(session) + for component in components: + reaggregate = component.rule.reaggregate + if ( + reaggregate + and not _reaggregate_dimension_requested( + reaggregate.dimension, + requested_dimensions, + ) + and ( + metric_name, + reaggregate.dimension, + reaggregate.fn, + ) + not in requirements + ): + requirements.append( + ( + metric_name, + reaggregate.dimension, + reaggregate.fn, + ), + ) + return requirements + + +def _reaggregate_requirements_for_decomposed_metrics( + decomposed_metrics: dict[str, DecomposedMetricInfo], + metrics: list[str], + requested_dimensions: list[str], +) -> list[ReaggregateRequirement]: + """ + Reaggregation requirements from already-decomposed planner context. + """ + requirements: list[ReaggregateRequirement] = [] + metric_names = set(metrics) + for metric_name, decomposed in decomposed_metrics.items(): + if metric_name not in metric_names: + continue + for component in decomposed.components: + reaggregate = component.rule.reaggregate + if reaggregate and not _reaggregate_dimension_requested( + reaggregate.dimension, + requested_dimensions, + ): + requirement = ( + metric_name, + reaggregate.dimension, + reaggregate.fn, + ) + if requirement not in requirements: + requirements.append(requirement) + return requirements + + +def _reaggregate_dimensions_for_cube_metrics( + cube: NodeRevision, + metrics: list[str], + requested_dimensions: list[str], +) -> list[str]: + """ + Protected dimensions a cube must materialize for semi-additive rollups. + """ + required: list[str] = [] + for _, dimension, _ in _reaggregate_requirements_for_cube_metrics( + cube, + metrics, + requested_dimensions, + ): + if dimension not in required: + required.append(dimension) + return required + + +def _format_reaggregate_requirements( + requirements: list[ReaggregateRequirement], +) -> list[str]: + """Human-readable semi-additive requirements for error messages.""" + return [ + f"{metric} -> {dimension} ({function.value})" + for metric, dimension, function in requirements + ] + + +def validate_cube_covers_reaggregate_dimensions( + cube: NodeRevision, + metrics: list[str], + dimensions: list[str], + *, + decomposed_metrics: dict[str, DecomposedMetricInfo] | None = None, + additional_requirements: list[ReaggregateRequirement] | None = None, + usage: str = "Cube", + http_status_code: int = 422, +) -> None: + """ + Fail loud when a materialized cube cannot safely serve semi-additive metrics. + + V1 materialized cube usage is only safe for semi-additive metrics when the + cube itself retains the protected dimension. Otherwise querying the cube can + silently merge across that dimension with the wrong Druid aggregator. + """ + requirements = _reaggregate_requirements_for_cube_metrics( + cube, + metrics, + dimensions, + ) + if decomposed_metrics is not None: + for requirement in _reaggregate_requirements_for_decomposed_metrics( + decomposed_metrics, + metrics, + dimensions, + ): + if requirement not in requirements: + requirements.append(requirement) + for requirement in additional_requirements or []: + if requirement not in requirements: + requirements.append(requirement) + + cube_dims = set(cube.cube_dimensions()) + missing_requirements = _missing_reaggregate_requirements(requirements, cube_dims) + if not missing_requirements: + return + + raise DJInvalidInputException( + f"{usage} `{cube.name}` cannot safely use materialized semi-additive " + "metric(s) because it does not materialize protected dimension(s) " + f"{_format_reaggregate_requirements(missing_requirements)}. " + "Add the protected dimension to the cube, or run the query with " + "use_materialized=false.", + http_status_code=http_status_code, + ) + + +def validate_cube_reaggregate_materialization( + cube: NodeRevision, + decomposed_metrics: dict[str, DecomposedMetricInfo] | None = None, +) -> None: + """ + Guard Druid cube materialization from baking in semi-additive misaggregation. + + V1 does not materialize collapsed semi-additive metrics unless the cube grain + includes the protected dimension. This avoids generating a Druid metricsSpec + that sums first/last/min/max collapse inputs across the protected dimension. + """ + validate_cube_covers_reaggregate_dimensions( + cube, + cube.cube_node_metrics, + cube.cube_node_dimensions, + decomposed_metrics=decomposed_metrics, + usage="Cube", + http_status_code=400, + ) async def _required_filter_dimensions( @@ -190,7 +520,15 @@ async def find_matching_cube( result = await session.execute(statement) candidate_cubes = result.unique().scalars().all() - + reaggregate_requirements = ( + await _reaggregate_requirements_for_metrics_if_needed( + session, + metrics, + dimensions, + ) + if candidate_cubes + else [] + ) # Find the best matching cube (smallest grain that covers all dimensions) best_match: NodeRevision | None = None best_grain_size = float("inf") @@ -214,12 +552,32 @@ async def find_matching_cube( # Druid SQL referencing a missing column). cube_dims = set(cube_rev.cube_dimensions()) - if not required_dims.issubset(cube_dims): + missing_required_dims = required_dims - cube_dims + cube_reaggregate_requirements = _reaggregate_requirements_for_cube_metrics( + cube_rev, + metrics, + dimensions, + ) + for requirement in reaggregate_requirements: + if requirement not in cube_reaggregate_requirements: + cube_reaggregate_requirements.append(requirement) + missing_reaggregate_requirements = _missing_reaggregate_requirements( + cube_reaggregate_requirements, + cube_dims, + ) + if missing_required_dims: logger.debug( f"[BuildV3] Cube {cube_rev.name} dims {cube_dims} " f"don't cover required {required_dims}", ) continue + if missing_reaggregate_requirements: + logger.debug( + f"[BuildV3] Cube {cube_rev.name} dims {cube_dims} " + f"don't cover reaggregate requirements " + f"{_format_reaggregate_requirements(missing_reaggregate_requirements)}", + ) + continue # Found a match - prefer smallest grain (less roll-up work) if len(cube_dims) < best_grain_size: @@ -244,6 +602,7 @@ async def validate_pinned_cube_covers_filters( cube: NodeRevision, dimensions: list[str], filters: list[str] | None, + metrics: list[str] | None = None, ) -> None: """ Ensure an explicitly pinned cube covers every filtered dimension. @@ -278,6 +637,18 @@ async def validate_pinned_cube_covers_filters( http_status_code=422, ) + validate_cube_covers_reaggregate_dimensions( + cube, + metrics or cube.cube_node_metrics, + dimensions, + additional_requirements=await _reaggregate_requirements_for_metrics_if_needed( + session, + metrics or cube.cube_node_metrics, + dimensions, + ), + usage="Pinned cube", + ) + async def resolve_dialect_and_engine_for_metrics( session: AsyncSession, @@ -465,6 +836,13 @@ def build_sql_from_cube_impl( Returns: GeneratedSQL with the query and column metadata. """ + validate_cube_covers_reaggregate_dimensions( + cube, + ctx.metrics, + ctx.dimensions, + decomposed_metrics=ctx.decomposed_metrics, + ) + # Build synthetic GrainGroupSQL for cube table # This applies all filters in the cube CTE's WHERE clause synthetic_grain_group = build_synthetic_grain_group( @@ -640,13 +1018,27 @@ def build_synthetic_grain_group( # (filter-only dimensions were added by add_dimensions_from_filters() in setup_build_context) dimension_aliases: dict[str, str] = {} - # Add all dimensions (requested + filter-only). We need all dimensions + output_dimensions = [ + dim for dim in ctx.dimensions if dim not in ctx.filter_dimensions + ] + internal_reaggregate_dimensions = missing_reaggregate_dimensions( + decomposed_metrics.values(), + output_dimensions, + ) + + # Add all dimensions (requested + filter-only + internal semi-additive + # protected dimensions). We need all dimensions # in the cube SELECT for proper filter resolution. # dim_short_names holds the alias (short name) used everywhere outside the CTE. # dim_physical_names holds the actual column name in the Druid table (may differ). dim_short_names = [] dim_physical_names = [] - for dim_ref in ctx.dimensions: + dimension_to_alias: dict[str, str] = {} + dimension_refs = list(ctx.dimensions) + for dim_ref in internal_reaggregate_dimensions: + if dim_ref not in dimension_refs: + dimension_refs.append(dim_ref) + for dim_ref in dimension_refs: parsed_dim = parse_dimension_ref(dim_ref) short_name = parsed_dim.column_name if parsed_dim.role: @@ -662,6 +1054,7 @@ def build_synthetic_grain_group( # the WHERE is applied directly on the cube table, so we must reference # the physical column (e.g. common_DOT_..._DOT_dateint) not the alias. dimension_aliases[dim_ref] = physical_name + dimension_to_alias[dim_ref] = short_name grain_group_columns.append( ColumnMetadata( name=short_name, @@ -735,6 +1128,17 @@ def build_synthetic_grain_group( if metric_node and not is_derived_metric(ctx, metric_node): base_metrics.append(metric_name) + reaggregate_dimension_aliases: dict[str, str] = {} + internal_reaggregate_dimension_set = set(internal_reaggregate_dimensions) + for comp in all_components: + if ( + comp.rule.reaggregate + and comp.rule.reaggregate.dimension in internal_reaggregate_dimension_set + ): + reaggregate_dimension_aliases[comp.name] = dimension_to_alias[ + comp.rule.reaggregate.dimension + ] + # Create the synthetic GrainGroupSQL # Note: We use a placeholder parent_name since the cube combines multiple parents return GrainGroupSQL( @@ -745,6 +1149,7 @@ def build_synthetic_grain_group( metrics=base_metrics, # Only base metrics, not derived parent_name=cube.name, # Use cube name as parent component_aliases=component_aliases, + reaggregate_dimension_aliases=reaggregate_dimension_aliases, is_merged=False, components=all_components, dialect=ctx.dialect, diff --git a/datajunction-server/datajunction_server/construction/build_v3/decomposition.py b/datajunction-server/datajunction_server/construction/build_v3/decomposition.py index 8a3c1b56f..6482a92e1 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/decomposition.py +++ b/datajunction-server/datajunction_server/construction/build_v3/decomposition.py @@ -9,6 +9,8 @@ from __future__ import annotations +from collections import defaultdict +from collections.abc import Iterable from typing import cast from sqlalchemy.ext.asyncio import AsyncSession @@ -26,6 +28,7 @@ from datajunction_server.sql.decompose import MetricComponentExtractor from datajunction_server.sql.parsing import ast from datajunction_server.sql.parsing.backends.antlr4 import parse +from datajunction_server.utils import SEPARATOR async def decompose_and_group_metrics( @@ -322,6 +325,76 @@ def get_native_grain(node: Node) -> list[str]: return pk_columns +def _dimension_ref_base(ref: str) -> str: + """Return a dimension ref without its role suffix.""" + return ref.split("[", 1)[0] + + +def _dimension_ref_role(ref: str) -> str | None: + """Return the role suffix for a dimension ref, if any.""" + if "[" not in ref: + return None + return ref.rsplit("[", 1)[1].rstrip("]") + + +def _dimension_ref_column(ref: str) -> str: + """Return the column part of a fully qualified or local dimension ref.""" + return _dimension_ref_base(ref).rsplit(SEPARATOR, 1)[-1] + + +def _reaggregate_dimension_requested( + protected_dimension: str, + requested_dimensions: list[str], +) -> bool: + """ + Whether the protected dimension is already in the requested output grain. + + Role-qualified dimensions only count when the role matches exactly. Coarser + dimensions on the same node do not count. + """ + protected_base = _dimension_ref_base(protected_dimension) + protected_role = _dimension_ref_role(protected_dimension) + protected_col = _dimension_ref_column(protected_dimension) + for requested in requested_dimensions: + requested_base = _dimension_ref_base(requested) + requested_role = _dimension_ref_role(requested) + if requested == protected_dimension: + return True + if requested_base == protected_base and requested_role == protected_role: + return True + if ( + protected_role is None + and requested_role is None + and requested == protected_col + ): + return True + return False + + +def missing_reaggregate_dimensions( + decomposed_metrics: Iterable[DecomposedMetricInfo], + requested_dimensions: list[str], +) -> list[str]: + """ + Protected dimensions needed as internal grain but absent from output grain. + """ + missing: list[str] = [] + for decomposed in decomposed_metrics: + for component in decomposed.components: + if not component.rule.reaggregate: + continue + dimension = component.rule.reaggregate.dimension + if ( + not _reaggregate_dimension_requested( + dimension, + requested_dimensions, + ) + and dimension not in missing + ): + missing.append(dimension) + return missing + + def analyze_grain_groups( metric_group: MetricGroup, requested_dimensions: list[str], @@ -341,7 +414,7 @@ def analyze_grain_groups( Args: metric_group: MetricGroup with decomposed metrics - requested_dimensions: Dimensions requested by user (column names only) + requested_dimensions: Dimension refs requested by the user Returns: List of GrainGroups, one per unique grain @@ -349,11 +422,15 @@ def analyze_grain_groups( parent_node = metric_group.parent_node # Group components by their effective grain - # Key: (aggregability, tuple of additional grain columns) + # Key: (aggregability, tuple of additional grain columns, tuple of internal dimensions) grain_buckets: dict[ - tuple[Aggregability, tuple[str, ...]], + tuple[Aggregability, tuple[str, ...], tuple[str, ...]], list[tuple[Node, MetricComponent]], ] = {} + reaggregate_component_dimensions: dict[ + tuple[Aggregability, tuple[str, ...], tuple[str, ...]], + dict[str, str], + ] = {} # Track non-decomposable metrics (those with no components) non_decomposable: list[DecomposedMetricInfo] = [] @@ -369,29 +446,50 @@ def analyze_grain_groups( agg_type = component.rule.type # Explicitly type the key to satisfy mypy - key: tuple[Aggregability, tuple[str, ...]] + key: tuple[Aggregability, tuple[str, ...], tuple[str, ...]] + reaggregate_dimension = ( + component.rule.reaggregate.dimension + if ( + component.rule.reaggregate + and not _reaggregate_dimension_requested( + component.rule.reaggregate.dimension, + requested_dimensions, + ) + ) + else None + ) + internal_dimensions = ( + (reaggregate_dimension,) if reaggregate_dimension else () + ) if agg_type == Aggregability.FULL: - # FULL: no additional grain columns needed - key = (Aggregability.FULL, ()) + # FULL: no additional grain columns needed, unless a semi-additive + # component must keep its protected dimension as internal grain. + key = (Aggregability.FULL, (), internal_dimensions) elif agg_type == Aggregability.LIMITED: # LIMITED: add level columns to grain level_cols = tuple(sorted(component.rule.level or [])) - key = (Aggregability.LIMITED, level_cols) + key = (Aggregability.LIMITED, level_cols, internal_dimensions) else: # NONE # NONE: use native grain (PK columns) native_grain = get_native_grain(parent_node) key = ( Aggregability.NONE, tuple(sorted(native_grain)), + internal_dimensions, ) # pragma: no cover if key not in grain_buckets: grain_buckets[key] = [] grain_buckets[key].append((decomposed.metric_node, component)) + if reaggregate_dimension: + reaggregate_component_dimensions.setdefault(key, {})[component.name] = ( + reaggregate_dimension + ) # Convert buckets to GrainGroup objects grain_groups = [] - for (agg_type, grain_cols), components in grain_buckets.items(): + for bucket_key, components in grain_buckets.items(): + agg_type, grain_cols, _internal_dimensions = bucket_key # Map each grain expression to its SQL alias. # LIMITED: alias comes from component.grain_alias (_make_component decides: # plain column maps to the bare name, complex expr maps to component.name). @@ -412,6 +510,10 @@ def analyze_grain_groups( grain_columns=list(grain_cols), components=components, grain_col_aliases=grain_col_aliases, + reaggregate_component_dimensions=reaggregate_component_dimensions.get( + bucket_key, + {}, + ), ), ) @@ -470,9 +572,10 @@ def merge_grain_groups(grain_groups: list[GrainGroup]) -> list[GrainGroup]: Returns: List of grain groups with compatible groups merged """ - from collections import defaultdict - - # Group by parent node name + # Group by parent node name first, then by the internal grain needed for + # semi-additive collapse. LIMITED/NONE groups must not be merged into a + # semi-additive FULL group, because their extra grain columns would make the + # protected-dimension bucket contain multiple rows and corrupt MAX_BY/MIN_BY. by_parent: dict[str, list[GrainGroup]] = defaultdict(list) for gg in grain_groups: by_parent[gg.parent_node.name].append(gg) @@ -483,6 +586,12 @@ def merge_grain_groups(grain_groups: list[GrainGroup]) -> list[GrainGroup]: if len(parent_groups) == 1: # Only one group for this parent - no merge needed merged_groups.append(parent_groups[0]) + elif any(group.reaggregate_component_dimensions for group in parent_groups): + # Keep semi-additive groups isolated. Merging a protected-dimension + # group with another grain can add rows inside the protected bucket, + # which makes MAX_BY/MIN_BY pick one lower-grain row instead of the + # already-aggregated value at that protected grain. + merged_groups.extend(parent_groups) else: # Multiple groups for same parent - merge them merged = _merge_parent_grain_groups(parent_groups) @@ -520,12 +629,16 @@ def _merge_parent_grain_groups(groups: list[GrainGroup]) -> GrainGroup: # Collect all components and track their original aggregabilities all_components: list[tuple[Node, MetricComponent]] = [] component_aggregabilities: dict[str, Aggregability] = {} + reaggregate_component_dimensions: dict[str, str] = {} for gg in groups: for metric_node, component in gg.components: all_components.append((metric_node, component)) # Track original aggregability for each component component_aggregabilities[component.name] = gg.aggregability + reaggregate_component_dimensions.update( + gg.reaggregate_component_dimensions, + ) # Carry over non-decomposable metrics from every contributing group. # Without this, merging a NONE group into a FULL/LIMITED neighbor @@ -559,5 +672,6 @@ def _merge_parent_grain_groups(groups: list[GrainGroup]) -> GrainGroup: is_merged=True, component_aggregabilities=component_aggregabilities, grain_col_aliases=merged_grain_col_aliases, + reaggregate_component_dimensions=reaggregate_component_dimensions, non_decomposable_metrics=all_non_decomposable, ) diff --git a/datajunction-server/datajunction_server/construction/build_v3/loaders.py b/datajunction-server/datajunction_server/construction/build_v3/loaders.py index 934bf069b..9c744c900 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/loaders.py +++ b/datajunction-server/datajunction_server/construction/build_v3/loaders.py @@ -87,6 +87,7 @@ async def batch_load_nodes_with_dependencies( NodeRevision.name, NodeRevision.query, NodeRevision.schema_, + NodeRevision.reaggregate, NodeRevision.table, ), # NOTE: don't noload Column.attributes — Columns are identity- diff --git a/datajunction-server/datajunction_server/construction/build_v3/measures.py b/datajunction-server/datajunction_server/construction/build_v3/measures.py index f6912654c..11414d7ad 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/measures.py +++ b/datajunction-server/datajunction_server/construction/build_v3/measures.py @@ -1078,6 +1078,8 @@ def build_select_ast( grain_columns: list[str] | None = None, grain_col_aliases: dict[str, str] | None = None, grain_col_specs: list[tuple[ast.Expression, str]] | None = None, + output_dimension_refs: set[str] | None = None, + internal_dimension_aliases: dict[str, str] | None = None, filters: list[str] | None = None, skip_aggregation: bool = False, ) -> tuple[ast.Query, list[str]]: @@ -1096,6 +1098,9 @@ def build_select_ast( the SQL alias to use in the generated SELECT. When provided, the alias is taken from this dict (keyed by the raw expression string from ``rule.level``). + output_dimension_refs: Optional set of dimension refs to expose as user-facing + dimensions. Other resolved dimensions are join/group-only. + internal_dimension_aliases: Dimension refs to project as private grain columns. filters: Optional list of filter strings to apply as WHERE clause. Filter strings can reference dimensions (e.g., "v3.product.category = 'Electronics'") or local columns (e.g., "status = 'active'"). @@ -1119,16 +1124,29 @@ def build_select_ast( spark_hints = _collect_spark_hints(resolved_dimensions, dim_aliases) # Add dimension columns to projection - # Filter-only dimensions are excluded from projection but included in GROUP BY + # Filter-only dimensions are excluded from projection but included in GROUP BY. + # Internal dimensions are projected under private aliases so the metrics layer + # can collapse semi-additive measures without exposing those dimensions. + internal_dimension_aliases = internal_dimension_aliases or {} for resolved_dim in resolved_dimensions: clean_alias = ctx.alias_registry.register(resolved_dim.original_ref) if resolved_dim.original_ref in ctx.filter_dimensions: continue + if ( + output_dimension_refs is not None + and resolved_dim.original_ref not in output_dimension_refs + and resolved_dim.original_ref not in internal_dimension_aliases + ): + continue + output_alias = internal_dimension_aliases.get( + resolved_dim.original_ref, + clean_alias, + ) col_expr = build_dimension_col_expr( resolved_dim, main_alias, dim_aliases, - clean_alias, + output_alias, ctx=ctx, resolved_dimensions=resolved_dimensions, parent_node_name=parent_node.name, @@ -1690,6 +1708,7 @@ def build_grain_group_from_preagg( select_items: list[ast.Aliasable | ast.Expression | ast.Column] = [] columns: list[ColumnMetadata] = [] component_aliases: dict[str, str] = {} + reaggregate_dimension_aliases: dict[str, str] = {} metrics_covered: set[str] = set() unique_components: list[MetricComponent] = [] seen_components: set[str] = set() @@ -1787,6 +1806,14 @@ def build_grain_group_from_preagg( ), ) + for ( + component_name, + dimension_ref, + ) in grain_group.reaggregate_component_dimensions.items(): + reaggregate_dimension_aliases[component_name] = ctx.alias_registry.register( + dimension_ref, + ) + # Add measure columns with re-aggregation (or grain columns if no merge func) for metric_node, component in grain_group.components: metrics_covered.add(metric_node.name) @@ -1947,6 +1974,7 @@ def build_grain_group_from_preagg( metrics=list(metrics_covered), parent_name=parent_node.name, component_aliases=component_aliases, + reaggregate_dimension_aliases=reaggregate_dimension_aliases, is_merged=grain_group.is_merged, component_aggregabilities=grain_group.component_aggregabilities, components=unique_components, @@ -1960,6 +1988,7 @@ def build_grain_group_sql( grain_group: GrainGroup, resolved_dimensions: list[ResolvedDimension], components_per_metric: dict[str, int], + output_dimension_refs: set[str] | None = None, ) -> GrainGroupSQL: """ Build SQL for a single grain group. @@ -2012,6 +2041,21 @@ def build_grain_group_sql( # Track mapping from component name to actual SQL alias # This is needed for metrics SQL to correctly reference component columns component_aliases: dict[str, str] = {} + reaggregate_dimension_aliases: dict[str, str] = {} + internal_dimension_aliases: dict[str, str] = {} + + if output_dimension_refs: + for resolved_dim in resolved_dimensions: + if resolved_dim.original_ref in output_dimension_refs: + ctx.alias_registry.register(resolved_dim.original_ref) + + for ( + component_name, + dimension_ref, + ) in grain_group.reaggregate_component_dimensions.items(): + dimension_alias = ctx.alias_registry.register(dimension_ref) + reaggregate_dimension_aliases[component_name] = dimension_alias + internal_dimension_aliases[dimension_ref] = dimension_alias for metric_node, component in grain_group.components: metrics_covered.add(metric_node.name) @@ -2180,6 +2224,8 @@ def build_grain_group_sql( resolved_dimensions=effective_resolved_dimensions, parent_node=parent_node, grain_columns=pass_through_columns, + output_dimension_refs=output_dimension_refs, + internal_dimension_aliases=internal_dimension_aliases, filters=ctx.dimension_filters, # Use dimension_filters only (not metric_filters) skip_aggregation=True, # Don't add GROUP BY ) @@ -2203,6 +2249,8 @@ def build_grain_group_sql( grain_columns=effective_grain_columns, grain_col_aliases=grain_group.grain_col_aliases or None, grain_col_specs=grain_col_specs, # already parsed above + output_dimension_refs=output_dimension_refs, + internal_dimension_aliases=internal_dimension_aliases, filters=ctx.dimension_filters, # Use dimension_filters only (not metric_filters) skip_aggregation=skip_agg, ) @@ -2211,7 +2259,12 @@ def build_grain_group_sql( columns_metadata = [] # Add dimension columns (skip filter-only dimensions as they're not in projection) + effective_output_dimension_refs = output_dimension_refs or { + resolved_dim.original_ref for resolved_dim in resolved_dimensions + } for resolved_dim in resolved_dimensions: + if resolved_dim.original_ref not in effective_output_dimension_refs: + continue # Skip filter-only dimensions from column metadata if resolved_dim.original_ref in ctx.filter_dimensions: continue @@ -2268,6 +2321,30 @@ def build_grain_group_sql( ), ) + for dimension_ref, dimension_alias in internal_dimension_aliases.items(): + internal_resolved_dim = next( + (dim for dim in resolved_dimensions if dim.original_ref == dimension_ref), + None, + ) + if not internal_resolved_dim: + continue # pragma: no cover + dim_node = ctx.nodes.get(internal_resolved_dim.node_name) + col_type = ( + get_column_type(parent_node, internal_resolved_dim.column_name) + if internal_resolved_dim.is_local + else get_column_type(dim_node, internal_resolved_dim.column_name) + if dim_node + else "string" + ) + columns_metadata.append( + ColumnMetadata( + name=dimension_alias, + semantic_name=dimension_ref, + type=col_type, + semantic_type="dimension", + ), + ) + # Add metric component columns # All decomposed metrics are now treated as components - no special case for single-component for comp_alias, component, metric_node in component_metadata: @@ -2313,6 +2390,8 @@ def build_grain_group_sql( if grain_group.aggregability != Aggregability.NONE: # FULL/LIMITED: dimensions are part of the grain for resolved_dim in resolved_dimensions: + if resolved_dim.original_ref not in effective_output_dimension_refs: + continue # Skip filter-only dimensions from grain if resolved_dim.original_ref in ctx.filter_dimensions: continue @@ -2327,6 +2406,9 @@ def build_grain_group_sql( for gc_alias in effective_grain_aliases: if gc_alias not in full_grain: # pragma: no branch full_grain.append(gc_alias) + for dimension_alias in internal_dimension_aliases.values(): + if dimension_alias not in full_grain: + full_grain.append(dimension_alias) # Sort for deterministic output full_grain.sort() @@ -2339,6 +2421,7 @@ def build_grain_group_sql( metrics=list(metrics_covered), parent_name=grain_group.parent_node.name, component_aliases=component_aliases, + reaggregate_dimension_aliases=reaggregate_dimension_aliases, is_merged=grain_group.is_merged, component_aggregabilities=grain_group.component_aggregabilities, components=unique_components, @@ -2375,18 +2458,15 @@ def process_metric_group( components_per_metric[decomposed.metric_node.name] = len(decomposed.components) # Analyze grain groups - split by aggregability - # Extract just the column names from dimensions for grain analysis - dim_column_names = [parse_dimension_ref(d).column_name for d in ctx.dimensions] - grain_groups = analyze_grain_groups(metric_group, dim_column_names) + output_dimensions = list(ctx.dimensions) + output_dimension_refs = set(output_dimensions) + grain_groups = analyze_grain_groups(metric_group, output_dimensions) # Merge compatible grain groups from same parent into single CTEs # This optimization reduces duplicate JOINs by outputting raw values # at finest grain, with aggregations applied in final SELECT grain_groups = merge_grain_groups(grain_groups) - # Resolve dimensions (find join paths) - shared across grain groups - resolved_dimensions = resolve_dimensions(ctx, parent_node) - # Build SQL for each grain group. Fan-out risk is flagged inside # build_grain_group_sql, where the full set of emitted join paths is known. grain_group_sqls: list[GrainGroupSQL] = [] @@ -2394,13 +2474,26 @@ def process_metric_group( # Reset alias registry for each grain group to avoid conflicts ctx.alias_registry = AliasRegistry() ctx._table_alias_counter = 0 - - grain_group_sql = build_grain_group_sql( - ctx, - grain_group, - resolved_dimensions, - components_per_metric, - ) + original_dimensions = ctx.dimensions + internal_dimensions = [ + dimension_ref + for dimension_ref in dict.fromkeys( + grain_group.reaggregate_component_dimensions.values(), + ) + if dimension_ref not in output_dimension_refs + ] + ctx.dimensions = output_dimensions + internal_dimensions + try: + resolved_dimensions = resolve_dimensions(ctx, parent_node) + grain_group_sql = build_grain_group_sql( + ctx, + grain_group, + resolved_dimensions, + components_per_metric, + output_dimension_refs=output_dimension_refs, + ) + finally: + ctx.dimensions = original_dimensions grain_group_sqls.append(grain_group_sql) return grain_group_sqls @@ -2514,6 +2607,33 @@ def find_parent_for_window_metric( else: return None, base_metrics # pragma: no cover + def find_base_metrics_for_window_parent( + metric_name: str, + visited: set[str], + ) -> set[str]: + """ + Recursively find grain-group metrics needed to compute a window parent. + + Window expressions may reference a derived metric directly, but the + window grain group still needs the underlying base metric components. + """ + if metric_name in visited: + return set() # pragma: no cover + visited.add(metric_name) + + for gg in existing_grain_groups: + if metric_name in gg.metrics: + return {metric_name} + + grain_group_metrics: set[str] = set() + for parent_name in ctx.parent_map.get(metric_name, []): + parent_node = ctx.nodes.get(parent_name) + if parent_node and parent_node.type.value == "metric": # pragma: no branch + grain_group_metrics.update( + find_base_metrics_for_window_parent(parent_name, visited), + ) + return grain_group_metrics + # Group window metrics by (ORDER BY grain, parent fact) # This ensures window metrics from different facts are processed separately # Key: (frozenset of grain cols, parent_name or "cross_fact") @@ -2525,13 +2645,18 @@ def find_parent_for_window_metric( parent_name, base_metrics = find_parent_for_window_metric(metric_name) # Use "cross_fact" as a marker for cross-fact window metrics parent_key = parent_name if parent_name else "cross_fact" + component_metrics = set() + for base_metric in base_metrics: + component_metrics.update( + find_base_metrics_for_window_parent(base_metric, set()), + ) group_key = (grain_key, parent_key) if group_key not in grain_parent_to_metrics: grain_parent_to_metrics[group_key] = [] grain_parent_to_base_metrics[group_key] = set() grain_parent_to_metrics[group_key].append(metric_name) - grain_parent_to_base_metrics[group_key].update(base_metrics) + grain_parent_to_base_metrics[group_key].update(component_metrics) additional_grain_groups: list[GrainGroupSQL] = [] diff --git a/datajunction-server/datajunction_server/construction/build_v3/metrics.py b/datajunction-server/datajunction_server/construction/build_v3/metrics.py index fb8cb3b28..31bc8eb8c 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/metrics.py +++ b/datajunction-server/datajunction_server/construction/build_v3/metrics.py @@ -50,13 +50,159 @@ ) from datajunction_server.errors import DJInvalidInputException from datajunction_server.models.decompose import Aggregability +from datajunction_server.models.dialect import Dialect from datajunction_server.models.node_type import NodeType +from datajunction_server.models.reaggregate import ReaggregationFunction from datajunction_server.sql.decompose import wrap_divisions_in_nullif from datajunction_server.sql.parsing import ast +from datajunction_server.sql.parsing.backends.antlr4 import parse logger = logging.getLogger(__name__) +def _build_reaggregate_collapse_expression( + collapse_function: ReaggregationFunction, + dialect: Dialect, + value_ref: ast.Expression, + protected_dim_ref: ast.Expression, +) -> ast.Function: + """ + Build the final aggregation for a semi-additive metric collapse. + """ + if collapse_function == ReaggregationFunction.LAST_VALUE: + function_name = "LATEST_BY" if dialect == Dialect.DRUID else "MAX_BY" + return ast.Function( + ast.Name(function_name), + args=[value_ref, protected_dim_ref], + ) + if collapse_function == ReaggregationFunction.FIRST_VALUE: + function_name = "EARLIEST_BY" if dialect == Dialect.DRUID else "MIN_BY" + return ast.Function( + ast.Name(function_name), + args=[value_ref, protected_dim_ref], + ) + if collapse_function == ReaggregationFunction.MIN: + return ast.Function(ast.Name("MIN"), args=[value_ref]) + if collapse_function == ReaggregationFunction.MAX: + return ast.Function(ast.Name("MAX"), args=[value_ref]) + + raise DJInvalidInputException( + f"Unsupported semi-additive collapse function: {collapse_function}", + ) + + +def _references_component(expr: ast.Node, component_name: str) -> bool: + """ + Return whether an expression references a decomposed metric component. + """ + for col in expr.find_all(ast.Column): + if col.name and col.name.name == component_name: + return True + return False + + +def _replace_reaggregate_merge_expression( + combiner_ast: ast.Expression, + component_name: str, + merge_function: str, + collapse_expr: ast.Expression, +) -> ast.Expression: + """ + Replace the component's merge aggregate in a combiner with a collapse expr. + + Semi-additive metrics still need the full decomposed combiner expression. + For example, ``SUM(balance) / 100`` should become + ``MAX_BY(balance_sum, date_id) / 100``, not just ``MAX_BY(...)``. + """ + expr_ast = ( + cast(ast.Expression, combiner_ast.child) + if isinstance(combiner_ast, ast.Alias) + else combiner_ast + ) + target_function = merge_function.upper() + + for func in list(expr_ast.find_all(ast.Function)): + if func.name.name.upper() == target_function and _references_component( + func, + component_name, + ): + if func is expr_ast: + return collapse_expr + if func.parent: + func.parent.replace(from_=func, to=collapse_expr) + return expr_ast + + raise DJInvalidInputException( + "Unsupported semi-additive metric shape: could not find the component " + "merge expression in the metric combiner.", + ) + + +def _dimension_ref_base(ref: str) -> str: + """Return a dimension ref without its role suffix.""" + return ref.split("[", 1)[0] + + +def _dimension_ref_role(ref: str) -> str | None: + """Return the role suffix for a dimension ref, if any.""" + if "[" not in ref: + return None + return ref.rsplit("[", 1)[1].rstrip("]") + + +def _source_dimension_alias(ctx: BuildContext, dimension_ref: str) -> str | None: + """ + Return the registered source column alias for a dimension ref. + """ + alias = ctx.alias_registry.get_alias(dimension_ref) + if alias or _dimension_ref_role(dimension_ref) is not None: + return alias + return ctx.alias_registry.get_alias(_dimension_ref_base(dimension_ref)) + + +def _metric_refs_from_query(ctx: BuildContext, metric_name: str) -> set[str]: + """ + Return metric refs used by a metric's query expression. + """ + metric_node = ctx.nodes.get(metric_name) + if not metric_node: + return set() # pragma: no cover + + if not metric_node.current or not metric_node.current.query: + return set() # pragma: no cover + + query = parse(metric_node.current.query) + refs: set[str] = set() + for col in query.select.projection[0].find_all(ast.Column): + full_name = get_column_full_name(col) + if not full_name and hasattr(col, "identifier"): + full_name = col.identifier() + node = ctx.nodes.get(full_name) if full_name else None + if node and node.type == NodeType.METRIC: + refs.add(full_name) + return refs + + +def _metric_parent_refs(ctx: BuildContext, metric_name: str) -> list[str]: + """ + Return metric parents for a metric, using the parsed query as a fallback. + """ + refs: list[str] = [] + seen: set[str] = set() + for parent_name in ctx.parent_map.get(metric_name, []): + parent_node = ctx.nodes.get(parent_name) + if parent_node and parent_node.type == NodeType.METRIC: + refs.append(parent_name) + seen.add(parent_name) + + for parent_name in sorted(_metric_refs_from_query(ctx, metric_name)): + if parent_name not in seen: + refs.append(parent_name) + seen.add(parent_name) + + return refs + + def classify_filters( filters: list[str], ctx: BuildContext, @@ -207,6 +353,31 @@ def get_comp_aggregability(comp_name: str) -> Aggregability: comp = decomposed.components[0] orig_agg = get_comp_aggregability(comp.name) + if comp.rule.reaggregate and comp.name in gg.reaggregate_dimension_aliases: + _, col_name = comp_mappings[comp.name] + value_ref = make_column_ref(col_name, cte_alias) + protected_dim_ref = make_column_ref( + gg.reaggregate_dimension_aliases[comp.name], + cte_alias, + ) + if not comp.merge: # pragma: no cover + raise DJInvalidInputException( + "Unsupported semi-additive metric shape: missing merge function.", + ) + collapse_expr = _build_reaggregate_collapse_expression( + comp.rule.reaggregate.fn, + gg.dialect, + value_ref, + protected_dim_ref, + ) + combiner_ast = parse(f"SELECT {decomposed.combiner}").select.projection[0] + return _replace_reaggregate_merge_expression( + cast(ast.Expression, combiner_ast), + comp.name, + comp.merge, + collapse_expr, + ) + if orig_agg == Aggregability.LIMITED: _, col_name = comp_mappings[comp.name] col_ref = make_column_ref(col_name, cte_alias) @@ -413,6 +584,41 @@ def collect_and_build_ctes( return all_cte_asts, cte_aliases +def _validate_reaggregate_base_group_join_safety( + grain_groups: list[GrainGroupSQL], +) -> None: + """ + Fail closed for final joins that can duplicate non-idempotent aggregates. + + Semi-additive groups retain the protected dimension as private grain, so a + final join by only the requested output dimensions can fan out other CTEs. + LIMITED groups that were pre-aggregated to one row per output dimension use + MAX passthrough in the final select and are safe. Reaggregate groups are + idempotent under duplicate protected-dimension rows for their collapse. + Other separate groups must be rejected unless they were merged into the + protected grain earlier. + """ + if len(grain_groups) <= 1 or not any( + gg.reaggregate_dimension_aliases for gg in grain_groups + ): + return + + unsafe_groups = [ + gg.parent_name + for gg in grain_groups + if not gg.reaggregate_dimension_aliases + and not (gg.aggregability == Aggregability.LIMITED and gg.is_pre_aggregated) + ] + if not unsafe_groups: + return + + raise DJInvalidInputException( + "Semi-additive live SQL with multiple base grain groups is not " + "supported when another group can be fanned out before final " + f"aggregation. Unsafe parents: {sorted(set(unsafe_groups))}.", + ) + + def get_dimension_types( grain_groups: list[GrainGroupSQL], ) -> dict[str, str]: @@ -1230,11 +1436,7 @@ def build_window_agg_cte_from_grain_group( # Find the base metrics that the window metrics reference base_metrics_needed: set[str] = set() for window_metric_name in window_grain_group.window_metrics_served: - parent_names = ctx.parent_map.get(window_metric_name, []) - for parent_name in parent_names: - parent_node = ctx.nodes.get(parent_name) - if parent_node and parent_node.type == NodeType.METRIC: # pragma: no branch - base_metrics_needed.add(parent_name) + base_metrics_needed.update(_metric_parent_refs(ctx, window_metric_name)) def is_derived_metric(metric_name: str) -> bool: """Check if a metric is derived (has metric parents, no direct components).""" @@ -1242,11 +1444,8 @@ def is_derived_metric(metric_name: str) -> bool: if not decomposed or not isinstance(decomposed, DecomposedMetricInfo): return False # pragma: no cover # Derived metrics have parent metrics but no direct components in grain groups - parent_metrics = ctx.parent_map.get(metric_name, []) - has_metric_parents = any( - ctx.nodes.get(p) and ctx.nodes.get(p).type == NodeType.METRIC # type: ignore - for p in parent_metrics - ) + parent_metrics = _metric_parent_refs(ctx, metric_name) + has_metric_parents = bool(parent_metrics) return has_metric_parents and not decomposed.components def get_metric_aggregation_expr( @@ -1284,7 +1483,7 @@ def get_metric_aggregation_expr( # Try to find matching metric parent_metric_name = None - for parent_name in ctx.parent_map.get(metric_name, []): + for parent_name in _metric_parent_refs(ctx, metric_name): parent_short = get_short_name(parent_name) if parent_short == short_col_name or col_name.endswith(parent_name): parent_metric_name = parent_name @@ -1300,21 +1499,11 @@ def get_metric_aggregation_expr( # Swap the column node with the parent's expression col_node.swap(parent_expr) else: - # Base metric: replace component references with CTE column refs - # Use base_grain_group's component_aliases since that's the source CTE - for col_node in combiner_ast.find_all(ast.Column): - col_full_name = ( - col_node.identifier() if hasattr(col_node, "identifier") else "" - ) - # Check if this matches a component alias - for ( - comp_name, - comp_alias, - ) in base_grain_group.component_aliases.items(): # pragma: no branch - if col_full_name == comp_name or col_full_name.endswith(comp_alias): - col_node.name = ast.Name(comp_alias) - col_node._table = ast.Table(ast.Name(source_cte_alias)) - break + combiner_ast, _ = build_base_metric_expression( + decomposed, + source_cte_alias, + base_grain_group, + ) return combiner_ast @@ -1326,13 +1515,13 @@ def get_metric_aggregation_expr( # Get aggregation expression (handles both base and derived metrics) combiner_ast = get_metric_aggregation_expr(base_metric_name, set()) - if not combiner_ast: + if combiner_ast is None: continue # pragma: no cover short_name = get_short_name(base_metric_name) - aliased = combiner_ast.set_alias(ast.Name(short_name)) # type: ignore - aliased.set_as(True) - projection.append(aliased) + metric_alias = ast.Alias(child=combiner_ast, alias=ast.Name(short_name)) + metric_alias.set_as(True) + projection.append(metric_alias) # Build FROM clause from_clause = ast.From( @@ -1394,62 +1583,107 @@ def build_window_agg_cte_from_base_metrics( # Find the base metrics that the window metrics reference base_metrics_needed: set[str] = set() for window_metric_name in window_grain_group.window_metrics_served: - parent_names = ctx.parent_map.get(window_metric_name, []) - for parent_name in parent_names: - parent_node = ctx.nodes.get(parent_name) - if parent_node and parent_node.type == NodeType.METRIC: # pragma: no branch - base_metrics_needed.add(parent_name) - - # Build reaggregation expressions for each base metric - # Since we're selecting from base_metrics, we reference the metric columns directly + base_metrics_needed.update(_metric_parent_refs(ctx, window_metric_name)) + + window_dimension_aliases = { + col.name + for col in window_grain_group.columns + if col.semantic_type == "dimension" + } + + def build_base_metric_reaggregation_expr( + metric_name: str, + decomposed: DecomposedMetricInfo, + ) -> ast.Expression: + """ + Build a reaggregation expression for a base metric from base_metrics. + """ + short_name = get_short_name(metric_name) + metric_ref = make_column_ref(short_name, base_metrics_cte_alias) + + if len(decomposed.components) == 1: + comp = decomposed.components[0] + if comp.rule.reaggregate: + protected_dim_alias = _source_dimension_alias( + ctx, + comp.rule.reaggregate.dimension, + ) + if ( + protected_dim_alias + and protected_dim_alias not in window_dimension_aliases + ): + return _build_reaggregate_collapse_expression( + comp.rule.reaggregate.fn, + ctx.dialect, + metric_ref, + make_column_ref(protected_dim_alias, base_metrics_cte_alias), + ) + + if decomposed.aggregability == Aggregability.NONE: # pragma: no cover + # NONE: non-additive (like AVG), need to recompute. + # For AVG, we need the raw sum and count, but those are in components. + # For now, just use the column directly (window function will handle it). + return metric_ref + + # base_metrics exposes metric columns, not raw component/grain columns. + # Reaggregate leaf metrics from those projected metric values. + return ast.Function(ast.Name("SUM"), args=[metric_ref]) + + def build_metric_reaggregation_expr( + metric_name: str, + visited: set[str], + ) -> ast.Expression | None: + """ + Build a reaggregation expression for base or derived metrics. + """ + if metric_name in visited: + return None # pragma: no cover + visited.add(metric_name) + + parent_refs = _metric_parent_refs(ctx, metric_name) + if not parent_refs: + decomposed = decomposed_metrics.get(metric_name) + if not decomposed: + return None # pragma: no cover + return build_base_metric_reaggregation_expr(metric_name, decomposed) + + metric_node = ctx.nodes.get(metric_name) + if not metric_node: + return None # pragma: no cover + if not metric_node.current or not metric_node.current.query: + return None # pragma: no cover + original_query = parse(metric_node.current.query) + expr_ast = deepcopy(original_query.select.projection[0]) + if isinstance(expr_ast, ast.Alias): + expr_ast = expr_ast.child # pragma: no cover + + for col_node in list(expr_ast.find_all(ast.Column)): + parent_name = get_column_full_name(col_node) + if not parent_name and hasattr(col_node, "identifier"): + parent_name = col_node.identifier() + if not parent_name or parent_name not in parent_refs: + continue + + parent_expr = build_metric_reaggregation_expr( + parent_name, + visited.copy(), + ) + if parent_expr is not None: + col_node.swap(parent_expr) + + wrap_divisions_in_nullif(cast(ast.Expression, expr_ast)) + return expr_ast # type: ignore + + # Build reaggregation expressions for each parent metric. for base_metric_name in sorted(base_metrics_needed): - decomposed = decomposed_metrics.get(base_metric_name) - if not decomposed: + agg_expr = build_metric_reaggregation_expr(base_metric_name, set()) + if agg_expr is None: continue # pragma: no cover short_name = get_short_name(base_metric_name) - - # For non-additive metrics (COUNT DISTINCT), we need COUNT(DISTINCT grain_col) - # For additive metrics, we can SUM/AVG the pre-computed column - if decomposed.aggregability == Aggregability.LIMITED: - # LIMITED: needs COUNT DISTINCT at this grain - # Find the grain column for COUNT DISTINCT - if decomposed.components and decomposed.components[0].rule.level: - grain_col = decomposed.components[0].rule.level[0] - # The grain column is in base_metrics as a dimension column - # Actually, for COUNT DISTINCT, the raw grain column should be in - # the base grain group, not base_metrics. We need to reference - # the original grain column from base_metrics. - # For now, use the metric column - this works because base_metrics - # computes COUNT DISTINCT at the fine grain, and we re-compute at coarser grain - col_ref = make_column_ref(grain_col, base_metrics_cte_alias) - agg_expr = ast.Function( - ast.Name("COUNT"), - args=[col_ref], - quantifier=ast.SetQuantifier.Distinct, - ) - else: # pragma: no cover - # Fallback: SUM the metric column - col_ref = make_column_ref(short_name, base_metrics_cte_alias) - agg_expr = ast.Function(ast.Name("SUM"), args=[col_ref]) - elif decomposed.aggregability == Aggregability.FULL: # pragma: no cover - # FULL: additive metric, use SUM - col_ref = make_column_ref(short_name, base_metrics_cte_alias) - agg_expr = ast.Function(ast.Name("SUM"), args=[col_ref]) - elif decomposed.aggregability == Aggregability.NONE: # pragma: no cover - # NONE: non-additive (like AVG), need to recompute - # For AVG, we need the raw sum and count, but those are in components - # For now, just use the column directly (window function will handle it) - col_ref = make_column_ref(short_name, base_metrics_cte_alias) - agg_expr = col_ref # type: ignore - else: # pragma: no cover - # Default: SUM - col_ref = make_column_ref(short_name, base_metrics_cte_alias) - agg_expr = ast.Function(ast.Name("SUM"), args=[col_ref]) - - aliased = agg_expr.set_alias(ast.Name(short_name)) # type: ignore - aliased.set_as(True) - projection.append(aliased) + metric_alias = ast.Alias(child=agg_expr, alias=ast.Name(short_name)) + metric_alias.set_as(True) + projection.append(metric_alias) # Build FROM clause - just base_metrics from_clause = ast.From( @@ -1760,6 +1994,7 @@ def generate_metrics_sql( base_grain_groups, skip_pre_agg=will_have_base_metrics_cte, ) + _validate_reaggregate_base_group_join_safety(base_grain_groups) # Build dimension info and projection # Filter out filter-only dimensions (they're needed for WHERE but not output) diff --git a/datajunction-server/datajunction_server/construction/build_v3/preagg_matcher.py b/datajunction-server/datajunction_server/construction/build_v3/preagg_matcher.py index 575a22901..4cdcdd311 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/preagg_matcher.py +++ b/datajunction-server/datajunction_server/construction/build_v3/preagg_matcher.py @@ -19,6 +19,8 @@ from datajunction_server.database.preaggregation import ( PreAggregation, compute_expression_hash, + get_measure_identities, + measure_identity_token, ) from datajunction_server.errors import DJInvalidInputException from datajunction_server.models.decompose import Aggregability, MetricComponent @@ -42,10 +44,9 @@ def get_required_measure_identities( grain_group: GrainGroup, -) -> set[tuple[str, str]]: +) -> set[str]: """ - Identity of each measure a grain group needs: (expression hash, Phase-1 - aggregation). + Identity token of each measure a grain group needs. Matching is name-independent, so ``expr_hash`` stands in for identity -- but it covers the expression alone, making ``SUM(x)`` and ``MAX(x)`` hash alike @@ -55,16 +56,40 @@ def get_required_measure_identities( Phase-1 ``aggregation`` rather than ``merge``, because merge is not unique -- COUNT accumulates with COUNT but merges with SUM. + + Built through ``measure_identity_token`` rather than assembled here, so the + definition of identity lives in one place -- it also folds in sketch tuning + parameters, which a locally-built tuple would silently omit. """ return { - ( + measure_identity_token( compute_expression_hash(component.expression), - component.normalized_aggregation, + component.aggregation, + component.params, ) for _, component in grain_group.components } +def required_reaggregate_grain( + ctx: BuildContext, + node_rev_id: int, + grain_group: GrainGroup, +) -> set[str]: + """ + Protected dimensions a pre-agg must retain for final semi-additive collapse. + + ``reaggregate_component_dimensions`` is populated only when the requested + output grain omitted the protected dimension. In that case a pre-agg at the + output grain would have already collapsed away the key needed by MAX_BY / + MIN_BY / min / max in the metrics layer, so it is not substitutable. + """ + return { + canonical_dimension_ref(ctx, node_rev_id, dimension_ref) + for dimension_ref in grain_group.reaggregate_component_dimensions.values() + } + + def canonical_dimension_ref( ctx: BuildContext, node_rev_id: int, @@ -256,6 +281,12 @@ def find_matching_preagg( if not required_measures: return None + reaggregate_grain = required_reaggregate_grain( + ctx, + node_rev_id, + grain_group, + ) + requested_grain_set = { canonical_dimension_ref(ctx, node_rev_id, rdim.original_ref) for rdim in resolved_dimensions @@ -282,6 +313,14 @@ def find_matching_preagg( for ref in (preagg.grain_columns or []) } + if not reaggregate_grain.issubset(preagg_grain_set): + logger.debug( + f"[BuildV3] Pre-agg {preagg.id} grain {preagg_grain_set} " + f"doesn't retain semi-additive protected grain " + f"{reaggregate_grain}", + ) + continue + # Check grain compatibility. For additive measures the pre-agg may be at # the same or a finer grain (roll-up allowed → subset match). For # non-additive measures the pre-agg grain must exactly match the request. @@ -315,13 +354,9 @@ def find_matching_preagg( continue join_back = coverage - # Coverage check on (expression hash, Phase-1 aggregation) -- see - # get_required_measure_identities for why the aggregation is part of it. - preagg_measures = { - (measure.expr_hash, measure.normalized_aggregation) - for measure in preagg.measures - if measure.expr_hash - } + # Coverage check on measure identity -- see + # get_required_measure_identities for what that comprises. + preagg_measures = get_measure_identities(preagg.measures) if not required_measures.issubset(preagg_measures): logger.debug( f"[BuildV3] Pre-agg {preagg.id} measures {preagg_measures} " @@ -373,13 +408,21 @@ def get_preagg_measure_column( Returns: Column name in the pre-agg, or None if not found """ - target = ( + target = measure_identity_token( compute_expression_hash(component.expression), - component.normalized_aggregation, + component.aggregation, + component.params, ) for measure in preagg.measures: - if (measure.expr_hash, measure.normalized_aggregation) == target: + if ( + measure_identity_token( + measure.expr_hash, + measure.aggregation, + measure.params, + ) + == target + ): # Externally-registered pre-aggs bind the measure to a physical # column name that may differ from the DJ component name. return measure.source_column or measure.name diff --git a/datajunction-server/datajunction_server/construction/build_v3/types.py b/datajunction-server/datajunction_server/construction/build_v3/types.py index faac2e34c..4fc4b8612 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/types.py +++ b/datajunction-server/datajunction_server/construction/build_v3/types.py @@ -316,6 +316,10 @@ class GrainGroupSQL: # Used by metrics SQL to correctly reference component columns component_aliases: dict[str, str] = field(default_factory=dict) + # Active semi-additive components that need collapse in the metrics layer. + # Maps component.name -> protected dimension column alias emitted by this CTE. + reaggregate_dimension_aliases: dict[str, str] = field(default_factory=dict) + # Merge tracking: when True, aggregations happen in final SELECT, not in CTE is_merged: bool = False @@ -743,6 +747,10 @@ class GrainGroup: # between decompose.py component identifiers and measures.py SQL aliases. grain_col_aliases: dict[str, str] = field(default_factory=dict) + # Active semi-additive components that require an internal protected dimension + # in this grain group. Maps component.name -> protected dimension ref. + reaggregate_component_dimensions: dict[str, str] = field(default_factory=dict) + # Non-decomposable metrics that couldn't be broken into components # These need their raw metric expression applied in the final SELECT non_decomposable_metrics: list[DecomposedMetricInfo] = field(default_factory=list) diff --git a/datajunction-server/datajunction_server/database/measure.py b/datajunction-server/datajunction_server/database/measure.py index bfdb81c11..32d0bc079 100644 --- a/datajunction-server/datajunction_server/database/measure.py +++ b/datajunction-server/datajunction_server/database/measure.py @@ -119,6 +119,12 @@ class FrozenMeasure(Base): # How to aggregate the resolved expression (e.g., SUM, COUNT, AVG) aggregation: Mapped[str] + # Tuning parameters for `aggregation`, for sketch-backed measures whose + # function name alone doesn't pin down the accumulate (a t-digest still needs + # a compression). Part of the measure's identity: a sketch frozen at one + # accuracy must not be reused for a metric asking for another. + params: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # Additivity or rollup rule - tells the planner if this measure can be summed, # needs special handling, or is non-additive. rule: Mapped[MeasureAggregationRule] = mapped_column(MeasureAggregationRuleType) diff --git a/datajunction-server/datajunction_server/database/node.py b/datajunction-server/datajunction_server/database/node.py index d93acc753..942bc2556 100644 --- a/datajunction-server/datajunction_server/database/node.py +++ b/datajunction-server/datajunction_server/database/node.py @@ -88,6 +88,7 @@ from datajunction_server.models.custom_metadata import CustomMetadataFilter from datajunction_server.models.node_type import NodeType from datajunction_server.models.partition import PartitionType +from datajunction_server.models.reaggregate import parse_reaggregate_spec from datajunction_server.models.unit import ( AtomicUnit, CompoundUnit, @@ -678,6 +679,7 @@ async def to_spec(self, session: AsyncSession) -> NodeSpec: ) extra_kwargs.update( required_dimensions=required_dimensions_spec, + reaggregate=parse_reaggregate_spec(self.current.reaggregate), direction=self.current.metric_metadata.direction if self.current.metric_metadata else None, @@ -1721,6 +1723,14 @@ class NodeRevision( uselist=False, ) + # Declares how a metric should roll up across dimensions. + # Stored as JSON here and validated at the API/deployment boundaries. + reaggregate: Mapped[dict[str, Any] | None] = mapped_column( + JSON, + nullable=True, + default=None, + ) + # Filters that are always applied when generating SQL for a cube node cube_filters: Mapped[list[str] | None] = mapped_column( JSON, @@ -2067,6 +2077,12 @@ def extra_validation(self) -> None: "bound dimensions which are only for metrics.", ) + if self.type != NodeType.METRIC and self.reaggregate: + raise DJInvalidInputException( + f"Node {self.name} of type {self.type} cannot have " + "reaggregate settings which are only for metrics.", + ) + if self.type == NodeType.METRIC: self.check_metric() diff --git a/datajunction-server/datajunction_server/database/preaggregation.py b/datajunction-server/datajunction_server/database/preaggregation.py index 2572ff418..937dd051d 100644 --- a/datajunction-server/datajunction_server/database/preaggregation.py +++ b/datajunction-server/datajunction_server/database/preaggregation.py @@ -6,6 +6,7 @@ import json from datetime import UTC, datetime from functools import partial +from typing import Any import sqlalchemy as sa from sqlalchemy import ( @@ -80,16 +81,64 @@ def compute_grain_group_hash( return hashlib.md5(content.encode()).hexdigest() -def measure_identity_token(expr_hash: str, aggregation: str | None) -> str: +def canonical_params(params: dict[str, Any] | None) -> str: + """ + Stable string form of a component's tuning parameters, for identity. + + Keys are sorted and numeric values normalized, so ``{"compression": 200}`` + and ``{"compression": 200.0}`` cannot yield two identities for one sketch. + """ + if not params: + return "" + + def norm(value: Any) -> Any: + # 200 and 200.0 are the same compression; don't let the literal's + # spelling fork the identity. (bools are not floats, so they pass + # through untouched.) + if isinstance(value, float) and value.is_integer(): + return int(value) + return value + + # `default=str` so an exotic value can't raise from inside pre-agg matching. + # Params arrive as parsed YAML/JSON, so this should be unreachable; an opaque + # TypeError deep in the matcher is a bad way to find out otherwise. + return json.dumps( + {key: norm(params[key]) for key in sorted(params)}, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + + +def measure_identity_token( + expr_hash: str, + aggregation: str | None, + params: dict[str, Any] | None = None, +) -> str: """ Canonical token identifying one measure: expression hash plus Phase-1 - aggregation. + aggregation, plus tuning parameters when the aggregation takes them. The hash alone is not an identity -- ``SUM(x)`` and ``MAX(x)`` hash alike -- and a partial is reusable only by a metric that accumulates the same way. Everything comparing or hashing measures goes through this. + + Parameters extend that reasoning to sketches: a t-digest accumulated at + ``compression=100`` is not interchangeable with one at ``compression=1000``, + though both are ``acme_tdigest_agg`` over the same expression. The segment is + appended **only** when params are present, so tokens for the overwhelming + majority of components -- plain ``SUM``, ``COUNT``, ``MAX`` -- are byte-identical + to what this returned before params existed. Stored ``preagg_hash`` values are + built from these tokens, so that stability is a compatibility requirement, not + a nicety. + + Note the quantile *fractions* a metric asks for are deliberately not here. One + sketch column serves p50, p95 and p99; folding fractions in would fragment it + into three identical columns. """ - return f"{expr_hash}:{(aggregation or '').strip().upper()}" + base = f"{expr_hash}:{(aggregation or '').strip().upper()}" + suffix = canonical_params(params) + return f"{base}:{suffix}" if suffix else base def get_measure_identities(measures: list[PreAggMeasure]) -> set[str]: @@ -103,7 +152,7 @@ def get_measure_identities(measures: list[PreAggMeasure]) -> set[str]: Set of identity tokens """ return { - measure_identity_token(m.expr_hash, m.aggregation) + measure_identity_token(m.expr_hash, m.aggregation, m.params) for m in measures if m.expr_hash } diff --git a/datajunction-server/datajunction_server/internal/client.py b/datajunction-server/datajunction_server/internal/client.py index 7964b4cd2..37c9d24ed 100644 --- a/datajunction-server/datajunction_server/internal/client.py +++ b/datajunction-server/datajunction_server/internal/client.py @@ -192,6 +192,11 @@ async def python_client_create_node( "required_dimensions": [ # type: ignore col.name for col in node.current.required_dimensions ], + **( + {"reaggregate": node.current.reaggregate} # type: ignore + if node.current.reaggregate + else {} + ), **( { "direction": ( # type: ignore diff --git a/datajunction-server/datajunction_server/internal/deployment/orchestrator.py b/datajunction-server/datajunction_server/internal/deployment/orchestrator.py index adbb15ee3..365c712f3 100644 --- a/datajunction-server/datajunction_server/internal/deployment/orchestrator.py +++ b/datajunction-server/datajunction_server/internal/deployment/orchestrator.py @@ -122,6 +122,7 @@ NodeStatus, NodeType, ) +from datajunction_server.models.reaggregate import dump_reaggregate_spec from datajunction_server.models.unit import ( AtomicUnit, CompoundUnit, @@ -5367,6 +5368,9 @@ async def _create_node_revision( dependency_nodes, ) new_revision.required_dimensions = matched_columns + new_revision.reaggregate = dump_reaggregate_spec( + metric_spec.rendered_reaggregate, + ) return new_revision def _resolve_metric_unit( diff --git a/datajunction-server/datajunction_server/internal/deployment/validation.py b/datajunction-server/datajunction_server/internal/deployment/validation.py index d6d65f3bf..92c7369cd 100644 --- a/datajunction-server/datajunction_server/internal/deployment/validation.py +++ b/datajunction-server/datajunction_server/internal/deployment/validation.py @@ -29,6 +29,9 @@ ) from datajunction_server.models.dimensionlink import JoinType from datajunction_server.models.node import NodeStatus, NodeType +from datajunction_server.models.reaggregate import ( + unsupported_dimension_reaggregate_functions, +) from datajunction_server.sql.dag import get_dimensions from datajunction_server.sql.parsing.ast import fast_parse_mode from datajunction_server.sql.parsing.backends.antlr4 import ast, parse_rule @@ -153,7 +156,7 @@ async def validate(self, node_specs: list[NodeSpec]) -> list[NodeValidationResul specs_needing_parse.append(spec) spec_indices.append(i) - # Pre-fetch dimension nodes for required_dimensions validation + # Pre-fetch dimension nodes for metric dimension-reference validation await self._prefetch_required_dimension_nodes(specs_needing_parse) # Pre-fetch dimension sets for cross-fact derived metric validation @@ -520,6 +523,10 @@ def validate_query_node( if req_dim_error is not None: errors.append(req_dim_error) + reaggregate_error = self._check_reaggregate_dimension(spec) + if reaggregate_error is not None: + errors.append(reaggregate_error) + cross_fact_error = self._check_cross_fact_dimensions(spec) if cross_fact_error is not None: errors.append(cross_fact_error) # pragma: no cover @@ -655,9 +662,9 @@ async def _prefetch_required_dimension_nodes( specs: list[NodeSpec], ) -> None: """ - Collect all dimension node names referenced via required_dimensions across the - batch, fetch any not already in dependency_nodes in a single DB query, and - store the combined map in self._all_dim_nodes. + Collect all dimension node names referenced via metric-only dimension + declarations across the batch, fetch any not already in dependency_nodes in a + single DB query, and store the combined map in self._all_dim_nodes. """ req_dim_node_names: set[str] = set() for spec in specs: @@ -667,6 +674,12 @@ async def _prefetch_required_dimension_nodes( if SEPARATOR in req_dim: dim_node_name = req_dim.rsplit(SEPARATOR, 1)[0] req_dim_node_names.add(dim_node_name) + reaggregate = getattr(spec, "rendered_reaggregate", None) + if reaggregate: + for rule in reaggregate.rules: + if SEPARATOR in rule.dimension: + dim_node_name = rule.dimension.rsplit(SEPARATOR, 1)[0] + req_dim_node_names.add(dim_node_name) self._all_dim_nodes = dict(self.context.dependency_nodes) @@ -780,6 +793,54 @@ def _check_required_dimensions(self, spec: NodeSpec) -> DJError | None: debug={"invalid_required_dimensions": list(invalid)}, ) + def _check_reaggregate_dimension(self, spec: NodeSpec) -> DJError | None: + """ + Validate that reaggregate rule dimensions resolve to real columns. + """ + reaggregate = getattr(spec, "rendered_reaggregate", None) + if not reaggregate or not reaggregate.rules: + return None + + invalid_functions = unsupported_dimension_reaggregate_functions(reaggregate) + if invalid_functions: + return DJError( + code=ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION, + message=( + "Node definition contains unsupported dimension " + "reaggregate function(s)." + ), + debug={ + "invalid_reaggregate_functions": invalid_functions, + }, + ) + + dep_names = self.context.node_graph.get(spec.rendered_name, []) + parent_columns = [ + col + for dep_name in dep_names + for dep_node in [self.context.dependency_nodes.get(dep_name)] + if dep_node and dep_node.current + for col in dep_node.current.columns + ] + + invalid, _ = _resolve_required_dimensions( + [rule.dimension for rule in reaggregate.rules], + parent_columns, + self._all_dim_nodes, + ) + + if not invalid: + return None + + return DJError( + code=ErrorCode.INVALID_COLUMN, + message=( + "Node definition contains references to columns as " + "reaggregate dimensions that are not on parent nodes." + ), + debug={"invalid_reaggregate_dimensions": list(invalid)}, + ) + async def _prefetch_metric_dimensions( self, specs: list[NodeSpec], diff --git a/datajunction-server/datajunction_server/internal/namespaces.py b/datajunction-server/datajunction_server/internal/namespaces.py index d3eb5f10b..32bae088f 100644 --- a/datajunction-server/datajunction_server/internal/namespaces.py +++ b/datajunction-server/datajunction_server/internal/namespaces.py @@ -1229,6 +1229,7 @@ def _metric_project_config(node: Node, namespace_requested: str) -> dict: "query": node.current.query, "tags": [tag.name for tag in node.tags], "required_dimensions": [dim.name for dim in node.current.required_dimensions], + "reaggregate": node.current.reaggregate, "direction": ( node.current.metric_metadata.direction.name.lower() if node.current.metric_metadata and node.current.metric_metadata.direction @@ -1642,6 +1643,14 @@ async def get_node_specs_for_export( ) for required_dim in metric_spec.required_dimensions ] + if metric_spec.reaggregate: + for rule in metric_spec.reaggregate.rules: + rule.dimension = _inject_prefix_for_cube_ref( + rule.dimension, + namespace, + parent_namespace, + namespace_suffixes, + ) if node_spec.node_type in ( NodeType.SOURCE, NodeType.TRANSFORM, diff --git a/datajunction-server/datajunction_server/internal/nodes.py b/datajunction-server/datajunction_server/internal/nodes.py index 9f81e399b..0937e5ad8 100644 --- a/datajunction-server/datajunction_server/internal/nodes.py +++ b/datajunction-server/datajunction_server/internal/nodes.py @@ -65,6 +65,7 @@ ) from datajunction_server.internal.access.authorization.context import AuthContext from datajunction_server.internal.caching.interface import Cache +from datajunction_server.internal.custom_metadata import validate_custom_metadata from datajunction_server.internal.history import ActivityType, EntityType from datajunction_server.internal.materializations import ( apply_cube_materialization_swap, @@ -95,6 +96,12 @@ UpsertCubeMaterialization, principal_refs, ) +from datajunction_server.models.decompose import ( + AggregationRule as DecomposeAggregationRule, +) +from datajunction_server.models.decompose import ( + MetricComponent, +) from datajunction_server.models.deployment import ( ChangeTier, CubeSpec, @@ -127,6 +134,7 @@ ) from datajunction_server.models.node_type import NodeType from datajunction_server.models.query import QueryCreate +from datajunction_server.models.reaggregate import dump_reaggregate_spec from datajunction_server.models.table_metadata import TableMetadata, TableOwner from datajunction_server.service_clients import QueryServiceClient from datajunction_server.sql.dag import ( @@ -140,7 +148,6 @@ from datajunction_server.sql.parsing.ast import CompileContext from datajunction_server.sql.parsing.backends.antlr4 import parse, parse_rule from datajunction_server.typing import UTCDatetime -from datajunction_server.internal.custom_metadata import validate_custom_metadata from datajunction_server.utils import ( SEPARATOR, Version, @@ -585,6 +592,11 @@ async def create_node_revision( query=data.query, mode=data.mode, required_dimensions=data.required_dimensions or [], + reaggregate=( + dump_reaggregate_spec(data.reaggregate) + if node_type == NodeType.METRIC + else None + ), created_by_id=current_user.id, custom_metadata=data.custom_metadata, ) @@ -886,12 +898,15 @@ async def _derive_frozen_measures_impl( session=session, name=measure.name, ) + if frozen_measure: + _raise_if_frozen_measure_conflicts(frozen_measure, measure) if not frozen_measure and measure.aggregation: frozen_measure = FrozenMeasure( name=measure.name, upstream_revision_id=upstream_revision_id, expression=measure.expression, aggregation=measure.aggregation, + params=measure.params, rule=measure.rule, used_by_node_revisions=[], ) @@ -1019,14 +1034,51 @@ async def derive_frozen_measures_bulk( upstream_revision_id=upstream_revision_id, expression=measure.expression, aggregation=measure.aggregation, - rule=measure.rule, + params=measure.params, + rule=_frozen_measure_rule(measure.rule), used_by_node_revisions=[], ) session.add(frozen_measure) fm_by_name[measure.name] = frozen_measure + else: + _raise_if_frozen_measure_conflicts(frozen_measure, measure) frozen_measure.used_by_node_revisions.append(rev) +def _aggregation_rule_identity(rule: DecomposeAggregationRule) -> dict[str, Any]: + """Return the stable JSON shape used for frozen-measure rule comparison.""" + return rule.model_dump(mode="json", exclude_none=True, exclude={"reaggregate"}) + + +def _frozen_measure_rule(rule: DecomposeAggregationRule) -> DecomposeAggregationRule: + """ + Return the metric-independent rule persisted on a shared frozen measure. + """ + return rule.model_copy(update={"reaggregate": None}) + + +def _raise_if_frozen_measure_conflicts( + frozen_measure: FrozenMeasure, + measure: MetricComponent, +) -> None: + """ + Prevent component-name collisions from reusing a different frozen measure. + """ + if ( + frozen_measure.expression == measure.expression + and frozen_measure.aggregation == measure.aggregation + and (frozen_measure.params or None) == (measure.params or None) + and _aggregation_rule_identity(frozen_measure.rule) + == _aggregation_rule_identity(measure.rule) + ): + return + + raise DJInvalidInputException( + f"Frozen measure `{measure.name}` already exists with a different " + "expression, aggregation, aggregation rule, or tuning parameters.", + ) + + async def save_node( session: AsyncSession, node_revision: NodeRevision, @@ -1128,6 +1180,7 @@ async def copy_to_new_node( table=old_revision.table, required_dimensions=list(old_revision.required_dimensions), metric_metadata=old_revision.metric_metadata, + reaggregate=old_revision.reaggregate, cube_elements=list(old_revision.cube_elements), cube_filters=old_revision.cube_filters, status=old_revision.status, @@ -1866,6 +1919,7 @@ async def cube_metric_component_identities( FrozenMeasure.name, FrozenMeasure.expression, FrozenMeasure.aggregation, + FrozenMeasure.params, ) .select_from(NodeRevisionFrozenMeasure) .join( @@ -1881,8 +1935,12 @@ async def cube_metric_component_identities( rows = (await session.execute(statement)).all() return { f"{metric_name}:{component_name}:" - + measure_identity_token(compute_expression_hash(expression), aggregation) - for metric_name, component_name, expression, aggregation in rows + + measure_identity_token( + compute_expression_hash(expression), + aggregation, + params, + ) + for metric_name, component_name, expression, aggregation, params in rows } @@ -2505,6 +2563,7 @@ def copy_existing_node_revision(old_revision: NodeRevision, current_user: User): status=old_revision.status, required_dimensions=list(old_revision.required_dimensions), metric_metadata=old_revision.metric_metadata, + reaggregate=old_revision.reaggregate, dimension_links=[ DimensionLink( dimension_id=link.dimension_id, @@ -2727,8 +2786,20 @@ async def create_new_revision_from_existing( and {col.name for col in old_revision.required_dimensions} != set(data.required_dimensions) ) + reaggregate_was_set = bool( + data and "reaggregate" in data.model_fields_set, + ) + reaggregate_changes = ( + reaggregate_was_set + and old_revision.reaggregate + != dump_reaggregate_spec(data.reaggregate if data else None) + ) major_changes = ( - query_changes or column_changes or pk_changes or required_dim_changes + query_changes + or column_changes + or pk_changes + or required_dim_changes + or reaggregate_changes ) # If nothing has changed, do not create the new node revision @@ -2778,6 +2849,13 @@ async def create_new_revision_from_existing( if data and data.metric_metadata else old_revision.metric_metadata ), + reaggregate=( + dump_reaggregate_spec(data.reaggregate) + if data and data.reaggregate is not None + else None + if reaggregate_was_set + else old_revision.reaggregate + ), dimension_links=[ DimensionLink( dimension_id=link.dimension_id, diff --git a/datajunction-server/datajunction_server/internal/preaggregations.py b/datajunction-server/datajunction_server/internal/preaggregations.py index 956053fe6..e1347e9d1 100644 --- a/datajunction-server/datajunction_server/internal/preaggregations.py +++ b/datajunction-server/datajunction_server/internal/preaggregations.py @@ -28,6 +28,7 @@ compute_grain_group_hash, compute_preagg_hash, get_measure_identities, + measure_identity_token, ) from datajunction_server.errors import DJInvalidInputException from datajunction_server.models.decompose import PreAggMeasure @@ -195,7 +196,7 @@ async def register_external_preaggregations( # identity -- (expression hash, Phase-1 aggregation) -- to the declared # column. Keying on the hash alone would collapse SUM(x) and MAX(x), # silently discarding one metric's declared column. - measure_identity_to_column: dict[tuple[str, str], str] = {} + measure_identity_to_column: dict[str, str] = {} for metric_name, physical_column in measure_columns.items(): node = await Node.get_by_name( session, @@ -224,9 +225,10 @@ async def register_external_preaggregations( # is_measure guarantees exactly one component. component = components[0] measure_identity_to_column[ - ( + measure_identity_token( compute_expression_hash(component.expression), - component.normalized_aggregation, + component.aggregation, + component.params, ) ] = physical_column @@ -311,7 +313,11 @@ async def register_external_preaggregations( grain_measures: list[PreAggMeasure] = [] for component in grain_group.components: expr_hash = compute_expression_hash(component.expression) - identity = (expr_hash, component.normalized_aggregation) + identity = measure_identity_token( + expr_hash, + component.aggregation, + component.params, + ) if identity not in measure_identity_to_column: raise DJInvalidInputException( message=( diff --git a/datajunction-server/datajunction_server/internal/sql.py b/datajunction-server/datajunction_server/internal/sql.py index 26689d09e..97644851e 100644 --- a/datajunction-server/datajunction_server/internal/sql.py +++ b/datajunction-server/datajunction_server/internal/sql.py @@ -242,6 +242,7 @@ async def generate_metrics_sql( matched_cube, dimensions, merged_filters, + metrics, ) # Auto-resolve dialect if not explicitly provided. diff --git a/datajunction-server/datajunction_server/internal/validation.py b/datajunction-server/datajunction_server/internal/validation.py index 6eb1af905..55df63941 100644 --- a/datajunction-server/datajunction_server/internal/validation.py +++ b/datajunction-server/datajunction_server/internal/validation.py @@ -23,6 +23,10 @@ from datajunction_server.models.base import labelize from datajunction_server.models.node import NodeRevisionBase, NodeStatus from datajunction_server.models.node_type import NodeType +from datajunction_server.models.reaggregate import ( + parse_reaggregate_spec, + unsupported_dimension_reaggregate_functions, +) from datajunction_server.sql.parsing import ast from datajunction_server.sql.parsing.backends.antlr4 import SqlSyntaxError, parse from datajunction_server.sql.parsing.backends.exceptions import DJParseException @@ -357,11 +361,34 @@ async def validate_node_data( parent_columns, ) node_validator.required_dimensions = matched_bound_columns + reaggregate_spec = parse_reaggregate_spec(validated_node.reaggregate) + invalid_reaggregate_dimensions: set[str] = set() + invalid_reaggregate_functions: list[str] = [] + if reaggregate_spec and reaggregate_spec.rules: + ( + invalid_reaggregate_dimensions, + _, + ) = await find_required_dimensions( + session, + [rule.dimension for rule in reaggregate_spec.rules], + parent_columns, + ) + invalid_reaggregate_functions = unsupported_dimension_reaggregate_functions( + reaggregate_spec, + ) except MissingGreenlet: invalid_required_dimensions = set() + invalid_reaggregate_dimensions = set() + invalid_reaggregate_functions = [] node_validator.required_dimensions = [] - if missing_parents_map or type_inference_failures or invalid_required_dimensions: + if ( + missing_parents_map + or type_inference_failures + or invalid_required_dimensions + or invalid_reaggregate_dimensions + or invalid_reaggregate_functions + ): # update status node_validator.status = NodeStatus.INVALID # build errors @@ -410,10 +437,46 @@ async def validate_node_data( if invalid_required_dimensions else [] ) + invalid_reaggregate_dimensions_error = ( + [ + DJError( + code=ErrorCode.INVALID_COLUMN, + message=( + "Node definition contains references to columns as " + "reaggregate dimensions that are not on parent nodes." + ), + debug={ + "invalid_reaggregate_dimensions": list( + invalid_reaggregate_dimensions, + ), + }, + ), + ] + if invalid_reaggregate_dimensions + else [] + ) + invalid_reaggregate_functions_error = ( + [ + DJError( + code=ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION, + message=( + "Node definition contains unsupported dimension " + "reaggregate function(s)." + ), + debug={ + "invalid_reaggregate_functions": invalid_reaggregate_functions, + }, + ), + ] + if invalid_reaggregate_functions + else [] + ) errors = ( missing_parents_error + type_inference_error + invalid_required_dimensions_error + + invalid_reaggregate_dimensions_error + + invalid_reaggregate_functions_error ) node_validator.errors.extend(errors) @@ -705,10 +768,30 @@ async def validate_node_data_v2( parent_columns, ) node_validator.required_dimensions = matched_bound_columns + reaggregate_spec = parse_reaggregate_spec(validated_node.reaggregate) + invalid_reaggregate_dimensions: set[str] = set() + invalid_reaggregate_functions: list[str] = [] + if reaggregate_spec and reaggregate_spec.rules: + ( + invalid_reaggregate_dimensions, + _, + ) = await find_required_dimensions( + session, + [rule.dimension for rule in reaggregate_spec.rules], + parent_columns, + ) + invalid_reaggregate_functions = unsupported_dimension_reaggregate_functions( + reaggregate_spec, + ) # --- Step 12: final error assembly for missing parents + invalid required # dims (matches legacy code shapes). - if node_validator.missing_parents_map or invalid_required_dimensions: + if ( + node_validator.missing_parents_map + or invalid_required_dimensions + or invalid_reaggregate_dimensions + or invalid_reaggregate_functions + ): node_validator.status = NodeStatus.INVALID if node_validator.missing_parents_map: node_validator.errors.append( @@ -740,6 +823,34 @@ async def validate_node_data_v2( }, ), ) + if invalid_reaggregate_dimensions: + node_validator.errors.append( + DJError( + code=ErrorCode.INVALID_COLUMN, + message=( + "Node definition contains references to columns as " + "reaggregate dimensions that are not on parent nodes." + ), + debug={ + "invalid_reaggregate_dimensions": list( + invalid_reaggregate_dimensions, + ), + }, + ), + ) + if invalid_reaggregate_functions: + node_validator.errors.append( + DJError( + code=ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION, + message=( + "Node definition contains unsupported dimension " + "reaggregate function(s)." + ), + debug={ + "invalid_reaggregate_functions": invalid_reaggregate_functions, + }, + ), + ) return node_validator diff --git a/datajunction-server/datajunction_server/models/cube_materialization.py b/datajunction-server/datajunction_server/models/cube_materialization.py index 1e12ebb40..23abfbd2b 100644 --- a/datajunction-server/datajunction_server/models/cube_materialization.py +++ b/datajunction-server/datajunction_server/models/cube_materialization.py @@ -16,11 +16,11 @@ ) from datajunction_server.models.materialization import ( DEFAULT_CUBE_RETENTION, - DRUID_AGG_MAPPING, CoverageSpec, MaterializationJobTypeEnum, MaterializationStrategy, SparkSpec, + get_druid_aggregator_spec, ) from datajunction_server.models.node_type import NodeNameVersion from datajunction_server.models.partition import Granularity @@ -385,22 +385,19 @@ def metrics_spec(self) -> list[dict[str, Any]]: Returns the Druid metrics spec for ingestion """ column_mapping = {col.name: col.type for col in self.columns} # type: ignore - return [ - { - "fieldName": measure.name, - "name": measure.name, - "type": DRUID_AGG_MAPPING[ - (column_mapping[measure.name], measure.aggregation.lower()) - ], - } - for measure in self.measures - if measure.aggregation - and ( - column_mapping.get(measure.name), - measure.aggregation.lower(), + specs = ( + get_druid_aggregator_spec( + column_name=measure.name, + column_type=column_mapping.get(measure.name), + aggregation=measure.aggregation, + merge=measure.merge, + params=measure.params, ) - in DRUID_AGG_MAPPING - ] + for measure in self.measures + ) + # Unmappable measures are omitted, as they always have been here -- the + # cube API substitutes longSum instead. See get_druid_aggregator_spec. + return [spec for spec in specs if spec is not None] @computed_field # type: ignore[misc] @property diff --git a/datajunction-server/datajunction_server/models/decompose.py b/datajunction-server/datajunction_server/models/decompose.py index 5e5655838..8eb1f606d 100644 --- a/datajunction-server/datajunction_server/models/decompose.py +++ b/datajunction-server/datajunction_server/models/decompose.py @@ -9,9 +9,12 @@ - DecomposedMetric: A metric broken into components + combiner expression """ +from typing import Any + from pydantic import BaseModel from datajunction_server.enum import StrEnum +from datajunction_server.models.reaggregate import DimensionReaggregateRule class Aggregability(StrEnum): @@ -54,6 +57,7 @@ class AggregationRule(BaseModel): type: Aggregability = Aggregability.NONE level: list[str] | None = None + reaggregate: DimensionReaggregateRule | None = None class MetricComponent(BaseModel): @@ -96,6 +100,12 @@ class MetricComponent(BaseModel): # ("order_id"); for complex expressions it is component.name so the identifier stays # valid and consistent with what decompose.py computed. grain_alias: str | None = None + # Tuning parameters for `aggregation`/`merge`, for sketch-backed components + # whose function name alone doesn't pin down the accumulate (a t-digest still + # needs a compression, a KLL a `k`). Part of measure identity -- see + # `measure_identity_token` -- because a sketch built at one accuracy must not + # satisfy a query asking for another. + params: dict[str, Any] | None = None @property def normalized_aggregation(self) -> str: diff --git a/datajunction-server/datajunction_server/models/deployment.py b/datajunction-server/datajunction_server/models/deployment.py index 1336b6aa7..ecf529394 100644 --- a/datajunction-server/datajunction_server/models/deployment.py +++ b/datajunction-server/datajunction_server/models/deployment.py @@ -40,6 +40,7 @@ NodeType, ) from datajunction_server.models.partition import Granularity, PartitionType +from datajunction_server.models.reaggregate import ReaggregateSpec from datajunction_server.models.unit import ( Unit, legacy_unit_to_structured, @@ -1022,6 +1023,7 @@ class MetricSpec(NodeSpec): # Excluded from serialization so it's never exported. columns: list[ColumnSpec] | None = Field(default=None, exclude=True) required_dimensions: list[str] | None = None # Field(default_factory=list) + reaggregate: ReaggregateSpec | None = None direction: MetricDirection | None = None unit_enum: MetricUnit | None = Field(default=None, exclude=True) # Structured unit form at the metric level — peer of `unit_enum`. @@ -1038,6 +1040,7 @@ class MetricSpec(NodeSpec): "columns": ChangeTier.NONE, # Required dimensions constrain which queries the metric can answer. "required_dimensions": ChangeTier.MAJOR, + "reaggregate": ChangeTier.MAJOR, # Everything below is presentation metadata on the metric's single output # column -- the same set the PATCH path already treats as minor via # `metric_metadata` in `create_new_revision_from_existing`. @@ -1179,6 +1182,29 @@ def diff(self, other: "NodeSpec") -> list[str]: changed.remove("required_dimensions") return changed + @property + def rendered_reaggregate(self) -> ReaggregateSpec | None: + """ + Reaggregate spec with `${prefix}` resolved to this spec's namespace. + """ + if not self.reaggregate: + return None + rules = [ + rule.model_copy( + update={ + "dimension": render_prefixes(rule.dimension, self.namespace) + if "${prefix}" in rule.dimension + else rule.dimension, + }, + ) + for rule in self.reaggregate.rules + ] + return self.reaggregate.model_copy( + update={ + "rules": rules, + }, + ) + def model_dump(self, **kwargs): # pragma: no cover base = super().model_dump(**kwargs) base["unit"] = self.unit @@ -1218,6 +1244,7 @@ def __eq__(self, other: object) -> bool: other.canonical_required_dimensions, preserve_order=False, ) + and self.rendered_reaggregate == other.rendered_reaggregate and eq_or_fallback(self.direction, other.direction, MetricDirection.NEUTRAL) and self._normalized_unit() == other._normalized_unit() and self.significant_digits == other.significant_digits diff --git a/datajunction-server/datajunction_server/models/materialization.py b/datajunction-server/datajunction_server/models/materialization.py index 421e6269c..e4dc53c6f 100644 --- a/datajunction-server/datajunction_server/models/materialization.py +++ b/datajunction-server/datajunction_server/models/materialization.py @@ -54,8 +54,124 @@ ("binary", "hll_sketch_agg"): "HLLSketchMerge", } -# Aggregation types that need special handling (extra config parameters) -DRUID_SKETCH_TYPES = {"HLLSketchMerge"} +# Aggregator types that carry extra config in the Druid metricsSpec, mapped to +# their default configuration. A sketch aggregator is not fully specified by its +# type: an HLL needs a precision, a t-digest a compression, a KLL a `k`. Defaults +# apply when a metric declares no `reaggregate.params`. +DRUID_SKETCH_CONFIG: dict[str, dict[str, Any]] = { + "HLLSketchMerge": { + "lgK": 12, # Log2 of K, controls precision (4-21) + "tgtHllType": "HLL_4", # HLL_4, HLL_6, or HLL_8 + }, +} + + +def register_druid_aggregator( + column_type: str, + merge_func: str, + aggregator: str, + default_config: dict[str, Any] | None = None, +) -> None: + """ + Register a Druid ingestion aggregator for a (column type, merge function) pair. + + Args: + column_type: Measures-table column type, e.g. "binary" + merge_func: Phase-2 merge function name, e.g. "acme_tdigest_agg" + aggregator: Druid aggregator type, e.g. "tDigestSketch" + default_config: Extra metricsSpec keys and their defaults, e.g. + ``{"compression": 200}``. Marks the aggregator as parameterized. + + Raises: + DJInvalidInputException: If the pair is registered to a different aggregator, + or the aggregator has different defaults. Re-registering identical defaults + is a no-op. + """ + key = (column_type, merge_func.lower()) + existing = DRUID_AGG_MAPPING.get(key) + if existing is not None and existing != aggregator: + raise DJInvalidInputException( + message=( + f"Druid aggregator for {key} is already registered as " + f"`{existing}`; refusing to replace it with `{aggregator}`." + ), + ) + new_config = dict(default_config) if default_config else None + existing_config = DRUID_SKETCH_CONFIG.get(aggregator) + if ( + existing_config is not None + and new_config is not None + and existing_config != new_config + ): + raise DJInvalidInputException( + message=( + f"Druid aggregator `{aggregator}` is already registered with " + f"different defaults ({existing_config}); refusing to replace " + f"them with {new_config}." + ), + ) + DRUID_AGG_MAPPING[key] = aggregator + if new_config: + DRUID_SKETCH_CONFIG[aggregator] = new_config + + +def get_druid_aggregator_spec( + column_name: str, + column_type: str | None, + aggregation: str | None, + merge: str | None, + params: dict[str, Any] | None = None, +) -> dict[str, Any] | None: + """ + Build a Druid metricsSpec entry for a measure column, or None if unmappable. + + The `merge` function takes precedence over `aggregation` since ingestion + operates on pre-aggregated partials. + + Returns None for unmappable measures, allowing callers to determine fallback behavior. + + Raises: + DJInvalidInputException: If `params` contains a key not declared by the aggregator. + """ + agg_func = merge or aggregation + if not agg_func or column_type is None: + return None + + aggregator = DRUID_AGG_MAPPING.get((column_type, agg_func.lower())) + if aggregator is None: + return None + + family_config = DRUID_SKETCH_CONFIG.get(aggregator) + if params: + # Restrict `params` to declared tuning knobs to prevent overwriting + # structural keys (like `type` or `fieldName`) and catch typos. + allowed = set(family_config or ()) + unknown = sorted(set(params) - allowed) + if unknown: + raise DJInvalidInputException( + message=( + f"Druid aggregator `{aggregator}` does not accept " + f"{', '.join(f'`{key}`' for key in unknown)}. " + + ( + f"Supported: {', '.join(sorted(allowed))}." + if allowed + else "It takes no parameters." + ) + ), + ) + + spec: dict[str, Any] = { + "fieldName": column_name, + "name": column_name, + "type": aggregator, + } + if family_config: + # Apply metric-specific params over family defaults. + spec.update(family_config) + if params: + spec.update(params) + return spec + # How long ingested cube data is kept in Druid. Without an explicit rule a datasource # inherits the cluster default, which is unrelated to the span the cube holds, so an @@ -458,16 +574,18 @@ def metrics_spec(self) -> dict: for measure in measure_group.measures if (measure.type.lower(), measure.agg.lower()) not in DRUID_AGG_MAPPING ] - return { - measure.name: { - "fieldName": measure.field_name, - "name": measure.field_name, - "type": DRUID_AGG_MAPPING[(measure.type.lower(), measure.agg.lower())], - } + # The V2 `Measure` model lacks a merge phase, so `agg` is used for lookup. + specs = { + measure.name: get_druid_aggregator_spec( + column_name=measure.field_name, + column_type=measure.type.lower(), + aggregation=measure.agg, + merge=None, + ) for measure_group in self.measures.values() # type: ignore for measure in measure_group.measures - if (measure.type.lower(), measure.agg.lower()) in DRUID_AGG_MAPPING } + return {name: spec for name, spec in specs.items() if spec is not None} def build_druid_spec(self, node_revision: "NodeRevision"): """ diff --git a/datajunction-server/datajunction_server/models/metric.py b/datajunction-server/datajunction_server/models/metric.py index 2ec7e50b6..43813ccec 100644 --- a/datajunction-server/datajunction_server/models/metric.py +++ b/datajunction-server/datajunction_server/models/metric.py @@ -16,6 +16,7 @@ MetricMetadataOutput, ) from datajunction_server.models.query import ColumnMetadata, V3ColumnMetadata +from datajunction_server.models.reaggregate import ReaggregateSpec from datajunction_server.models.sql import ScanEstimate, TranspiledSQL from datajunction_server.models.unit import unit_to_dict from datajunction_server.sql.decompose import MetricComponentExtractor @@ -50,6 +51,7 @@ class Metric(BaseModel): # going forward. `None` when no unit is set, regardless of input shape. unit: dict | None = None required_dimensions: list[str] + reaggregate: ReaggregateSpec | None = None # Whether the metric is a single aggregation call (a "measure") that can map # 1:1 to a column in an externally-built pre-aggregation table. Derived/ratio @@ -104,6 +106,7 @@ async def parse_node( node.current.columns[0].unit if node.current.columns else None, ), required_dimensions=[dim.name for dim in node.current.required_dimensions], + reaggregate=node.current.reaggregate, is_measure=node.current.is_measure, incompatible_druid_functions=incompatible_druid_functions, measures=measures, diff --git a/datajunction-server/datajunction_server/models/node.py b/datajunction-server/datajunction_server/models/node.py index e60ff6b67..13da677a4 100644 --- a/datajunction-server/datajunction_server/models/node.py +++ b/datajunction-server/datajunction_server/models/node.py @@ -33,6 +33,7 @@ from datajunction_server.models.materialization import MaterializationConfigOutput from datajunction_server.models.node_type import NodeNameOutput, NodeType from datajunction_server.models.partition import PartitionOutput +from datajunction_server.models.reaggregate import ReaggregateSpec from datajunction_server.models.tag import TagMinimum, TagOutput from datajunction_server.models.unit import ( AtomicUnit, @@ -878,6 +879,7 @@ class MetricNodeFields(BaseModel): """ required_dimensions: list[str] | None = None + reaggregate: ReaggregateSpec | None = None metric_metadata: MetricMetadataInput | None = None @@ -1033,6 +1035,7 @@ class NodeRevisionOutput(BaseModel): materializations: list[MaterializationConfigOutput] parents: list[NodeNameOutput] metric_metadata: MetricMetadataOutput | None = None + reaggregate: ReaggregateSpec | None = None dimension_links: list[LinkDimensionOutput] | None = None custom_metadata: dict | None = None @@ -1065,6 +1068,7 @@ class NodeOutput(GenericNodeOutputModel): materializations: list[MaterializationConfigOutput] parents: list[NodeNameOutput] metric_metadata: MetricMetadataOutput | None = None + reaggregate: ReaggregateSpec | None = None dimension_links: list[LinkDimensionOutput] = Field(default_factory=list) created_at: UTCDatetime created_by: UserNameOnly | None = None diff --git a/datajunction-server/datajunction_server/models/reaggregate.py b/datajunction-server/datajunction_server/models/reaggregate.py new file mode 100644 index 000000000..49f781dc0 --- /dev/null +++ b/datajunction-server/datajunction_server/models/reaggregate.py @@ -0,0 +1,144 @@ +"""Models for metric reaggregation declarations.""" + +from typing import Any + +from pydantic import BaseModel, Field, model_validator + +from datajunction_server.enum import StrEnum + + +class ReaggregationFunction(StrEnum): + """ + Supported functions for metric reaggregation. + """ + + AUTO = "auto" + NONE = "none" + SUM = "sum" + AVG = "avg" + WEIGHTED_AVG = "weighted_avg" + LAST_VALUE = "last_value" + FIRST_VALUE = "first_value" + MIN = "min" + MAX = "max" + + +DIMENSION_REAGGREGATE_FUNCTIONS = frozenset( + { + ReaggregationFunction.LAST_VALUE, + ReaggregationFunction.FIRST_VALUE, + ReaggregationFunction.MIN, + ReaggregationFunction.MAX, + }, +) + + +PARAMETERIZED_REAGGREGATE_FUNCTIONS: frozenset[ReaggregationFunction] = frozenset() + + +def is_parameterized_reaggregate_function( + function: ReaggregationFunction | None, +) -> bool: + """ + Return whether a function accepts tuning parameters in `params`. + """ + return function in PARAMETERIZED_REAGGREGATE_FUNCTIONS + + +def is_supported_dimension_reaggregate_function( + function: ReaggregationFunction, +) -> bool: + """ + Return whether a function can collapse across a protected dimension. + """ + return function in DIMENSION_REAGGREGATE_FUNCTIONS + + +def unsupported_dimension_reaggregate_functions( + spec: "ReaggregateSpec | dict | None", +) -> list[str]: + """ + Unsupported dimension-specific collapse functions in a reaggregate spec. + """ + reaggregate = parse_reaggregate_spec(spec) + if not reaggregate: + return [] + return sorted( + { + rule.fn.value + for rule in reaggregate.rules + if not is_supported_dimension_reaggregate_function(rule.fn) + }, + ) + + +class DimensionReaggregateRule(BaseModel): + """ + Dimension-specific metric reaggregation rule. + """ + + dimension: str + fn: ReaggregationFunction + + +class ReaggregateSpec(BaseModel): + """ + Declaration for how a metric rolls up from its accumulation grain. + """ + + fn: ReaggregationFunction | None = None + weight: str | None = None + rules: list[DimensionReaggregateRule] = Field(default_factory=list) + + # Tuning parameters for `fn` (e.g., compression for t-digest). + # Rejected for functions that do not accept parameters. + params: dict[str, Any] | None = None + + @model_validator(mode="after") + def check_params_supported(self) -> "ReaggregateSpec": + """ + Reject `params` for unparameterized functions. + """ + if self.params and not is_parameterized_reaggregate_function(self.fn): + named = f"`{self.fn.value}`" if self.fn else "no reaggregation function" + raise ValueError( + f"Reaggregation function {named} does not accept parameters, " + f"but got: {', '.join(sorted(self.params))}.", + ) + return self + + +def dump_reaggregate_spec( + spec: ReaggregateSpec | dict | None, +) -> dict | None: + """ + Return a JSON-serializable reaggregate spec dictionary. + """ + if spec is None: + return None + if isinstance(spec, ReaggregateSpec): + return spec.model_dump(mode="json") + return ReaggregateSpec.model_validate(spec).model_dump(mode="json") + + +def parse_reaggregate_spec( + spec: ReaggregateSpec | dict | None, +) -> ReaggregateSpec | None: + """ + Return a validated reaggregate spec model. + """ + if spec is None: + return None + if isinstance(spec, ReaggregateSpec): + return spec + return ReaggregateSpec.model_validate(spec) + + +def dimension_reaggregate_rules( + spec: ReaggregateSpec | dict | None, +) -> list[DimensionReaggregateRule]: + """ + Return the dimension-specific reaggregation rules from a spec. + """ + reaggregate = parse_reaggregate_spec(spec) + return reaggregate.rules if reaggregate else [] diff --git a/datajunction-server/datajunction_server/semantic_fingerprints/v1.py b/datajunction-server/datajunction_server/semantic_fingerprints/v1.py index 08cc05e83..d40f99c16 100644 --- a/datajunction-server/datajunction_server/semantic_fingerprints/v1.py +++ b/datajunction-server/datajunction_server/semantic_fingerprints/v1.py @@ -45,7 +45,7 @@ "primary_key", "query", ), - MetricSpec: ("query", "required_dimensions"), + MetricSpec: ("query", "required_dimensions", "reaggregate"), CubeSpec: ("metrics", "dimensions", "filters", "columns"), } diff --git a/datajunction-server/datajunction_server/sql/decompose.py b/datajunction-server/datajunction_server/sql/decompose.py index 3003ad291..eb4fd2f82 100644 --- a/datajunction-server/datajunction_server/sql/decompose.py +++ b/datajunction-server/datajunction_server/sql/decompose.py @@ -10,12 +10,19 @@ from sqlalchemy.orm import aliased from datajunction_server.database.node import Node, NodeRelationship, NodeRevision +from datajunction_server.errors import DJInvalidInputException from datajunction_server.models.decompose import ( Aggregability, AggregationRule, MetricComponent, ) from datajunction_server.models.node_type import NodeType +from datajunction_server.models.reaggregate import ( + ReaggregateSpec, + dimension_reaggregate_rules, + is_supported_dimension_reaggregate_function, + parse_reaggregate_spec, +) from datajunction_server.naming import amenable_col_names from datajunction_server.sql import functions as dj_functions from datajunction_server.sql.parsing.backends.antlr4 import ast, parse @@ -746,6 +753,7 @@ class BaseMetricData: name: str query: str + reaggregate: ReaggregateSpec | dict | None = None @dataclass @@ -930,7 +938,10 @@ async def extract( else: # True base metric - decompose aggregations base_ast = parse(base_metric.query) - base_components, derived_ast = self._extract_base(base_ast) + base_components, derived_ast = self._extract_base( + base_ast, + parse_reaggregate_spec(base_metric.reaggregate), + ) for comp in base_components: if comp.name not in components_tracker: @@ -984,6 +995,12 @@ def _build_metric_data_from_cache( ] if metric_parents: + if metric_node.current.reaggregate: + raise DJInvalidInputException( + "Semi-additive metrics must be base metrics in V1. " + f"Derived metric `{metric_node.name}` declares reaggregate.", + ) + # Derived metric - base metrics are the parent metrics return MetricData( query=metric_node.current.query, @@ -992,6 +1009,7 @@ def _build_metric_data_from_cache( BaseMetricData( name=parent_name, query=nodes_cache[parent_name].current.query, + reaggregate=nodes_cache[parent_name].current.reaggregate, ) for parent_name in metric_parents if nodes_cache[parent_name].current @@ -1007,6 +1025,7 @@ def _build_metric_data_from_cache( BaseMetricData( name=metric_node.name, query=metric_node.current.query, + reaggregate=metric_node.current.reaggregate, ), ], ) @@ -1023,6 +1042,7 @@ async def _load_metric_data(self, session: AsyncSession) -> MetricData: select( Node.name.label("parent_name"), NodeRevision.query.label("parent_query"), + NodeRevision.reaggregate.label("parent_reaggregate"), ) .select_from(NodeRelationship) .join(Node, NodeRelationship.parent_id == Node.id) @@ -1043,6 +1063,7 @@ async def _load_metric_data(self, session: AsyncSession) -> MetricData: this_metric_stmt = ( select( NodeRevision.query, + NodeRevision.reaggregate, Node.name, ) .join(Node, NodeRevision.node_id == Node.id) @@ -1055,12 +1076,22 @@ async def _load_metric_data(self, session: AsyncSession) -> MetricData: this_row = this_result.one() if parent_rows: + if this_row.reaggregate: + raise DJInvalidInputException( + "Semi-additive metrics must be base metrics in V1. " + f"Derived metric `{this_row.name}` declares reaggregate.", + ) + # Derived metric - base metrics are the parents return MetricData( query=this_row.query, is_derived=True, base_metrics=[ - BaseMetricData(name=row.parent_name, query=row.parent_query) + BaseMetricData( + name=row.parent_name, + query=row.parent_query, + reaggregate=row.parent_reaggregate, + ) for row in parent_rows ], ) @@ -1069,12 +1100,19 @@ async def _load_metric_data(self, session: AsyncSession) -> MetricData: return MetricData( query=this_row.query, is_derived=False, - base_metrics=[BaseMetricData(name=this_row.name, query=this_row.query)], + base_metrics=[ + BaseMetricData( + name=this_row.name, + query=this_row.query, + reaggregate=this_row.reaggregate, + ), + ], ) def _extract_base( self, query_ast: ast.Query, + reaggregate: ReaggregateSpec | None = None, ) -> tuple[list[MetricComponent], ast.Query]: """ Extract components from a base metric by decomposing aggregations. @@ -1106,6 +1144,11 @@ def _extract_base( # If any aggregation is non-decomposable, abort decomposition # entirely — the metric is non-decomposable as a whole. if any(get_decomposition(dj_fn) is None for _, dj_fn in agg_funcs): + if dimension_reaggregate_rules(reaggregate): + self._raise_unsupported_reaggregate_shape( + "dimension-specific reaggregation requires a " + "decomposable aggregation", + ) return [], query_ast for func, dj_function in agg_funcs: @@ -1128,8 +1171,78 @@ def _extract_base( for proj in query_ast.select.projection: wrap_divisions_in_nullif(cast(ast.Expression, proj)) + if reaggregate is not None and dimension_reaggregate_rules(reaggregate): + self._attach_reaggregate_spec(components, reaggregate) + + if reaggregate is not None and reaggregate.params: + self._attach_reaggregate_params(components, reaggregate) + return components, query_ast + def _attach_reaggregate_params( + self, + components: list[MetricComponent], + reaggregate: ReaggregateSpec, + ) -> None: + """ + Propagate tuning parameters from the reaggregate spec to aggregating components. + """ + configurable = [ + component + for component in components + if component.aggregation is not None and component.merge is not None + ] + if not configurable: + self._raise_unsupported_reaggregate_shape( + "parameterized reaggregation requires an aggregating component", + ) + for component in configurable: + component.params = dict(reaggregate.params or {}) + + def _attach_reaggregate_spec( + self, + components: list[MetricComponent], + reaggregate: ReaggregateSpec, + ) -> None: + """ + Attach a dimension-specific reaggregation rule to one ordinary measure. + """ + dimension_rules = dimension_reaggregate_rules(reaggregate) + if len(dimension_rules) != 1: + self._raise_unsupported_reaggregate_shape( + "dimension-specific reaggregation requires exactly one rule", + ) + + if not is_supported_dimension_reaggregate_function(dimension_rules[0].fn): + self._raise_unsupported_reaggregate_shape( + f"unsupported dimension reaggregation function " + f"`{dimension_rules[0].fn.value}`", + ) + + if len(components) != 1: + self._raise_unsupported_reaggregate_shape( + "dimension-specific reaggregation requires exactly one component", + ) + + component = components[0] + if ( + component.rule.type != Aggregability.FULL + or component.aggregation is None + or component.merge is None + ): + self._raise_unsupported_reaggregate_shape( + "dimension-specific reaggregation requires one non-distinct " + "fully-aggregatable component", + ) + + component.rule.reaggregate = dimension_rules[0] + + @staticmethod + def _raise_unsupported_reaggregate_shape(reason: str) -> None: + raise DJInvalidInputException( + f"Unsupported reaggregate metric shape: {reason}.", + ) + def _substitute_metric_references( self, query_ast: ast.Query, diff --git a/datajunction-server/datajunction_server/sql/functions.py b/datajunction-server/datajunction_server/sql/functions.py index b194d46e4..87c6891ae 100644 --- a/datajunction-server/datajunction_server/sql/functions.py +++ b/datajunction-server/datajunction_server/sql/functions.py @@ -552,16 +552,16 @@ class ApproxPercentile(Function): def infer_type( col: ct.NumberType, percentage: ct.ListType, - accuracy: ct.NumberType | None, -) -> ct.DoubleType: + accuracy: ct.NumberType | None = None, +) -> ct.ListType: return ct.ListType(element_type=col.type) # type: ignore @ApproxPercentile.register def infer_type( col: ct.NumberType, - percentage: ct.FloatType, - accuracy: ct.NumberType | None, + percentage: ct.FloatingBase, + accuracy: ct.NumberType | None = None, ) -> ct.NumberType: return col.type # type: ignore diff --git a/datajunction-server/tests/api/cubes_build_metrics_spec_test.py b/datajunction-server/tests/api/cubes_build_metrics_spec_test.py index 64c14bb34..e196fa151 100644 --- a/datajunction-server/tests/api/cubes_build_metrics_spec_test.py +++ b/datajunction-server/tests/api/cubes_build_metrics_spec_test.py @@ -4,7 +4,19 @@ This function builds Druid metricsSpec from measure columns and their components. """ +import pytest + from datajunction_server.api.cubes import _build_metrics_spec +from datajunction_server.models.cube_materialization import MetricMeasures +from datajunction_server.models.materialization import ( + DRUID_AGG_MAPPING, + DRUID_SKETCH_CONFIG, + DruidMeasuresCubeConfig, + Measure, + get_druid_aggregator_spec, + register_druid_aggregator, +) +from datajunction_server.errors import DJInvalidInputException from datajunction_server.models.decompose import AggregationRule, MetricComponent from datajunction_server.models.query import ColumnMetadata @@ -374,3 +386,293 @@ def test_preserves_column_order(self): result = _build_metrics_spec(columns, components, aliases) assert [r["name"] for r in result] == ["z_col", "a_col", "m_col"] + + +class TestDruidAggregatorSpec: + """ + Tests for the shared aggregator helper behind both metricsSpec builders. + """ + + def test_merge_is_preferred_over_accumulate(self): + """ + Ingestion merges stored partials, so the merge function keys the lookup. + """ + spec = get_druid_aggregator_spec( + column_name="orders_count", + column_type="bigint", + aggregation="COUNT", + merge="SUM", + ) + assert spec == { + "fieldName": "orders_count", + "name": "orders_count", + "type": "longSum", + } + + def test_falls_back_to_accumulate_without_merge(self): + """A component with no merge phase keys off its accumulate.""" + spec = get_druid_aggregator_spec( + column_name="revenue_sum", + column_type="double", + aggregation="SUM", + merge=None, + ) + assert spec is not None + assert spec["type"] == "doubleSum" + + def test_unmappable_returns_none(self): + """ + Return None for unmappable measures so callers determine fallback behavior. + """ + assert ( + get_druid_aggregator_spec( + column_name="x", + column_type="varchar", + aggregation="SUM", + merge="SUM", + ) + is None + ) + assert ( + get_druid_aggregator_spec( + column_name="x", + column_type=None, + aggregation="SUM", + merge="SUM", + ) + is None + ) + assert ( + get_druid_aggregator_spec( + column_name="x", + column_type="bigint", + aggregation=None, + merge=None, + ) + is None + ) + + def test_sketch_gets_default_config(self): + """An HLL sketch carries its precision defaults into the spec.""" + spec = get_druid_aggregator_spec( + column_name="accounts_hll", + column_type="binary", + aggregation="hll_sketch_agg", + merge="hll_union_agg", + ) + assert spec == { + "fieldName": "accounts_hll", + "name": "accounts_hll", + "type": "HLLSketchMerge", + "lgK": 12, + "tgtHllType": "HLL_4", + } + + def test_declared_params_override_defaults_partially(self): + """ + Metric-level params override defaults for specific keys while retaining others. + """ + spec = get_druid_aggregator_spec( + column_name="accounts_hll", + column_type="binary", + aggregation="hll_sketch_agg", + merge="hll_union_agg", + params={"lgK": 17}, + ) + assert spec is not None + assert spec["lgK"] == 17 + assert spec["tgtHllType"] == "HLL_4" + + def test_params_cannot_overwrite_structural_keys(self): + """ + `params` must not overwrite structural keys like `type` or `fieldName`. + """ + with pytest.raises(DJInvalidInputException) as excinfo: + get_druid_aggregator_spec( + column_name="accounts_hll", + column_type="binary", + aggregation="hll_sketch_agg", + merge="hll_union_agg", + params={"type": "doubleSum", "fieldName": "some_other_column"}, + ) + assert "does not accept" in str(excinfo.value) + assert "`fieldName`" in str(excinfo.value) + assert "`type`" in str(excinfo.value) + + def test_unknown_param_is_rejected(self): + """Unknown parameters are rejected loudly.""" + with pytest.raises(DJInvalidInputException) as excinfo: + get_druid_aggregator_spec( + column_name="accounts_hll", + column_type="binary", + aggregation="hll_sketch_agg", + merge="hll_union_agg", + params={"lgk": 17}, # lowercase k + ) + assert "lgk" in str(excinfo.value) + assert "lgK" in str(excinfo.value) # the supported spelling is named + + def test_params_rejected_for_unparameterized_aggregator(self): + """Params passed to an unparameterized aggregator raise an error.""" + with pytest.raises(DJInvalidInputException) as excinfo: + get_druid_aggregator_spec( + column_name="orders_count", + column_type="bigint", + aggregation="COUNT", + merge="SUM", + params={"compression": 200}, + ) + assert "takes no parameters" in str(excinfo.value) + + +class TestRegisterDruidAggregator: + """ + Tests for registering custom sketch families. + """ + + def test_register_and_use(self): + """A registered family is mappable and carries its default config.""" + try: + register_druid_aggregator( + column_type="binary", + merge_func="test_tdigest_agg", + aggregator="testTDigestSketch", + default_config={"compression": 200}, + ) + spec = get_druid_aggregator_spec( + column_name="latency_tdigest", + column_type="binary", + aggregation="test_tdigest", + merge="test_tdigest_agg", + ) + assert spec == { + "fieldName": "latency_tdigest", + "name": "latency_tdigest", + "type": "testTDigestSketch", + "compression": 200, + } + # A metric's own params win over the family default. + tuned = get_druid_aggregator_spec( + column_name="latency_tdigest", + column_type="binary", + aggregation="test_tdigest", + merge="test_tdigest_agg", + params={"compression": 1000}, + ) + assert tuned is not None + assert tuned["compression"] == 1000 + assert "testTDigestSketch" in DRUID_SKETCH_CONFIG + finally: + DRUID_AGG_MAPPING.pop(("binary", "test_tdigest_agg"), None) + DRUID_SKETCH_CONFIG.pop("testTDigestSketch", None) + + def test_register_without_config(self): + """A family with nothing to tune registers without extra config keys.""" + try: + register_druid_aggregator( + column_type="bigint", + merge_func="test_plain_merge", + aggregator="testPlainAgg", + ) + spec = get_druid_aggregator_spec( + column_name="c", + column_type="bigint", + aggregation="test_plain", + merge="test_plain_merge", + ) + assert spec == { + "fieldName": "c", + "name": "c", + "type": "testPlainAgg", + } + finally: + DRUID_AGG_MAPPING.pop(("bigint", "test_plain_merge"), None) + + def test_conflicting_defaults_are_rejected(self): + """ + Re-registering an aggregator with different defaults raises an error. + """ + try: + register_druid_aggregator( + column_type="binary", + merge_func="test_conflict_agg", + aggregator="testConflictSketch", + default_config={"compression": 100}, + ) + with pytest.raises(DJInvalidInputException) as excinfo: + register_druid_aggregator( + column_type="binary", + merge_func="test_conflict_agg", + aggregator="testConflictSketch", + default_config={"compression": 200}, + ) + assert "already registered with different defaults" in str(excinfo.value) + # Re-registering exactly what is there is a no-op. + register_druid_aggregator( + column_type="binary", + merge_func="test_conflict_agg", + aggregator="testConflictSketch", + default_config={"compression": 100}, + ) + assert DRUID_SKETCH_CONFIG["testConflictSketch"] == {"compression": 100} + finally: + DRUID_AGG_MAPPING.pop(("binary", "test_conflict_agg"), None) + DRUID_SKETCH_CONFIG.pop("testConflictSketch", None) + + def test_conflicting_aggregator_is_rejected(self): + """Two families cannot claim the same (column type, merge func) pair.""" + try: + register_druid_aggregator( + column_type="binary", + merge_func="test_claimed", + aggregator="testFirstSketch", + ) + with pytest.raises(DJInvalidInputException) as excinfo: + register_druid_aggregator( + column_type="binary", + merge_func="test_claimed", + aggregator="testSecondSketch", + ) + assert "already registered as" in str(excinfo.value) + finally: + DRUID_AGG_MAPPING.pop(("binary", "test_claimed"), None) + + def test_v2_measures_cube_config_gets_family_defaults(self): + """ + The V2 path uses the aggregator fallback with `merge=None` to pull correct specs. + """ + try: + register_druid_aggregator( + column_type="binary", + merge_func="test_v2_tdigest", + aggregator="testV2Sketch", + default_config={"compression": 200}, + ) + config = DruidMeasuresCubeConfig.model_construct( + dimensions=[], + measures={ + "some.metric": MetricMeasures.model_construct( + metric="some.metric", + combiner="x", + measures=[ + Measure( + name="latency_tdigest", + field_name="latency_tdigest", + agg="test_v2_tdigest", + type="binary", + ), + ], + ), + }, + ) + assert config.metrics_spec() == { + "latency_tdigest": { + "fieldName": "latency_tdigest", + "name": "latency_tdigest", + "type": "testV2Sketch", + "compression": 200, + }, + } + finally: + DRUID_AGG_MAPPING.pop(("binary", "test_v2_tdigest"), None) + DRUID_SKETCH_CONFIG.pop("testV2Sketch", None) diff --git a/datajunction-server/tests/api/cubes_test.py b/datajunction-server/tests/api/cubes_test.py index aa00f9671..4aae6784d 100644 --- a/datajunction-server/tests/api/cubes_test.py +++ b/datajunction-server/tests/api/cubes_test.py @@ -3584,10 +3584,12 @@ async def test_cube_materialization_metadata( "aggregation": "COUNT", "expression": "*", "grain_alias": None, + "params": None, "merge": "SUM", "name": "count_c8e42e74", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -3595,10 +3597,12 @@ async def test_cube_materialization_metadata( "aggregation": "SUM", "expression": "if(discount > 0.0, 1, 0)", "grain_alias": None, + "params": None, "merge": "SUM", "name": "discount_sum_30b84e6c", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -3606,10 +3610,12 @@ async def test_cube_materialization_metadata( "aggregation": "COUNT", "expression": "price", "grain_alias": None, + "params": None, "merge": "SUM", "name": "price_count_935e7117", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -3617,10 +3623,12 @@ async def test_cube_materialization_metadata( "aggregation": "SUM", "expression": "price * discount", "grain_alias": None, + "params": None, "merge": "SUM", "name": "price_discount_sum_e4ba5456", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -3628,10 +3636,12 @@ async def test_cube_materialization_metadata( "aggregation": "SUM", "expression": "price", "grain_alias": None, + "params": None, "merge": "SUM", "name": "price_sum_935e7117", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -3639,10 +3649,12 @@ async def test_cube_materialization_metadata( "aggregation": "COUNT", "expression": "repair_order_id", "grain_alias": None, + "params": None, "merge": "SUM", "name": "repair_order_id_count_bd241964", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -3650,10 +3662,12 @@ async def test_cube_materialization_metadata( "aggregation": "SUM", "expression": "total_repair_cost", "grain_alias": None, + "params": None, "merge": "SUM", "name": "total_repair_cost_sum_67874507", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -3769,10 +3783,12 @@ async def test_cube_materialization_metadata( "aggregation": "SUM", "expression": "price", "grain_alias": None, + "params": None, "merge": "SUM", "name": "price_sum_252381cf", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -4023,65 +4039,73 @@ async def test_cube_materialization_metadata( "name": "count_c8e42e74", "expression": "*", "grain_alias": None, + "params": None, "aggregation": "COUNT", "merge": "SUM", - "rule": {"type": "full", "level": None}, + "rule": {"type": "full", "level": None, "reaggregate": None}, }, { "name": "discount_sum_30b84e6c", "expression": "if(discount > 0.0, 1, 0)", "grain_alias": None, + "params": None, "aggregation": "SUM", "merge": "SUM", - "rule": {"type": "full", "level": None}, + "rule": {"type": "full", "level": None, "reaggregate": None}, }, { "name": "price_count_935e7117", "expression": "price", "grain_alias": None, + "params": None, "aggregation": "COUNT", "merge": "SUM", - "rule": {"type": "full", "level": None}, + "rule": {"type": "full", "level": None, "reaggregate": None}, }, { "name": "price_discount_sum_e4ba5456", "expression": "price * discount", "grain_alias": None, + "params": None, "aggregation": "SUM", "merge": "SUM", - "rule": {"type": "full", "level": None}, + "rule": {"type": "full", "level": None, "reaggregate": None}, }, { "name": "price_sum_935e7117", "expression": "price", "grain_alias": None, + "params": None, "aggregation": "SUM", "merge": "SUM", - "rule": {"type": "full", "level": None}, + "rule": {"type": "full", "level": None, "reaggregate": None}, }, { "name": "repair_order_id_count_bd241964", "expression": "repair_order_id", "grain_alias": None, + "params": None, "aggregation": "COUNT", "merge": "SUM", - "rule": {"type": "full", "level": None}, + "rule": {"type": "full", "level": None, "reaggregate": None}, }, { "name": "total_repair_cost_sum_67874507", "expression": "total_repair_cost", "grain_alias": None, + "params": None, "aggregation": "SUM", "merge": "SUM", - "rule": {"type": "full", "level": None}, + "rule": {"type": "full", "level": None, "reaggregate": None}, }, { "name": "price_sum_252381cf", "expression": "price", "grain_alias": None, + "params": None, "aggregation": "SUM", "merge": "SUM", - "rule": {"type": "full", "level": None}, + "rule": {"type": "full", "level": None, "reaggregate": None}, }, ], "timestamp_column": "hire_date", @@ -4804,6 +4828,133 @@ class TestCubeMaterializeV2SuccessPaths: and mock the query service client to test the full endpoint flow. """ + @pytest.fixture(autouse=True) + def _preserve_app_dependency_overrides( + self, + client_with_repairs_cube: AsyncClient, + ): + """Protect module-scoped client overrides from function-scoped clients.""" + original_overrides = dict(client_with_repairs_cube.app.dependency_overrides) + yield + client_with_repairs_cube.app.dependency_overrides.clear() + client_with_repairs_cube.app.dependency_overrides.update(original_overrides) + + @pytest.mark.asyncio + async def test_materialize_reaggregate_cube_without_protected_dimension_fails( + self, + client_with_build_v3: AsyncClient, + mocker, + ): + """Druid cube materialization refuses to bake in collapsed + semi-additive metrics without the protected dimension.""" + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.daily_balance", + "description": "Semi-additive balance measured by order date", + "query": "SELECT SUM(line_total) FROM v3.order_details", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "v3.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + }, + ) + assert response.status_code in (200, 201), response.json() + + cube_name = "v3.test_daily_balance_materialization_guard" + response = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": cube_name, + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + "mode": "published", + "description": "Unsafe semi-additive materialization", + }, + ) + assert response.status_code == 201, response.json() + + combiner = mocker.patch( + "datajunction_server.api.cubes.build_combiner_sql_from_preaggs", + ) + + response = await client_with_build_v3.post( + f"/cubes/{cube_name}/materialize", + json={"strategy": "full", "schedule": "0 0 * * *"}, + ) + + assert response.status_code == 400, response.json() + assert "protected dimension" in response.json()["message"] + combiner.assert_not_called() + + @pytest.mark.asyncio + async def test_materialize_derived_reaggregate_cube_without_protected_dimension_fails( + self, + client_with_build_v3: AsyncClient, + mocker, + ): + """Druid cube materialization also refuses derived metrics whose base + metric has an omitted semi-additive protected dimension.""" + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.daily_balance", + "description": "Semi-additive balance measured by order date", + "query": "SELECT SUM(line_total) FROM v3.order_details", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "v3.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + }, + ) + assert response.status_code in (200, 201), response.json() + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.daily_balance_index", + "description": "Derived semi-additive balance index", + "query": "SELECT 10.0 / v3.daily_balance", + "mode": "published", + }, + ) + assert response.status_code in (200, 201), response.json() + + cube_name = "v3.test_daily_balance_index_materialization_guard" + response = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": cube_name, + "metrics": ["v3.daily_balance_index"], + "dimensions": ["v3.product.category"], + "mode": "published", + "description": "Unsafe derived semi-additive materialization", + }, + ) + assert response.status_code == 201, response.json() + + combiner = mocker.patch( + "datajunction_server.api.cubes.build_combiner_sql_from_preaggs", + ) + + response = await client_with_build_v3.post( + f"/cubes/{cube_name}/materialize", + json={"strategy": "full", "schedule": "0 0 * * *"}, + ) + + assert response.status_code == 400, response.json() + assert "protected dimension" in response.json()["message"] + combiner.assert_not_called() + @pytest.mark.asyncio async def test_materialize_cube_full_strategy_success( self, @@ -5596,6 +5747,12 @@ async def test_materialize_cube_returns_metric_combiners( mock_qs_client.materialize_cube_v2.return_value = mocker.MagicMock( urls=["http://workflow/test-cube"], ) + had_query_service_override = get_query_service_client in ( + client.app.dependency_overrides + ) + original_query_service_override = client.app.dependency_overrides.get( + get_query_service_client, + ) client.app.dependency_overrides[get_query_service_client] = lambda: ( mock_qs_client ) @@ -5783,8 +5940,12 @@ async def test_materialize_cube_returns_metric_combiners( ] finally: # Clean up dependency override - if get_query_service_client in client.app.dependency_overrides: - del client.app.dependency_overrides[get_query_service_client] + if had_query_service_override: + client.app.dependency_overrides[get_query_service_client] = ( + original_query_service_override + ) + else: + client.app.dependency_overrides.pop(get_query_service_client, None) class TestCubeDeactivateSuccessPaths: @@ -6020,6 +6181,12 @@ async def test_deactivate_cube_uses_stored_workflow_names( urls=["http://workflow/cube-workflow"], workflow_names=["cube_wf_name_1"], ) + had_query_service_override = get_query_service_client in ( + client.app.dependency_overrides + ) + original_query_service_override = client.app.dependency_overrides.get( + get_query_service_client, + ) client.app.dependency_overrides[get_query_service_client] = lambda: ( mock_qs_client ) @@ -6047,8 +6214,12 @@ async def test_deactivate_cube_uses_stored_workflow_names( # The old path should NOT have been called mock_qs_client.deactivate_cube_workflow.assert_not_called() finally: - if get_query_service_client in client.app.dependency_overrides: - del client.app.dependency_overrides[get_query_service_client] + if had_query_service_override: + client.app.dependency_overrides[get_query_service_client] = ( + original_query_service_override + ) + else: + client.app.dependency_overrides.pop(get_query_service_client, None) class TestCubeBackfillSuccessPaths: diff --git a/datajunction-server/tests/api/graphql/resolvers/test_node_resolver.py b/datajunction-server/tests/api/graphql/resolvers/test_node_resolver.py index 6a47982db..f38a5501a 100644 --- a/datajunction-server/tests/api/graphql/resolvers/test_node_resolver.py +++ b/datajunction-server/tests/api/graphql/resolvers/test_node_resolver.py @@ -70,6 +70,86 @@ def test_columns_resolver_filters_by_attribute(): assert result[0].name == "user_id" +def test_reaggregate_resolver_returns_none_for_non_metric(): + """ + Test that the reaggregate resolver only resolves for metric revisions. + """ + from datajunction_server.api.graphql.scalars.node import NodeRevision + from datajunction_server.database import NodeRevision as DBNodeRevision + from datajunction_server.models.node_type import NodeType + + db_node_revision = DBNodeRevision( + name="source_rev", + type=NodeType.SOURCE, + version="1", + reaggregate={ + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + }, + ) + + result = NodeRevision.reaggregate(NodeRevision, root=db_node_revision) + + assert result is None + + +def test_reaggregate_resolver_returns_none_for_empty_metric_spec(): + """ + Test that the reaggregate resolver returns None for metrics without a spec. + """ + from datajunction_server.api.graphql.scalars.node import NodeRevision + from datajunction_server.database import NodeRevision as DBNodeRevision + from datajunction_server.models.node_type import NodeType + + db_node_revision = DBNodeRevision( + name="metric_rev", + type=NodeType.METRIC, + version="1", + reaggregate=None, + ) + + result = NodeRevision.reaggregate(NodeRevision, root=db_node_revision) + + assert result is None + + +def test_reaggregate_resolver_returns_metric_spec(): + """ + Test that the reaggregate resolver exposes metric reaggregation declarations. + """ + from datajunction_server.api.graphql.scalars.node import NodeRevision + from datajunction_server.database import NodeRevision as DBNodeRevision + from datajunction_server.models.node_type import NodeType + from datajunction_server.models.reaggregate import ReaggregationFunction + + db_node_revision = DBNodeRevision( + name="metric_rev", + type=NodeType.METRIC, + version="1", + reaggregate={ + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + }, + ) + + result = NodeRevision.reaggregate(NodeRevision, root=db_node_revision) + + assert result is not None + assert result.fn is None + assert result.weight is None + assert len(result.rules) == 1 + assert result.rules[0].dimension == "default.date_dim.date" + assert result.rules[0].fn == ReaggregationFunction.LAST_VALUE + + @pytest.mark.asyncio @patch("datajunction_server.api.graphql.resolvers.nodes.DBNode.get_by_name") @patch("datajunction_server.api.graphql.resolvers.nodes.load_node_options") diff --git a/datajunction-server/tests/api/measures_test.py b/datajunction-server/tests/api/measures_test.py index 15cb18343..4d3c0d943 100644 --- a/datajunction-server/tests/api/measures_test.py +++ b/datajunction-server/tests/api/measures_test.py @@ -292,6 +292,7 @@ async def test_list_frozen_measures( "name": "repair_order_id_count_bd241964", "rule": { "level": None, + "reaggregate": None, "type": "full", }, "upstream_revision": { diff --git a/datajunction-server/tests/api/metrics_test.py b/datajunction-server/tests/api/metrics_test.py index b6d939b76..aea246385 100644 --- a/datajunction-server/tests/api/metrics_test.py +++ b/datajunction-server/tests/api/metrics_test.py @@ -3,6 +3,7 @@ """ from unittest.mock import patch +from uuid import uuid4 import pytest import pytest_asyncio @@ -464,10 +465,12 @@ async def test_read_metrics(module__client_with_roads: AsyncClient) -> None: "aggregation": "SUM", "expression": "if(discount > 0.0, 1, 0)", "grain_alias": None, + "params": None, "name": "discount_sum_30b84e6c", "merge": "SUM", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -475,10 +478,12 @@ async def test_read_metrics(module__client_with_roads: AsyncClient) -> None: "aggregation": "COUNT", "expression": "*", "grain_alias": None, + "params": None, "merge": "SUM", "name": "count_c8e42e74", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -494,6 +499,111 @@ async def test_read_metrics(module__client_with_roads: AsyncClient) -> None: assert data["custom_metadata"] is None +@pytest.mark.asyncio +async def test_metric_reaggregate_roundtrip_and_validation( + client_with_roads: AsyncClient, +) -> None: + """ + Semi-additive declarations round-trip and validate their protected dimension. + """ + metric_name = f"default.reaggregate_repairs_{uuid4().hex}" + response = await client_with_roads.post( + "/nodes/metric/", + json={ + "name": metric_name, + "description": "Repair orders with semi-additive declaration", + "query": "SELECT COUNT(repair_order_id) FROM default.repair_orders_fact", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "repair_order_id", + "fn": "last_value", + }, + ], + }, + }, + ) + assert response.status_code in (200, 201), response.json() + assert response.json()["reaggregate"] == { + "fn": None, + "weight": None, + "params": None, + "rules": [ + { + "dimension": "repair_order_id", + "fn": "last_value", + }, + ], + } + + response = await client_with_roads.get(f"/nodes/{metric_name}/") + assert response.status_code == 200 + assert response.json()["reaggregate"] == { + "fn": None, + "weight": None, + "params": None, + "rules": [ + { + "dimension": "repair_order_id", + "fn": "last_value", + }, + ], + } + + response = await client_with_roads.get(f"/metrics/{metric_name}/") + assert response.status_code == 200 + assert response.json()["reaggregate"] == { + "fn": None, + "weight": None, + "params": None, + "rules": [ + { + "dimension": "repair_order_id", + "fn": "last_value", + }, + ], + } + + invalid_metric_name = f"default.invalid_reaggregate_{uuid4().hex}" + response = await client_with_roads.post( + "/nodes/metric/", + json={ + "name": invalid_metric_name, + "description": "Invalid semi-additive declaration", + "query": "SELECT COUNT(repair_order_id) FROM default.repair_orders_fact", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "default.repair_orders_fact.nope", + "fn": "last_value", + }, + ], + }, + }, + ) + assert response.status_code == 422 + assert response.json() == { + "message": "Node definition contains references to columns as " + "reaggregate dimensions that are not on parent nodes.", + "errors": [ + { + "code": "INVALID_COLUMN", + "message": "Node definition contains references to columns as " + "reaggregate dimensions that are not on parent nodes.", + "debug": { + "invalid_reaggregate_dimensions": [ + "default.repair_orders_fact.nope", + ], + }, + "context": "", + }, + ], + "warnings": [], + } + + @pytest_asyncio.fixture(scope="module") async def module__current_user(module__session: AsyncSession) -> User: """ diff --git a/datajunction-server/tests/api/preaggregations_test.py b/datajunction-server/tests/api/preaggregations_test.py index 72a8b9804..02bab8715 100644 --- a/datajunction-server/tests/api/preaggregations_test.py +++ b/datajunction-server/tests/api/preaggregations_test.py @@ -736,11 +736,13 @@ async def test_get_preagg_by_id(self, client_with_preaggs): "expr_hash": "83632b779d87", "expression": "line_total", "grain_alias": None, + "params": None, "merge": "SUM", "name": "line_total_sum_e1f61696", "source_column": None, "rule": { "level": None, + "reaggregate": None, "type": "full", }, "used_by_metrics": [ @@ -803,11 +805,13 @@ async def test_get_preagg_by_id(self, client_with_preaggs): "expr_hash": "221d2a4bfdae", "expression": "quantity", "grain_alias": None, + "params": None, "merge": "SUM", "name": "quantity_sum_06b64d2e", "source_column": None, "rule": { "level": None, + "reaggregate": None, "type": "full", }, "used_by_metrics": [ diff --git a/datajunction-server/tests/api/sql_v2_test.py b/datajunction-server/tests/api/sql_v2_test.py index 42c58eb52..21db493ac 100644 --- a/datajunction-server/tests/api/sql_v2_test.py +++ b/datajunction-server/tests/api/sql_v2_test.py @@ -1201,9 +1201,14 @@ async def create_metric_distinct_single_column(client: AsyncClient): "aggregation": None, "expression": "hard_hat_id", "grain_alias": "hard_hat_id", + "params": None, "merge": None, "name": "hard_hat_id", - "rule": {"level": ["hard_hat_id"], "type": "limited"}, + "rule": { + "level": ["hard_hat_id"], + "reaggregate": None, + "type": "limited", + }, }, ] assert metric_data["derived_expression"] == "COUNT( DISTINCT hard_hat_id)" @@ -1228,9 +1233,14 @@ async def create_metric_distinct_expression(client: AsyncClient): "aggregation": None, "expression": "IF(hard_hat_id = 1, 1, 0)", "grain_alias": "hard_hat_id_distinct_0291ee39", + "params": None, "merge": None, "name": "hard_hat_id_distinct_0291ee39", - "rule": {"level": ["IF(hard_hat_id = 1, 1, 0)"], "type": "limited"}, + "rule": { + "level": ["IF(hard_hat_id = 1, 1, 0)"], + "reaggregate": None, + "type": "limited", + }, }, ] assert ( @@ -1623,10 +1633,12 @@ async def test_metric_definitions_with_nonjoinable_dimensions( "aggregation": "SUM", "expression": "default.local_hard_hats_2.hard_hat_id", "grain_alias": None, + "params": None, "merge": "SUM", "name": "default_DOT_local_hard_hats_2_DOT_hard_hat_id_sum_bf8a8419", "rule": { "level": None, + "reaggregate": None, "type": "full", }, }, @@ -1687,10 +1699,12 @@ async def test_metric_definitions_with_single_joinable_dimensions( "aggregation": None, "expression": "default.municipality_dim.contact_name", "grain_alias": "contact_name", + "params": None, "merge": None, "name": "contact_name", "rule": { "level": ["default.municipality_dim.contact_name"], + "reaggregate": None, "type": "limited", }, }, @@ -1792,6 +1806,7 @@ async def test_metric_definition_with_multiple_joinable_dimensions( "expression": "IF(default.hard_hat.state = 'NY', default.hard_hat.first_name, " "NULL)", "grain_alias": "default_DOT_hard_hat_DOT_state_default_DOT_hard_hat_DOT_first_name_distinct_1a99d6a7", + "params": None, "merge": None, "name": "default_DOT_hard_hat_DOT_state_default_DOT_hard_hat_DOT_first_name_distinct_1a99d6a7", "rule": { @@ -1799,6 +1814,7 @@ async def test_metric_definition_with_multiple_joinable_dimensions( "IF(default.hard_hat.state = 'NY', " "default.hard_hat.first_name, NULL)", ], + "reaggregate": None, "type": "limited", }, }, diff --git a/datajunction-server/tests/construction/build_v3/cube_matcher_test.py b/datajunction-server/tests/construction/build_v3/cube_matcher_test.py index 89a3cf630..e8019e4b4 100644 --- a/datajunction-server/tests/construction/build_v3/cube_matcher_test.py +++ b/datajunction-server/tests/construction/build_v3/cube_matcher_test.py @@ -29,12 +29,49 @@ from datajunction_server.construction.build_v3.utils import ( extract_filter_dimension_refs, ) +from datajunction_server.database.node import Node from datajunction_server.errors import DJInvalidInputException from datajunction_server.models.decompose import Aggregability from datajunction_server.models.dialect import Dialect from tests.construction.build_v3 import assert_sql_equal +async def _create_daily_balance_metric(client_with_build_v3): + """Create a semi-additive metric protected by order date.""" + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.daily_balance", + "description": "Semi-additive balance measured by order date", + "query": "SELECT SUM(line_total) FROM v3.order_details", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "v3.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + }, + ) + assert response.status_code in (200, 201), response.json() + + +async def _create_daily_balance_index_metric(client_with_build_v3): + """Create a derived metric that depends on the semi-additive balance.""" + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.daily_balance_index", + "description": "Derived semi-additive balance index", + "query": "SELECT 10.0 / v3.daily_balance", + "mode": "published", + }, + ) + assert response.status_code in (200, 201), response.json() + + def test_materialized_dimension_lookup_preserves_roles(): """Two roles with one short name resolve to distinct physical columns.""" cube = SimpleNamespace( @@ -1156,6 +1193,197 @@ def _boom(_filters): ) assert "could not be parsed" in str(exc.value) + @pytest.mark.asyncio + async def test_reaggregate_cube_missing_protected_dimension_not_matched( + self, + client_with_build_v3, + session, + ): + """A materialized cube that dropped a semi-additive protected dimension + is not safe to auto-route to.""" + await _create_daily_balance_metric(client_with_build_v3) + response = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": "v3.test_daily_balance_category_cube", + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + "mode": "published", + "description": "Category cube missing semi-additive protected grain", + }, + ) + assert response.status_code == 201, response.json() + + response = await client_with_build_v3.post( + "/data/v3.test_daily_balance_category_cube/availability/", + json={ + "catalog": "default", + "schema_": "analytics", + "table": "daily_balance_category_cube", + "valid_through_ts": int(time.time() * 1000), + }, + ) + assert response.status_code == 200, response.json() + + result = await find_matching_cube( + session, + metrics=["v3.daily_balance"], + dimensions=["v3.product.category"], + ) + assert result is None + + @pytest.mark.asyncio + async def test_reaggregate_unsafe_cube_falls_back_to_live_sql( + self, + client_with_build_v3, + session, + ): + """An unsafe materialized cube is ignored, leaving the live path usable.""" + await _create_daily_balance_metric(client_with_build_v3) + response = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": "v3.test_daily_balance_live_fallback_cube", + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + "mode": "published", + "description": "Unsafe cube for live fallback", + }, + ) + assert response.status_code == 201, response.json() + + response = await client_with_build_v3.post( + "/data/v3.test_daily_balance_live_fallback_cube/availability/", + json={ + "catalog": "default", + "schema_": "analytics", + "table": "daily_balance_live_fallback_cube", + "valid_through_ts": int(time.time() * 1000), + }, + ) + assert response.status_code == 200, response.json() + + result = await build_metrics_sql( + session=session, + metrics=["v3.daily_balance"], + dimensions=["v3.product.category"], + ) + + assert result.cube_name is None + assert "daily_balance_live_fallback_cube" not in result.sql + assert "MAX_BY(" in result.sql + assert "date_id_order" in result.sql + + @pytest.mark.asyncio + async def test_derived_reaggregate_cube_missing_protected_dimension_not_matched( + self, + client_with_build_v3, + session, + ): + """Derived metric cubes inherit base semi-additive materialization needs.""" + await _create_daily_balance_metric(client_with_build_v3) + await _create_daily_balance_index_metric(client_with_build_v3) + response = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": "v3.test_daily_balance_index_category_cube", + "metrics": ["v3.daily_balance_index"], + "dimensions": ["v3.product.category"], + "mode": "published", + "description": "Derived cube missing semi-additive protected grain", + }, + ) + assert response.status_code == 201, response.json() + + response = await client_with_build_v3.post( + "/data/v3.test_daily_balance_index_category_cube/availability/", + json={ + "catalog": "default", + "schema_": "analytics", + "table": "daily_balance_index_category_cube", + "valid_through_ts": int(time.time() * 1000), + }, + ) + assert response.status_code == 200, response.json() + + result = await find_matching_cube( + session, + metrics=["v3.daily_balance_index"], + dimensions=["v3.product.category"], + ) + assert result is None + + @pytest.mark.asyncio + async def test_pinned_reaggregate_cube_missing_protected_dimension_raises( + self, + client_with_build_v3, + session, + ): + """Pinned cube validation also blocks unsafe semi-additive materialization.""" + await _create_daily_balance_metric(client_with_build_v3) + response = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": "v3.test_daily_balance_pinned_unsafe_cube", + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + "mode": "published", + "description": "Pinned unsafe semi-additive cube", + }, + ) + assert response.status_code == 201, response.json() + cube_node = await Node.get_cube_by_name( + session, + "v3.test_daily_balance_pinned_unsafe_cube", + ) + assert cube_node and cube_node.current + + with pytest.raises(DJInvalidInputException) as exc: + await validate_pinned_cube_covers_filters( + session, + cube_node.current, + dimensions=["v3.product.category"], + filters=[], + metrics=["v3.daily_balance"], + ) + assert "protected dimension" in str(exc.value) + + @pytest.mark.asyncio + async def test_pinned_derived_reaggregate_cube_missing_protected_dimension_raises( + self, + client_with_build_v3, + session, + ): + """Pinned derived metric cubes also inherit base semi-additive needs.""" + await _create_daily_balance_metric(client_with_build_v3) + await _create_daily_balance_index_metric(client_with_build_v3) + response = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": "v3.test_daily_balance_index_pinned_unsafe_cube", + "metrics": ["v3.daily_balance_index"], + "dimensions": ["v3.product.category"], + "mode": "published", + "description": "Pinned unsafe derived semi-additive cube", + }, + ) + assert response.status_code == 201, response.json() + cube_node = await Node.get_cube_by_name( + session, + "v3.test_daily_balance_index_pinned_unsafe_cube", + ) + assert cube_node and cube_node.current + + with pytest.raises(DJInvalidInputException) as exc: + await validate_pinned_cube_covers_filters( + session, + cube_node.current, + dimensions=["v3.product.category"], + filters=[], + metrics=["v3.daily_balance_index"], + ) + assert "protected dimension" in str(exc.value) + @pytest.mark.asyncio async def test_pinned_cube_non_druid_dialect_skips_validation( self, diff --git a/datajunction-server/tests/construction/build_v3/decomposition_test.py b/datajunction-server/tests/construction/build_v3/decomposition_test.py index 5a8a6af39..179a95372 100644 --- a/datajunction-server/tests/construction/build_v3/decomposition_test.py +++ b/datajunction-server/tests/construction/build_v3/decomposition_test.py @@ -5,9 +5,11 @@ import pytest from datajunction_server.construction.build_v3.decomposition import ( + _reaggregate_dimension_requested, get_base_metrics_for_derived, is_derived_metric, ) +from datajunction_server.construction.build_v3.metrics import _source_dimension_alias from datajunction_server.construction.build_v3.types import BuildContext from datajunction_server.models.node_type import NodeType @@ -223,6 +225,42 @@ def test_metric_with_only_dimension_parent_not_derived(self): assert result is False +@pytest.mark.parametrize( + ("protected_dimension", "requested_dimensions", "expected"), + [ + ("v3.date.date_id[order]", ["v3.date.date_id[order]"], True), + ("v3.date.date_id[order]", ["v3.date.date_id[order]]"], True), + ("v3.date.date_id[order]", ["v3.date.date_id"], False), + ("v3.date.date_id[order]", ["v3.date.date_id[ship]"], False), + ("v3.date.date_id[order]", ["date_id"], False), + ("v3.date.date_id", ["v3.date.date_id"], True), + ("v3.date.date_id", ["date_id"], True), + ("v3.date.date_id", ["v3.date.date_id[order]"], False), + ], +) +def test_reaggregate_dimension_requested_is_role_sensitive( + protected_dimension, + requested_dimensions, + expected, +): + """A role-less ref must not satisfy a role-qualified protected dimension.""" + assert ( + _reaggregate_dimension_requested(protected_dimension, requested_dimensions) + is expected + ) + + +def test_source_dimension_alias_does_not_fall_back_for_roled_refs(): + """A roled protected dimension must collapse by that role's physical alias.""" + ctx = BuildContext(session=MagicMock(), metrics=[], dimensions=[]) + ctx.alias_registry.register("v3.date.date_id") + ctx.alias_registry.register("v3.date.date_id[ship]") + + assert _source_dimension_alias(ctx, "v3.date.date_id[order]") is None + assert _source_dimension_alias(ctx, "v3.date.date_id[ship]") == "date_id_ship" + assert _source_dimension_alias(ctx, "v3.date.date_id") == "date_id" + + @pytest.mark.asyncio async def test_decomposition_with_dimension_parent_integration( module__client_with_build_v3, diff --git a/datajunction-server/tests/construction/build_v3/metrics_sql_test.py b/datajunction-server/tests/construction/build_v3/metrics_sql_test.py index 697457e2d..847053723 100644 --- a/datajunction-server/tests/construction/build_v3/metrics_sql_test.py +++ b/datajunction-server/tests/construction/build_v3/metrics_sql_test.py @@ -1,3 +1,5 @@ +import time + import pytest from . import assert_sql_equal @@ -99,6 +101,651 @@ async def test_simple_single_metric(self, client_with_build_v3): }, ] + @staticmethod + async def _create_daily_balance_metric(client_with_build_v3): + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.daily_balance", + "description": "Semi-additive balance measured by order date", + "query": "SELECT SUM(line_total) FROM v3.order_details", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "v3.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + }, + ) + assert response.status_code in (200, 201), response.json() + + @staticmethod + async def _create_scaled_daily_balance_metric(client_with_build_v3): + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.scaled_daily_balance", + "description": "Scaled semi-additive balance measured by order date", + "query": "SELECT SUM(line_total) / 100 FROM v3.order_details", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "v3.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + }, + ) + assert response.status_code in (200, 201), response.json() + + @staticmethod + async def _create_first_daily_balance_metric(client_with_build_v3): + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.first_daily_balance", + "description": "Semi-additive balance measured by first order date", + "query": "SELECT SUM(line_total) FROM v3.order_details", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "v3.date.date_id[order]", + "fn": "first_value", + }, + ], + }, + }, + ) + assert response.status_code in (200, 201), response.json() + + @staticmethod + async def _create_daily_balance_index_metric(client_with_build_v3): + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.daily_balance_index", + "description": "Derived semi-additive balance index", + "query": "SELECT 10.0 / v3.daily_balance", + "mode": "published", + }, + ) + assert response.status_code in (200, 201), response.json() + + @staticmethod + async def _create_wow_daily_balance_index_metric(client_with_build_v3): + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.wow_daily_balance_index", + "description": "Week-over-week daily balance index change", + "query": """ + SELECT + (v3.daily_balance_index - LAG(v3.daily_balance_index, 1) + OVER (ORDER BY v3.date.week[order])) + / NULLIF( + LAG(v3.daily_balance_index, 1) + OVER (ORDER BY v3.date.week[order]), + 0 + ) * 100 + """, + "mode": "published", + }, + ) + assert response.status_code in (200, 201), response.json() + + @staticmethod + async def _create_product_balance_metric(client_with_build_v3): + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.product_balance", + "description": "Semi-additive balance protected by product", + "query": "SELECT SUM(line_total) FROM v3.order_details", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "v3.product.product_id", + "fn": "last_value", + }, + ], + }, + }, + ) + assert response.status_code in (200, 201), response.json() + + @staticmethod + async def _create_product_balance_index_metric(client_with_build_v3): + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.product_balance_index", + "description": "Derived semi-additive product balance index", + "query": "SELECT 10.0 / v3.product_balance", + "mode": "published", + }, + ) + assert response.status_code in (200, 201), response.json() + + @staticmethod + async def _create_wow_product_balance_index_metric(client_with_build_v3): + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.wow_product_balance_index", + "description": "Week-over-week product balance index change", + "query": """ + SELECT + (v3.product_balance_index - LAG(v3.product_balance_index, 1) + OVER (ORDER BY v3.date.week[order])) + / NULLIF( + LAG(v3.product_balance_index, 1) + OVER (ORDER BY v3.date.week[order]), + 0 + ) * 100 + """, + "mode": "published", + }, + ) + assert response.status_code in (200, 201), response.json() + + @pytest.mark.asyncio + async def test_reaggregate_collapses_when_protected_dimension_omitted( + self, + client_with_build_v3, + ): + """A semi-additive metric keeps its protected dimension as private grain.""" + await self._create_daily_balance_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + assert_sql_equal( + response.json()["sql"], + """ + WITH + v3_order_details AS ( + SELECT o.order_date, oi.product_id, oi.quantity * oi.unit_price AS line_total + FROM default.v3.orders o + JOIN default.v3.order_items oi ON o.order_id = oi.order_id + ), + v3_product AS ( + SELECT product_id, category + FROM default.v3.products + ), + order_details_0 AS ( + SELECT t2.category, + t1.order_date AS date_id_order, + SUM(t1.line_total) AS line_total_sum_e1f61696 + FROM v3_order_details t1 + LEFT OUTER JOIN v3_product t2 ON t1.product_id = t2.product_id + GROUP BY t2.category, t1.order_date + ) + SELECT order_details_0.category AS category, + MAX_BY( + order_details_0.line_total_sum_e1f61696, + order_details_0.date_id_order + ) AS daily_balance + FROM order_details_0 + GROUP BY order_details_0.category + """, + ) + assert response.json()["columns"] == [ + { + "name": "category", + "type": "string", + "semantic_entity": "v3.product.category", + "semantic_type": "dimension", + }, + { + "name": "daily_balance", + "type": "double", + "semantic_entity": "v3.daily_balance", + "semantic_type": "metric", + }, + ] + + @pytest.mark.asyncio + async def test_reaggregate_with_limited_metric_keeps_protected_grain( + self, + client_with_build_v3, + ): + """Merged metric queries must not add limited grain to the collapse CTE.""" + await self._create_daily_balance_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance", "v3.order_count"], + "dimensions": ["v3.product.category"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + sql = response.json()["sql"] + normalized_sql = " ".join(sql.split()) + assert "order_details_0 AS" in sql + assert "GROUP BY t2.category, t1.order_date" in normalized_sql + assert "GROUP BY t2.category, t1.order_date, t1.order_id" not in normalized_sql + assert "order_details_1_agg AS" in sql + assert "COUNT( DISTINCT order_id)" in normalized_sql + assert "MAX_BY(" in sql + assert "order_details_0.date_id_order" in sql + + @pytest.mark.asyncio + async def test_reaggregate_preserves_single_component_combiner_wrapper( + self, + client_with_build_v3, + ): + """Semi-additive collapse preserves arithmetic around the aggregate.""" + await self._create_scaled_daily_balance_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.scaled_daily_balance"], + "dimensions": ["v3.product.category"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + sql = response.json()["sql"] + normalized_sql = " ".join(sql.split()) + assert "MAX_BY(" in sql + assert "order_details_0.date_id_order" in sql + assert "/ 100 AS scaled_daily_balance" in normalized_sql + assert ( + "MAX_BY(order_details_0.line_total_sum_e1f61696, order_details_0.date_id_order) / 100" + in normalized_sql + ) + + @pytest.mark.asyncio + async def test_reaggregate_druid_uses_latest_by_for_last_value( + self, + client_with_build_v3, + ): + """Druid live SQL uses Druid-native collapse functions.""" + await self._create_daily_balance_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + "dialect": "druid", + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + assert response.json()["dialect"] == "druid" + + assert_sql_equal( + response.json()["sql"], + """ + WITH + v3_order_details AS ( + SELECT o.order_date, oi.product_id, oi.quantity * oi.unit_price AS line_total + FROM default.v3.orders o + JOIN default.v3.order_items oi ON o.order_id = oi.order_id + ), + v3_product AS ( + SELECT product_id, category + FROM default.v3.products + ), + order_details_0 AS ( + SELECT t2.category, + t1.order_date AS date_id_order, + SUM(t1.line_total) AS line_total_sum_e1f61696 + FROM v3_order_details t1 + LEFT OUTER JOIN v3_product t2 ON t1.product_id = t2.product_id + GROUP BY t2.category, t1.order_date + ) + SELECT order_details_0.category AS category, + LATEST_BY( + order_details_0.line_total_sum_e1f61696, + order_details_0.date_id_order + ) AS daily_balance + FROM order_details_0 + GROUP BY order_details_0.category + """, + ) + assert "MAX_BY" not in response.json()["sql"] + assert "ARG_MAX" not in response.json()["sql"] + + @pytest.mark.asyncio + async def test_reaggregate_auto_routed_druid_uses_latest_by( + self, + client_with_build_v3, + ): + """Auto-routed Druid cube SQL uses Druid-native collapse functions.""" + await self._create_daily_balance_metric(client_with_build_v3) + + response = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": "v3.daily_balance_category_cube", + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + "mode": "published", + "description": "Category-only decoy cube", + }, + ) + assert response.status_code == 201, response.json() + + response = await client_with_build_v3.post( + "/data/v3.daily_balance_category_cube/availability/", + json={ + "catalog": "default", + "schema_": "analytics", + "table": "daily_balance_category_cube", + "valid_through_ts": int(time.time() * 1000), + }, + ) + assert response.status_code == 200, response.json() + + response = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": "v3.daily_balance_cube", + "metrics": ["v3.daily_balance"], + "dimensions": [ + "v3.product.category", + "v3.date.date_id[order]", + ], + "mode": "published", + "description": "Daily balance cube at protected grain", + }, + ) + assert response.status_code == 201, response.json() + + response = await client_with_build_v3.post( + "/data/v3.daily_balance_cube/availability/", + json={ + "catalog": "default", + "schema_": "analytics", + "table": "daily_balance_cube", + "valid_through_ts": int(time.time() * 1000), + }, + ) + assert response.status_code == 200, response.json() + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + }, + ) + assert response.status_code == 200, response.json() + assert response.json()["dialect"] == "druid" + + sql = response.json()["sql"] + assert "FROM daily_balance_cube" in sql + assert "daily_balance_category_cube" not in sql + assert "LATEST_BY(" in sql + assert "date_id_order" in sql + assert "MAX_BY" not in sql + assert "ARG_MAX" not in sql + + @pytest.mark.asyncio + async def test_reaggregate_druid_uses_earliest_by_for_first_value( + self, + client_with_build_v3, + ): + """Druid first-value collapse renders as EARLIEST_BY.""" + await self._create_first_daily_balance_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.first_daily_balance"], + "dimensions": ["v3.product.category"], + "dialect": "druid", + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + sql = response.json()["sql"] + assert "EARLIEST_BY(" in sql + assert "MIN_BY" not in sql + assert "ARG_MIN" not in sql + + @pytest.mark.asyncio + async def test_reaggregate_derived_metric_uses_collapsed_base( + self, + client_with_build_v3, + ): + """Derived metrics inline semi-additive collapse in denominator position.""" + await self._create_daily_balance_metric(client_with_build_v3) + await self._create_daily_balance_index_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance_index"], + "dimensions": ["v3.product.category"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + sql = response.json()["sql"] + assert "MAX_BY(" in sql + assert "order_details_0.date_id_order" in sql + assert "MAX_BY" in sql.split(" AS daily_balance_index")[0] + assert "10.0 / NULLIF(MAX_BY(" in sql + assert "10.0 / NULLIF(SUM(" not in sql + + @pytest.mark.asyncio + async def test_reaggregate_derived_metric_preserves_requested_protected_dimension( + self, + client_with_build_v3, + ): + """Derived metrics use normal aggregation when the protected dimension is requested.""" + await self._create_daily_balance_metric(client_with_build_v3) + await self._create_daily_balance_index_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance_index"], + "dimensions": ["v3.date.date_id[order]"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + sql = response.json()["sql"] + assert "10.0 / NULLIF(SUM(" in sql + assert "date_id_order" in sql + assert "MAX_BY(" not in sql + + @pytest.mark.asyncio + async def test_reaggregate_nested_window_metric_reaggregates_with_collapse( + self, + client_with_build_v3, + ): + """Window reaggregation of derived metrics uses semi-additive parents.""" + await self._create_daily_balance_metric(client_with_build_v3) + await self._create_daily_balance_index_metric(client_with_build_v3) + await self._create_wow_daily_balance_index_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.wow_daily_balance_index"], + "dimensions": ["v3.product.category"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + sql = response.json()["sql"] + assert "base_metrics AS" in sql + assert "MAX_BY(" in sql + assert "order_details_0.date_id_order" in sql + assert "10.0 / NULLIF(MAX_BY(" in sql + assert "LAG(base_metrics.daily_balance_index, 1)" in sql + + @pytest.mark.asyncio + async def test_reaggregate_window_grain_reaggregation_uses_collapse( + self, + client_with_build_v3, + ): + """Window aggregation CTEs collapse semi-additive derived parents.""" + await self._create_product_balance_metric(client_with_build_v3) + await self._create_product_balance_index_metric(client_with_build_v3) + await self._create_wow_product_balance_index_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.wow_product_balance_index"], + "dimensions": ["v3.date.date_id[order]"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + sql = response.json()["sql"] + assert "order_details_week_agg AS" in sql + assert "MAX_BY(" in sql + assert "product_id" in sql + assert "10.0 / NULLIF(MAX_BY(" in sql + assert "10.0 / NULLIF(SUM(" not in sql + assert "LAG(order_details_week_agg.product_balance_index, 1)" in sql + + @pytest.mark.asyncio + async def test_reaggregate_uses_normal_aggregation_when_protected_dimension_requested( + self, + client_with_build_v3, + ): + """Requesting the protected dimension means there is nothing to collapse.""" + await self._create_daily_balance_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.date.date_id[order]"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + assert_sql_equal( + response.json()["sql"], + """ + WITH + v3_order_details AS ( + SELECT o.order_date, oi.quantity * oi.unit_price AS line_total + FROM default.v3.orders o + JOIN default.v3.order_items oi ON o.order_id = oi.order_id + ), + order_details_0 AS ( + SELECT t1.order_date AS date_id_order, + SUM(t1.line_total) AS line_total_sum_e1f61696 + FROM v3_order_details t1 + GROUP BY t1.order_date + ) + SELECT order_details_0.date_id_order AS date_id_order, + SUM(order_details_0.line_total_sum_e1f61696) AS daily_balance + FROM order_details_0 + GROUP BY order_details_0.date_id_order + """, + ) + assert "MAX_BY" not in response.json()["sql"] + + @pytest.mark.asyncio + async def test_reaggregate_collapses_when_roleless_base_dimension_requested( + self, + client_with_build_v3, + ): + """A role-less dimension request does not satisfy a roled protected grain.""" + await self._create_daily_balance_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.date.date_id"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + sql = response.json()["sql"] + assert "MAX_BY(" in sql + assert "order_details_0.date_id_order" in sql + assert ( + "SUM(order_details_0.line_total_sum_e1f61696) AS daily_balance" not in sql + ) + + @pytest.mark.asyncio + async def test_reaggregate_collapses_with_coarser_time_dimension( + self, + client_with_build_v3, + ): + """Coarser time output still collapses across the protected date grain.""" + await self._create_daily_balance_metric(client_with_build_v3) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.date.month[order]"], + "use_materialized": "false", + }, + ) + assert response.status_code == 200, response.json() + + assert_sql_equal( + response.json()["sql"], + """ + WITH + v3_date AS ( + SELECT date_id, month + FROM default.v3.dates + ), + v3_order_details AS ( + SELECT o.order_date, oi.quantity * oi.unit_price AS line_total + FROM default.v3.orders o + JOIN default.v3.order_items oi ON o.order_id = oi.order_id + ), + order_details_0 AS ( + SELECT t2.month AS month_order, + COALESCE(t1.order_date, t2.date_id) AS date_id_order, + SUM(t1.line_total) AS line_total_sum_e1f61696 + FROM v3_order_details t1 + LEFT OUTER JOIN v3_date t2 ON t1.order_date = t2.date_id + GROUP BY t2.month, COALESCE(t1.order_date, t2.date_id) + ) + SELECT order_details_0.month_order AS month_order, + MAX_BY( + order_details_0.line_total_sum_e1f61696, + order_details_0.date_id_order + ) AS daily_balance + FROM order_details_0 + GROUP BY order_details_0.month_order + """, + ) + @pytest.mark.asyncio async def test_multiple_metrics_same_grain(self, client_with_build_v3): """ @@ -3763,15 +4410,27 @@ async def test_cross_fact_window_metric_with_finer_grain( CAST(COUNT( DISTINCT order_details_0.order_id) AS DOUBLE) / NULLIF(COUNT( DISTINCT page_views_enriched_0.customer_id), 0) AS conversion_rate FROM order_details_0 FULL OUTER JOIN page_views_enriched_0 ON order_details_0.date_id = page_views_enriched_0.date_id AND order_details_0.category = page_views_enriched_0.category AND order_details_0.week = page_views_enriched_0.week GROUP BY 1, 2, 3 + ), + order_details_week_agg AS ( + SELECT base_metrics.category AS category, + base_metrics.week AS week, + CAST(SUM(base_metrics.order_count) AS DOUBLE) / NULLIF(SUM(base_metrics.visitor_count), 0) AS conversion_rate + FROM base_metrics + GROUP BY base_metrics.category, base_metrics.week + ), + order_details_week AS ( + SELECT order_details_week_agg.category AS category, + order_details_week_agg.week AS week, + (order_details_week_agg.conversion_rate - LAG(order_details_week_agg.conversion_rate, 1) OVER ( PARTITION BY order_details_week_agg.category + ORDER BY order_details_week_agg.week) ) / NULLIF(LAG(order_details_week_agg.conversion_rate, 1) OVER ( PARTITION BY order_details_week_agg.category + ORDER BY order_details_week_agg.week) , 0) * 100 AS wow_conversion_rate_change + FROM order_details_week_agg ) - SELECT base_metrics.date_id AS date_id, base_metrics.category AS category, base_metrics.week AS week, - (base_metrics.conversion_rate - LAG(base_metrics.conversion_rate, 1) OVER ( PARTITION BY base_metrics.category - ORDER BY base_metrics.week) ) / NULLIF(LAG(base_metrics.conversion_rate, 1) OVER ( PARTITION BY base_metrics.category - ORDER BY base_metrics.week) , 0) * 100 AS wow_conversion_rate_change - FROM base_metrics + order_details_week.wow_conversion_rate_change AS wow_conversion_rate_change + FROM base_metrics LEFT OUTER JOIN order_details_week ON base_metrics.category = order_details_week.category AND base_metrics.week = order_details_week.week """, ) @@ -3877,15 +4536,27 @@ async def test_cross_fact_window_on_derived_metric(self, client_with_build_v3): SUM(page_views_enriched_0.view_id_count_f41e2db4) / NULLIF(COUNT( DISTINCT page_views_enriched_0.session_id), 0) AS pages_per_session FROM order_details_0 FULL OUTER JOIN page_views_enriched_0 ON order_details_0.date_id = page_views_enriched_0.date_id AND order_details_0.category = page_views_enriched_0.category AND order_details_0.week = page_views_enriched_0.week GROUP BY 1, 2, 3 + ), + order_details_week_agg AS ( + SELECT base_metrics.category AS category, + base_metrics.week AS week, + SUM(base_metrics.total_revenue) / NULLIF(SUM(base_metrics.order_count), 0) / NULLIF(SUM(base_metrics.page_view_count) / NULLIF(SUM(base_metrics.session_count), 0), 0) AS efficiency_ratio + FROM base_metrics + GROUP BY base_metrics.category, base_metrics.week + ), + order_details_week AS ( + SELECT order_details_week_agg.category AS category, + order_details_week_agg.week AS week, + (order_details_week_agg.efficiency_ratio - LAG(order_details_week_agg.efficiency_ratio, 1) OVER ( PARTITION BY order_details_week_agg.category + ORDER BY order_details_week_agg.week) ) / NULLIF(LAG(order_details_week_agg.efficiency_ratio, 1) OVER ( PARTITION BY order_details_week_agg.category + ORDER BY order_details_week_agg.week) , 0) * 100 AS wow_efficiency_ratio_change + FROM order_details_week_agg ) - SELECT base_metrics.date_id AS date_id, base_metrics.category AS category, base_metrics.week AS week, - (base_metrics.efficiency_ratio - LAG(base_metrics.efficiency_ratio, 1) OVER ( PARTITION BY base_metrics.category - ORDER BY base_metrics.week) ) / NULLIF(LAG(base_metrics.efficiency_ratio, 1) OVER ( PARTITION BY base_metrics.category - ORDER BY base_metrics.week) , 0) * 100 AS wow_efficiency_ratio_change - FROM base_metrics""", + order_details_week.wow_efficiency_ratio_change AS wow_efficiency_ratio_change + FROM base_metrics LEFT OUTER JOIN order_details_week ON base_metrics.category = order_details_week.category AND base_metrics.week = order_details_week.week""", ) @pytest.mark.asyncio @@ -3989,8 +4660,8 @@ async def test_cross_fact_window_on_base_metrics(self, client_with_build_v3): order_details_week_agg AS ( SELECT base_metrics.category AS category, base_metrics.week AS week, - COUNT( DISTINCT base_metrics.order_id) AS order_count, - COUNT( DISTINCT base_metrics.customer_id) AS visitor_count + SUM(base_metrics.order_count) AS order_count, + SUM(base_metrics.visitor_count) AS visitor_count FROM base_metrics GROUP BY base_metrics.category, base_metrics.week ), diff --git a/datajunction-server/tests/construction/build_v3/preagg_matcher_test.py b/datajunction-server/tests/construction/build_v3/preagg_matcher_test.py index 294428b82..e88dfb1ec 100644 --- a/datajunction-server/tests/construction/build_v3/preagg_matcher_test.py +++ b/datajunction-server/tests/construction/build_v3/preagg_matcher_test.py @@ -32,6 +32,7 @@ from datajunction_server.database.preaggregation import ( PreAggregation, compute_expression_hash, + measure_identity_token, ) from datajunction_server.database.user import User from datajunction_server.errors import DJInvalidInputException @@ -220,7 +221,7 @@ async def test_returns_identities_for_all_components( parent_node: Node, metric_node: Node, ): - """Should return an (expression hash, aggregation) pair per component.""" + """Should return one identity token per component.""" components = [ (metric_node, make_component("sum_revenue", "price * quantity")), (metric_node, make_component("sum_quantity", "quantity")), @@ -230,8 +231,8 @@ async def test_returns_identities_for_all_components( identities = get_required_measure_identities(grain_group) assert identities == { - (compute_expression_hash("price * quantity"), "SUM"), - (compute_expression_hash("quantity"), "SUM"), + measure_identity_token(compute_expression_hash("price * quantity"), "SUM"), + measure_identity_token(compute_expression_hash("quantity"), "SUM"), } @pytest.mark.asyncio @@ -264,7 +265,9 @@ async def test_deduplicates_same_expression_and_aggregation( identities = get_required_measure_identities(grain_group) - assert identities == {(compute_expression_hash("price * quantity"), "SUM")} + assert identities == { + measure_identity_token(compute_expression_hash("price * quantity"), "SUM"), + } @pytest.mark.asyncio async def test_same_expression_different_aggregation_are_distinct( @@ -294,8 +297,8 @@ async def test_same_expression_different_aggregation_are_distinct( identities = get_required_measure_identities(grain_group) assert identities == { - (compute_expression_hash("unit_price"), "SUM"), - (compute_expression_hash("unit_price"), "MAX"), + measure_identity_token(compute_expression_hash("unit_price"), "SUM"), + measure_identity_token(compute_expression_hash("unit_price"), "MAX"), } diff --git a/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py b/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py index 0edd7753e..f8a045a70 100644 --- a/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py +++ b/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py @@ -86,6 +86,28 @@ async def _fake_columns(*args, **kwargs): del client.app.dependency_overrides[get_query_service_client] +async def _create_daily_balance_metric(client_with_build_v3): + """Create a semi-additive metric protected by order date.""" + response = await client_with_build_v3.post( + "/nodes/metric/", + json={ + "name": "v3.daily_balance", + "description": "Semi-additive balance measured by order date", + "query": "SELECT SUM(line_total) FROM v3.order_details", + "mode": "published", + "reaggregate": { + "rules": [ + { + "dimension": "v3.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + }, + ) + assert response.status_code in (200, 201), response.json() + + class TestExternalPreAggRouting: """Queries route to externally-registered pre-agg tables via source_column.""" @@ -198,6 +220,98 @@ async def test_external_preagg_rolls_up_additive(self, client_with_build_v3): """, ) + @pytest.mark.asyncio + async def test_external_preagg_retaining_reaggregate_dimension_rolls_up( + self, + client_with_build_v3, + ): + """A semi-additive metric can read a fine pre-agg that still has the + protected dimension, then collapse in the final metrics query.""" + await _create_daily_balance_metric(client_with_build_v3) + await _register_external_preagg( + client_with_build_v3, + metrics=["v3.daily_balance"], + dimensions=[ + "v3.product.category", + "v3.date.date_id[order]", + ], + table_ref={ + "catalog": "default", + "schema": "analytics", + "table": "daily_balance_by_category_day", + "valid_through_ts": 20250101, + }, + measure_columns={"v3.daily_balance": "balance_sum"}, + dimension_columns={"v3.date.date_id[order]": "order_day"}, + table_columns={ + "category": "string", + "order_day": "int", + "balance_sum": "double", + }, + ) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + }, + ) + assert response.status_code == 200, response.json() + assert_sql_equal( + response.json()["sql"], + """ + WITH order_details_0 AS ( + SELECT category, + order_day date_id_order, + SUM(balance_sum) balance_sum + FROM default.analytics.daily_balance_by_category_day + GROUP BY category, order_day + ) + SELECT order_details_0.category AS category, + MAX_BY(order_details_0.balance_sum, order_details_0.date_id_order) + AS daily_balance + FROM order_details_0 + GROUP BY order_details_0.category + """, + ) + + @pytest.mark.asyncio + async def test_external_preagg_missing_reaggregate_dimension_is_not_used( + self, + client_with_build_v3, + ): + """A semi-additive metric must not read a pre-agg that dropped the key + required by the final collapse.""" + await _create_daily_balance_metric(client_with_build_v3) + await _register_external_preagg( + client_with_build_v3, + metrics=["v3.daily_balance"], + dimensions=["v3.product.category"], + table_ref={ + "catalog": "default", + "schema": "analytics", + "table": "daily_balance_by_category", + "valid_through_ts": 20250101, + }, + measure_columns={"v3.daily_balance": "balance_sum"}, + table_columns={"category": "string", "balance_sum": "double"}, + ) + + response = await client_with_build_v3.get( + "/sql/metrics/v3/", + params={ + "metrics": ["v3.daily_balance"], + "dimensions": ["v3.product.category"], + }, + ) + assert response.status_code == 200, response.json() + + sql = response.json()["sql"] + assert "default.analytics.daily_balance_by_category" not in sql + assert "MAX_BY(" in sql + assert "date_id_order" in sql + @pytest.mark.asyncio async def test_external_non_additive_not_rolled_up(self, client_with_build_v3): """A non-additive measure (COUNT DISTINCT) does not roll up to a coarser diff --git a/datajunction-server/tests/construction/build_v3/reaggregate_coverage_test.py b/datajunction-server/tests/construction/build_v3/reaggregate_coverage_test.py new file mode 100644 index 000000000..358f14d11 --- /dev/null +++ b/datajunction-server/tests/construction/build_v3/reaggregate_coverage_test.py @@ -0,0 +1,764 @@ +"""Focused coverage tests for build-v3 reaggregation helpers.""" + +from types import SimpleNamespace + +import pytest + +from datajunction_server.construction.build_v3.cube_matcher import ( + _cube_dimension_covers_reaggregate_dimension, + _metric_graph_has_reaggregate, + _reaggregate_dimensions_for_cube_metrics, + _reaggregate_requirements_for_cube_metrics, + _reaggregate_requirements_for_metrics, + _reaggregate_requirements_for_decomposed_metrics, + _reaggregate_requirements_for_metrics_if_needed, + build_synthetic_grain_group, +) +from datajunction_server.construction.build_v3 import ( + cube_matcher as cube_matcher_module, +) +from datajunction_server.construction.build_v3 import builder as builder_module +from datajunction_server.construction.build_v3 import metrics as metrics_module +from datajunction_server.construction.build_v3.builder import setup_build_context +from datajunction_server.construction.build_v3.measures import ( + build_grain_group_sql, + build_select_ast, + build_window_metric_grain_groups, +) +from datajunction_server.construction.build_v3.metrics import ( + _build_reaggregate_collapse_expression, + _dimension_ref_base, + _dimension_ref_role, + _references_component, + _replace_reaggregate_merge_expression, + _metric_parent_refs, + _source_dimension_alias, + build_window_agg_cte_from_base_metrics, + generate_metrics_sql, +) +from datajunction_server.construction.build_v3.types import ( + BuildContext, + ColumnMetadata, + DecomposedMetricInfo, + GeneratedMeasuresSQL, + GrainGroup, + GrainGroupSQL, + ResolvedDimension, +) +from datajunction_server.errors import DJInvalidInputException +from datajunction_server.models.decompose import ( + Aggregability, + AggregationRule, + MetricComponent, +) +from datajunction_server.models.dialect import Dialect +from datajunction_server.models.node import NodeType +from datajunction_server.models.reaggregate import ( + DimensionReaggregateRule, + ReaggregationFunction, +) +from datajunction_server.sql.parsing import ast +from datajunction_server.sql.parsing.backends.antlr4 import parse +from datajunction_server.sql.parsing.types import BigIntType, DoubleType, StringType + + +def _metric_node(name: str, query: str = "SELECT SUM(value) FROM test.parent"): + return SimpleNamespace( + name=name, + type=NodeType.METRIC, + current=SimpleNamespace( + query=query, + columns=[SimpleNamespace(type=DoubleType())], + ), + ) + + +def _source_node(name: str = "test.parent"): + return SimpleNamespace( + name=name, + type=NodeType.SOURCE, + current=SimpleNamespace( + catalog=SimpleNamespace(name="default"), + schema_="analytics", + table="parent", + columns=[ + SimpleNamespace(name="id", type=BigIntType()), + SimpleNamespace(name="value", type=DoubleType()), + SimpleNamespace(name="region", type=StringType()), + SimpleNamespace(name="date_id", type=BigIntType()), + ], + ), + ) + + +def _semi_additive_component( + name: str = "balance_sum", + dimension: str = "v3.date.date_id", + fn: ReaggregationFunction = ReaggregationFunction.LAST_VALUE, +) -> MetricComponent: + return MetricComponent( + name=name, + expression="value", + aggregation="SUM", + rule=AggregationRule( + type=Aggregability.FULL, + reaggregate=DimensionReaggregateRule( + dimension=dimension, + fn=fn, + ), + ), + ) + + +def _decomposed_metric( + metric_name: str, + component: MetricComponent | None = None, + query: str = "SELECT SUM(value) FROM test.parent", +) -> DecomposedMetricInfo: + return DecomposedMetricInfo( + metric_node=_metric_node(metric_name, query), + components=[component or _semi_additive_component()], + aggregability=Aggregability.FULL, + combiner=query, + derived_ast=parse(query), + ) + + +def test_reaggregate_collapse_expression_handles_min_max_and_rejects_unsupported(): + """MIN/MAX collapse ignore the protected dimension; unsupported functions fail.""" + value_ref = ast.Column(name=ast.Name("balance")) + protected_ref = ast.Column(name=ast.Name("date_id")) + + assert ( + str( + _build_reaggregate_collapse_expression( + ReaggregationFunction.MIN, + Dialect.SPARK, + value_ref, + protected_ref, + ), + ) + == "MIN(balance)" + ) + assert ( + str( + _build_reaggregate_collapse_expression( + ReaggregationFunction.MAX, + Dialect.SPARK, + value_ref, + protected_ref, + ), + ) + == "MAX(balance)" + ) + + with pytest.raises(DJInvalidInputException, match="Unsupported semi-additive"): + _build_reaggregate_collapse_expression( + ReaggregationFunction.SUM, + Dialect.SPARK, + value_ref, + protected_ref, + ) + + +def test_references_component_checks_all_columns_and_handles_no_match(): + """Component reference detection handles misses before matches.""" + expression = parse( + "SELECT SUM(other_value) + SUM(balance_sum) FROM base", + ).select.projection[0] + + assert _references_component(expression, "balance_sum") + assert not _references_component(expression, "missing_component") + + +def test_replace_reaggregate_merge_expression_skips_nonmatching_functions(): + """Combiner replacement skips unrelated functions before replacing the match.""" + combiner = parse( + "SELECT SUM(other_value) + SUM(balance_sum) FROM base", + ).select.projection[0] + collapse_expr = ast.Function( + ast.Name("MAX"), + args=[ast.Column(name=ast.Name("balance_sum"))], + ) + + rendered = str( + _replace_reaggregate_merge_expression( + combiner, + "balance_sum", + "SUM", + collapse_expr, + ), + ) + + assert "SUM(other_value)" in rendered + assert "MAX(balance_sum)" in rendered + + +def test_replace_reaggregate_merge_expression_rejects_missing_component(): + """Combiner replacement fails if the expected merge function is absent.""" + combiner = parse("SELECT AVG(balance_sum) FROM base").select.projection[0] + + with pytest.raises(DJInvalidInputException, match="could not find"): + _replace_reaggregate_merge_expression( + combiner, + "balance_sum", + "SUM", + ast.Function( + ast.Name("MAX"), + args=[ast.Column(name=ast.Name("balance_sum"))], + ), + ) + + +def test_replace_reaggregate_merge_expression_skips_parentless_nested_match(): + """A matching child without a parent is skipped before the final error.""" + + class ParentlessExpression(ast.Expression): + def __str__(self): + return "parentless" + + @property + def children(self): + yield ast.Function( + ast.Name("SUM"), + args=[ast.Column(name=ast.Name("balance_sum"))], + ) + + with pytest.raises(DJInvalidInputException, match="could not find"): + _replace_reaggregate_merge_expression( + ParentlessExpression(), + "balance_sum", + "SUM", + ast.Function( + ast.Name("MAX"), + args=[ast.Column(name=ast.Name("balance_sum"))], + ), + ) + + +def test_reaggregate_dimension_ref_helpers_handle_roleless_refs_without_alias(): + """Roleless refs return None for role and do not require an alias fallback.""" + ctx = BuildContext(session=SimpleNamespace(), metrics=[], dimensions=[]) + + assert _dimension_ref_base("v3.date.date_id[order]") == "v3.date.date_id" + assert _dimension_ref_role("v3.date.date_id") is None + assert _source_dimension_alias(ctx, "v3.date.date_id") is None + + +def test_metric_parent_refs_adds_metric_refs_discovered_from_query(): + """Parsed-query metric references supplement the cached parent map.""" + balance = _metric_node("balance") + extra = _metric_node("extra_metric") + derived = _metric_node( + "derived", + "SELECT balance + extra_metric + raw_col FROM test.parent", + ) + ctx = BuildContext(session=SimpleNamespace(), metrics=[], dimensions=[]) + ctx.nodes = { + "balance": balance, + "extra_metric": extra, + "derived": derived, + } + ctx.parent_map = {"derived": ["balance"]} + + assert _metric_parent_refs(ctx, "derived") == ["balance", "extra_metric"] + + +def test_window_agg_from_base_metrics_collapses_semi_additive_derived_parent( + monkeypatch, +): + """Cross-fact window reaggregation replaces derived metric parents recursively.""" + monkeypatch.setattr(metrics_module, "get_column_full_name", lambda _col: "") + balance = _metric_node("balance") + extra = _metric_node("extra_metric") + derived = _metric_node( + "derived", + "SELECT balance + extra_metric + raw_col FROM test.parent", + ) + window = _metric_node("window_metric") + ctx = BuildContext( + session=SimpleNamespace(), + metrics=["window_metric"], + dimensions=["category"], + ) + ctx.nodes = { + "balance": balance, + "extra_metric": extra, + "derived": derived, + "window_metric": window, + } + ctx.parent_map = { + "window_metric": ["derived"], + "derived": ["balance"], + } + ctx.alias_registry.register("v3.date.date_id") + + query = build_window_agg_cte_from_base_metrics( + GrainGroupSQL( + query=parse("SELECT category FROM base_metrics"), + columns=[ + ColumnMetadata( + name="category", + semantic_name="category", + type="string", + semantic_type="dimension", + ), + ], + grain=["category"], + aggregability=Aggregability.FULL, + metrics=[], + parent_name="cross_fact", + is_window_grain_group=True, + window_metrics_served=["window_metric"], + ), + "base_metrics", + ctx, + { + "balance": _decomposed_metric( + "balance", + _semi_additive_component(fn=ReaggregationFunction.MAX), + ), + }, + ) + + rendered = str(query) + assert "MAX(base_metrics.balance)" in rendered + assert "extra_metric" in rendered + assert "raw_col" in rendered + + +def test_generate_metrics_sql_rejects_multiple_base_grain_groups_with_reaggregate(): + """Live SQL does not allow protected-dimension fanout across base groups.""" + ctx = BuildContext(session=SimpleNamespace(), metrics=["balance"], dimensions=[]) + grain_groups = [ + GrainGroupSQL( + query=parse("SELECT category, balance_sum FROM fact_a"), + columns=[], + grain=["category"], + aggregability=Aggregability.FULL, + metrics=["balance"], + parent_name="fact_a", + reaggregate_dimension_aliases={"balance_sum": "date_id"}, + ), + GrainGroupSQL( + query=parse("SELECT category, orders FROM fact_b"), + columns=[], + grain=["category"], + aggregability=Aggregability.FULL, + metrics=["orders"], + parent_name="fact_b", + ), + ] + measures_result = GeneratedMeasuresSQL( + grain_groups=grain_groups, + dialect=Dialect.SPARK, + requested_dimensions=[], + ctx=ctx, + ) + + with pytest.raises(DJInvalidInputException, match="multiple base grain groups"): + generate_metrics_sql(ctx, measures_result, {}) + + +def test_cube_reaggregate_requirements_skip_satisfied_dimensions_and_dedupe(): + """Cube requirement helpers skip requested protected dims and de-duplicate.""" + metric_revision = SimpleNamespace( + name="balance", + reaggregate={ + "rules": [ + { + "dimension": "v3.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + ) + cube = SimpleNamespace(metric_node_revisions=lambda: [metric_revision]) + + assert ( + _reaggregate_requirements_for_cube_metrics( + cube, + ["balance"], + ["v3.date.date_id[order]"], + ) + == [] + ) + + duplicate_cube = SimpleNamespace( + metric_node_revisions=lambda: [metric_revision, metric_revision], + ) + assert _reaggregate_dimensions_for_cube_metrics( + duplicate_cube, + ["balance"], + ["v3.product.category"], + ) == ["v3.date.date_id[order]"] + + +def test_cube_dimension_coverage_accepts_bare_protected_parent_column(): + """Cube coverage accepts bare parent-column protected dimensions.""" + assert _cube_dimension_covers_reaggregate_dimension( + "order_date", + "v3.order_details.order_date", + ) + assert not _cube_dimension_covers_reaggregate_dimension( + "order_date", + "v3.order_details.ship_date", + ) + assert not _cube_dimension_covers_reaggregate_dimension( + "order_date", + "v3.order_details.order_date[ship]", + ) + assert not _cube_dimension_covers_reaggregate_dimension( + "v3.date.date_id", + "v3.order_details.order_date", + ) + + +@pytest.mark.asyncio +async def test_metric_graph_has_reaggregate_skips_empty_metric_list(): + """An empty metric list does not touch the database.""" + + class Session: + async def execute(self, _stmt): + raise AssertionError("empty metric graph should not query") + + assert not await _metric_graph_has_reaggregate(Session(), []) + + +@pytest.mark.asyncio +async def test_cube_reaggregate_requirements_skip_full_extract_without_reaggregate( + monkeypatch, +): + """No reaggregate in the metric graph skips full component extraction.""" + + async def fail_full_extract(*_args, **_kwargs): + raise AssertionError("full metric extraction should not run") + + class Result: + def all(self): + return [("v3.total_revenue", None, None)] + + class Session: + async def execute(self, _stmt): + return Result() + + monkeypatch.setattr( + cube_matcher_module, + "_reaggregate_requirements_for_metrics", + fail_full_extract, + ) + + assert ( + await _reaggregate_requirements_for_metrics_if_needed( + Session(), + ["v3.total_revenue"], + ["v3.product.category"], + ) + == [] + ) + + +@pytest.mark.asyncio +async def test_cube_reaggregate_requirements_checks_derived_metric_ancestors( + monkeypatch, +): + """Derived metrics still use full extraction when an ancestor has reaggregate.""" + expected = [("v3.balance_index", "v3.date.date_id", ReaggregationFunction.MAX)] + + async def full_extract(*_args, **_kwargs): + return expected + + class Result: + def __init__(self, rows): + self.rows = rows + + def all(self): + return self.rows + + class Session: + def __init__(self): + self.calls = 0 + + async def execute(self, _stmt): + self.calls += 1 + if self.calls == 1: + return Result([("v3.balance_index", None, "v3.daily_balance")]) + return Result([("v3.daily_balance", {"rules": []}, None)]) + + monkeypatch.setattr( + cube_matcher_module, + "_reaggregate_requirements_for_metrics", + full_extract, + ) + + assert ( + await _reaggregate_requirements_for_metrics_if_needed( + Session(), + ["v3.balance_index"], + ["v3.product.category"], + ) + == expected + ) + + +@pytest.mark.asyncio +async def test_reaggregate_requirements_skip_components_without_reaggregate( + monkeypatch, +): + """Full extraction ignores ordinary components before collecting reaggregate.""" + + class Extractor: + async def extract(self, _session): + return [ + MetricComponent( + name="ordinary_sum", + expression="value", + aggregation="SUM", + rule=AggregationRule(type=Aggregability.FULL), + ), + _semi_additive_component( + "balance_sum", + "v3.date.date_id", + ReaggregationFunction.MAX, + ), + ], None + + class FakeMetricComponentExtractor: + @classmethod + async def from_node_name(cls, _metric_name, _session): + return Extractor() + + monkeypatch.setattr( + cube_matcher_module, + "MetricComponentExtractor", + FakeMetricComponentExtractor, + ) + + assert await _reaggregate_requirements_for_metrics( + SimpleNamespace(), + ["balance"], + ["v3.product.category"], + ) == [("balance", "v3.date.date_id", ReaggregationFunction.MAX)] + + +def test_decomposed_reaggregate_requirements_dedupe_duplicate_components(): + """Duplicate component rules produce one cube materialization requirement.""" + rule = DimensionReaggregateRule( + dimension="v3.date.date_id", + fn=ReaggregationFunction.LAST_VALUE, + ) + components = [ + MetricComponent( + name=f"balance_sum_{idx}", + expression="value", + aggregation="SUM", + rule=AggregationRule(type=Aggregability.FULL, reaggregate=rule), + ) + for idx in range(2) + ] + decomposed = _decomposed_metric("balance", components[0]) + decomposed.components = components + + assert _reaggregate_requirements_for_decomposed_metrics( + {"balance": decomposed}, + ["balance"], + [], + ) == [("balance", "v3.date.date_id", ReaggregationFunction.LAST_VALUE)] + + +def test_build_synthetic_grain_group_skips_duplicate_internal_dimension(): + """Filter-only protected dims can already be present in ctx.dimensions.""" + ctx = BuildContext( + session=SimpleNamespace(), + metrics=["balance"], + dimensions=["v3.date.date_id"], + dialect=Dialect.DRUID, + ) + ctx.filter_dimensions = {"v3.date.date_id"} + ctx.nodes = {"balance": _metric_node("balance")} + ctx.parent_map = {"balance": ["test.parent"]} + cube = SimpleNamespace( + name="daily_balance_cube", + availability=SimpleNamespace(table="daily_balance_cube"), + materializations=[], + ) + + grain_group = build_synthetic_grain_group( + ctx, + {"balance": _decomposed_metric("balance")}, + cube, + ) + + assert grain_group.grain == ["date_id"] + assert [ + col.name for col in grain_group.columns if col.semantic_type == "dimension" + ] == [ + "date_id", + ] + + +def test_build_select_ast_skips_non_output_dimensions(): + """Resolved dimensions outside output refs are join/group-only.""" + ctx = BuildContext(session=SimpleNamespace(), metrics=[], dimensions=[]) + parent = _source_node() + resolved_dimensions = [ + ResolvedDimension( + original_ref="test.parent.region", + node_name="test.parent", + column_name="region", + role=None, + join_path=None, + is_local=True, + ), + ResolvedDimension( + original_ref="test.parent.date_id", + node_name="test.parent", + column_name="date_id", + role=None, + join_path=None, + is_local=True, + ), + ] + + query, _ = build_select_ast( + ctx, + metric_expressions=[], + resolved_dimensions=resolved_dimensions, + parent_node=parent, + output_dimension_refs={"test.parent.region"}, + ) + + rendered = str(query) + assert "t1.region" in rendered + assert "t1.date_id AS date_id" not in rendered + + +def test_build_grain_group_sql_handles_internal_alias_already_in_grain(): + """The internal reaggregate alias can match a requested dimension alias.""" + ctx = BuildContext( + session=SimpleNamespace(), + metrics=["balance"], + dimensions=["test.parent.date_id"], + use_materialized=False, + ) + parent = _source_node() + metric = _metric_node("balance") + component = _semi_additive_component(dimension="test.parent.date_id") + resolved_dimension = ResolvedDimension( + original_ref="test.parent.date_id", + node_name="test.parent", + column_name="date_id", + role=None, + join_path=None, + is_local=True, + ) + grain_group = GrainGroup( + parent_node=parent, + aggregability=Aggregability.FULL, + grain_columns=[], + components=[(metric, component)], + reaggregate_component_dimensions={component.name: "test.parent.date_id"}, + ) + + result = build_grain_group_sql( + ctx, + grain_group, + [resolved_dimension], + {"balance": 1}, + output_dimension_refs={"test.parent.date_id"}, + ) + + assert result.grain == ["date_id"] + + +def test_build_window_metric_grain_groups_skips_missing_base_components(): + """Window grain planning skips parent groups with no decomposed components.""" + ctx = BuildContext(session=SimpleNamespace(), metrics=["window"], dimensions=[]) + ctx.parent_map = {"window": ["missing_base"]} + existing_group = GrainGroupSQL( + query=parse("SELECT missing_base FROM test_parent_0"), + columns=[], + grain=[], + aggregability=Aggregability.FULL, + metrics=["missing_base"], + parent_name="test.parent", + ) + + assert ( + build_window_metric_grain_groups( + ctx, + {"window": {"v3.date.week"}}, + [existing_group], + {}, + ) + == [] + ) + + +def test_build_window_metric_grain_groups_skips_when_parent_node_missing(): + """Window grain planning skips groups whose source parent is not loaded.""" + ctx = BuildContext(session=SimpleNamespace(), metrics=["window"], dimensions=[]) + ctx.nodes = {"missing_base": _metric_node("missing_base")} + ctx.parent_map = {"window": ["missing_base"]} + existing_group = GrainGroupSQL( + query=parse("SELECT missing_base FROM missing_parent_0"), + columns=[], + grain=[], + aggregability=Aggregability.FULL, + metrics=["missing_base"], + parent_name="missing.parent", + ) + + assert ( + build_window_metric_grain_groups( + ctx, + {"window": {"v3.date.week"}}, + [existing_group], + { + "window": _decomposed_metric("window"), + "missing_base": _decomposed_metric("missing_base"), + }, + ) + == [] + ) + + +@pytest.mark.asyncio +async def test_setup_build_context_skips_duplicate_internal_reaggregate_dimension( + monkeypatch, +): + """setup_build_context does not duplicate internal reaggregate dimensions.""" + load_calls = [] + + async def fake_load_nodes(ctx): + load_calls.append(list(ctx.dimensions)) + ctx.nodes = {"balance": _metric_node("balance")} + + async def fake_decompose_and_group_metrics(ctx): + return [], {"balance": _decomposed_metric("balance")} + + monkeypatch.setattr(builder_module, "load_nodes", fake_load_nodes) + monkeypatch.setattr( + builder_module, + "decompose_and_group_metrics", + fake_decompose_and_group_metrics, + ) + monkeypatch.setattr( + builder_module, + "add_dimensions_from_metric_expressions", + lambda _ctx, _decomposed_metrics: None, + ) + monkeypatch.setattr( + builder_module, + "missing_reaggregate_dimensions", + lambda _decomposed_metrics, _requested_dimensions: ["v3.date.date_id"], + ) + + ctx = await setup_build_context( + session=SimpleNamespace(), + metrics=["balance"], + dimensions=["v3.date.date_id"], + ) + + assert ctx.dimensions == ["v3.date.date_id"] + assert load_calls == [["v3.date.date_id"], ["v3.date.date_id"]] diff --git a/datajunction-server/tests/database/preaggregation_test.py b/datajunction-server/tests/database/preaggregation_test.py index 0364ab65f..fbe61eaea 100644 --- a/datajunction-server/tests/database/preaggregation_test.py +++ b/datajunction-server/tests/database/preaggregation_test.py @@ -716,3 +716,81 @@ async def test_find_matching_no_candidates( }, ) assert result is None + + +class TestMeasureIdentityParams: + """ + Tuning parameters participate in measure identity. + """ + + def test_existing_tokens_are_unchanged(self): + """ + A component without params produces the token byte-for-byte. + """ + assert measure_identity_token("abc123", "SUM") == "abc123:SUM" + assert measure_identity_token("abc123", "SUM", None) == "abc123:SUM" + assert measure_identity_token("abc123", "SUM", {}) == "abc123:SUM" + assert measure_identity_token("abc123", None) == "abc123:" + assert measure_identity_token("abc123", " count ") == "abc123:COUNT" + + def test_differing_params_differ(self): + """Two sketches over one column at different accuracies are distinct.""" + low = measure_identity_token("abc123", "TDIGEST_AGG", {"compression": 100}) + high = measure_identity_token("abc123", "TDIGEST_AGG", {"compression": 1000}) + assert low != high + # ...and neither collides with the un-parameterized token. + assert low != measure_identity_token("abc123", "TDIGEST_AGG") + + def test_params_are_order_and_spelling_insensitive(self): + """ + One sketch declaration yields one identity. Key order and type variations (e.g. 200 vs 200.0) do not change the identity token. + """ + assert measure_identity_token( + "abc123", + "TDIGEST_AGG", + {"compression": 200, "k": 12}, + ) == measure_identity_token( + "abc123", + "TDIGEST_AGG", + {"k": 12, "compression": 200}, + ) + assert measure_identity_token( + "abc123", + "TDIGEST_AGG", + {"compression": 200.0}, + ) == measure_identity_token("abc123", "TDIGEST_AGG", {"compression": 200}) + # Fractional values are preserved. + assert measure_identity_token( + "abc123", + "TDIGEST_AGG", + {"compression": 200.5}, + ) != measure_identity_token("abc123", "TDIGEST_AGG", {"compression": 200}) + + def test_get_measure_identities_carries_params(self): + """Stored measures round-trip their params into the identity set.""" + plain = make_measure("latency_sum", "latency_ms") + sketched = make_measure( + "latency_tdigest", + "latency_ms", + aggregation="TDIGEST_AGG", + merge="TDIGEST_AGG", + ) + sketched.params = {"compression": 200} + identities = get_measure_identities([plain, sketched]) + assert identities == { + measure_identity_token(compute_expression_hash("latency_ms"), "SUM"), + measure_identity_token( + compute_expression_hash("latency_ms"), + "TDIGEST_AGG", + {"compression": 200}, + ), + } + # Ensure the sketch identity distinguishes different compressions. + assert ( + measure_identity_token( + compute_expression_hash("latency_ms"), + "TDIGEST_AGG", + {"compression": 1000}, + ) + not in identities + ) diff --git a/datajunction-server/tests/internal/deployment/orchestration_test.py b/datajunction-server/tests/internal/deployment/orchestration_test.py index a163ba5ab..974895046 100644 --- a/datajunction-server/tests/internal/deployment/orchestration_test.py +++ b/datajunction-server/tests/internal/deployment/orchestration_test.py @@ -27,6 +27,7 @@ from datajunction_server.internal.deployment.orchestrator import ( DeploymentOrchestrator, DeploymentPlan, + DeploymentTimer, ResourceRegistry, column_changed, tag_needs_update, @@ -82,6 +83,26 @@ def mock_deployment_context(current_user: User): return context +def test_deployment_timer_logs_unaccounted_overhead(monkeypatch, caplog): + """ + Deployment timing summary includes unaccounted overhead when it is meaningful. + """ + timer = DeploymentTimer() + timer.record("validate", 10, "1 node") + monkeypatch.setattr( + "datajunction_server.internal.deployment.orchestrator.time.perf_counter", + lambda: timer._start + 0.2, + ) + + with caplog.at_level( + "INFO", + logger="datajunction_server.internal.deployment.orchestrator", + ): + timer.log_summary("default", "deployment-1") + + assert "(unaccounted overhead)" in caplog.text + + @pytest.fixture def sample_deployment_spec(): """Sample deployment specification for testing""" diff --git a/datajunction-server/tests/internal/deployment/validation_test.py b/datajunction-server/tests/internal/deployment/validation_test.py index c252035a5..6a270b7c7 100644 --- a/datajunction-server/tests/internal/deployment/validation_test.py +++ b/datajunction-server/tests/internal/deployment/validation_test.py @@ -793,6 +793,97 @@ async def test_invalid_short_name_not_in_parent_columns( assert err.debug is not None assert "no_such_col" in err.debug["invalid_required_dimensions"] + @pytest.mark.asyncio + async def test_valid_reaggregate_dimension_full_path( + self, + session: AsyncSession, + parent_node: Node, + ): + """reaggregate.rules[].dimension full-path column found in dim nodes is valid.""" + dim_node = self._make_dim_node("test.dim", ["dateint"]) + context = self._make_context(session, parent_node) + spec = MetricSpec( + name="test.metric", + query="SELECT SUM(value) FROM test.parent", + reaggregate={ + "rules": [ + { + "dimension": "test.dim.dateint", + "fn": "last_value", + }, + ], + }, + ) + validator = NodeSpecBulkValidator(context) + validator._all_dim_nodes = {**context.dependency_nodes, "test.dim": dim_node} + result = validator.validate_query_node(spec) + + assert result.status == NodeStatus.VALID + error_codes = [e.code for e in result.errors] + assert ErrorCode.INVALID_COLUMN not in error_codes + + @pytest.mark.asyncio + async def test_invalid_reaggregate_dimension_column( + self, + session: AsyncSession, + parent_node: Node, + ): + """reaggregate.rules[].dimension full-path column absent from dim node is invalid.""" + dim_node = self._make_dim_node("test.dim", ["dateint"]) + context = self._make_context(session, parent_node) + spec = MetricSpec( + name="test.metric", + query="SELECT SUM(value) FROM test.parent", + reaggregate={ + "rules": [ + { + "dimension": "test.dim.nonexistent_col", + "fn": "last_value", + }, + ], + }, + ) + validator = NodeSpecBulkValidator(context) + validator._all_dim_nodes = {**context.dependency_nodes, "test.dim": dim_node} + result = validator.validate_query_node(spec) + + assert result.status == NodeStatus.INVALID + err = next(e for e in result.errors if e.code == ErrorCode.INVALID_COLUMN) + assert err.debug is not None + assert "test.dim.nonexistent_col" in err.debug["invalid_reaggregate_dimensions"] + + @pytest.mark.asyncio + async def test_invalid_reaggregate_function( + self, + session: AsyncSession, + parent_node: Node, + ): + """unsupported dimension-specific reaggregate functions are invalid.""" + context = self._make_context(session, parent_node) + spec = MetricSpec( + name="test.metric", + query="SELECT SUM(value) FROM test.parent", + reaggregate={ + "rules": [ + { + "dimension": "id", + "fn": "sum", + }, + ], + }, + ) + validator = NodeSpecBulkValidator(context) + validator._all_dim_nodes = dict(context.dependency_nodes) + result = validator.validate_query_node(spec) + + assert result.status == NodeStatus.INVALID + err = next( + e + for e in result.errors + if e.code == ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION + ) + assert err.debug == {"invalid_reaggregate_functions": ["sum"]} + @pytest.mark.asyncio async def test_no_required_dimensions_is_noop( self, @@ -866,6 +957,64 @@ async def test_prefetch_fetches_dim_node_from_db( col_names = {c.name for c in fetched.current.columns} assert "region" in col_names + @pytest.mark.asyncio + async def test_prefetch_fetches_reaggregate_dim_node_from_db( + self, + session: AsyncSession, + parent_node: Node, + dim_node_in_db: Node, + ): + """_prefetch_required_dimension_nodes includes reaggregate dimensions.""" + context = self._make_context(session, parent_node) + spec = MetricSpec( + name="test.metric", + query="SELECT SUM(value) FROM test.parent", + reaggregate={ + "rules": [ + { + "dimension": "test.external_dim.region", + "fn": "last_value", + }, + ], + }, + ) + validator = NodeSpecBulkValidator(context) + + await validator._prefetch_required_dimension_nodes([spec]) + + assert "test.external_dim" in validator._all_dim_nodes + fetched = validator._all_dim_nodes["test.external_dim"] + assert fetched.current is not None + assert {c.name for c in fetched.current.columns} == {"region"} + + @pytest.mark.asyncio + async def test_prefetch_skips_short_reaggregate_dimension( + self, + session: AsyncSession, + parent_node: Node, + ): + """Short-form reaggregate dimensions do not trigger dimension-node prefetch.""" + context = self._make_context(session, parent_node) + spec = MetricSpec( + name="test.metric", + query="SELECT SUM(value) FROM test.parent", + reaggregate={ + "rules": [ + { + "dimension": "value", + "fn": "last_value", + }, + ], + }, + ) + validator = NodeSpecBulkValidator(context) + + await validator._prefetch_required_dimension_nodes([spec]) + + assert set(validator._all_dim_nodes.keys()) == set( + context.dependency_nodes.keys(), + ) + @pytest.mark.asyncio async def test_prefetch_with_no_required_dimensions( self, diff --git a/datajunction-server/tests/internal/namespaces_test.py b/datajunction-server/tests/internal/namespaces_test.py index 1c563211d..839f99086 100644 --- a/datajunction-server/tests/internal/namespaces_test.py +++ b/datajunction-server/tests/internal/namespaces_test.py @@ -3,6 +3,7 @@ """ from datetime import UTC, datetime +from types import SimpleNamespace import pytest from ruamel.yaml import YAML @@ -22,6 +23,7 @@ _merge_list_with_key, _merge_yaml_preserving_comments, create_or_reactivate_namespace, + get_node_specs_for_export, node_spec_to_yaml, provision_namespace_boundary, ) @@ -1108,6 +1110,81 @@ def test_no_yaml_document_start_marker(self): "query: SELECT SUM(rev) FROM ns.transforms.t", ] + def test_metric_reaggregate_exports_dimension_rules(self): + """metric reaggregate exports the rule list without null future fields""" + spec = MetricSpec( + name="ns.metrics.daily_balance", + node_type=NodeType.METRIC, + query="SELECT SUM(balance) FROM ns.transforms.accounts", + reaggregate={ + "rules": [ + { + "dimension": "${prefix}v3.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + ) + assert node_spec_to_yaml(spec).splitlines() == [ + "name: ns.metrics.daily_balance", + "node_type: metric", + "mode: published", + "query: SELECT SUM(balance) FROM ns.transforms.accounts", + "reaggregate:", + " rules:", + " - dimension: ${prefix}v3.date.date_id[order]", + " fn: last_value", + ] + + @pytest.mark.asyncio + async def test_metric_reaggregate_export_injects_dimension_prefix( + self, + monkeypatch, + ): + """namespace export parameterizes reaggregate rule dimensions.""" + spec = MetricSpec( + name="demo.branch.metrics.daily_balance", + node_type=NodeType.METRIC, + query="SELECT SUM(balance) FROM demo.branch.transforms.accounts", + reaggregate={ + "rules": [ + { + "dimension": "demo.branch.dims.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + ) + + async def to_spec(_session): + return spec + + fake_node = SimpleNamespace( + name="demo.branch.metrics.daily_balance", + current=SimpleNamespace(parents=[], status="valid"), + to_spec=to_spec, + ) + + async def namespace_get(_session, _namespace, raise_if_not_exists=False): + return SimpleNamespace(parent_namespace="demo.main") + + async def list_all_nodes(_session, _namespace, options=None): + return [fake_node] + + monkeypatch.setattr(NodeNamespace, "get", namespace_get) + monkeypatch.setattr(NodeNamespace, "list_all_nodes", list_all_nodes) + + exported = await get_node_specs_for_export(SimpleNamespace(), "demo.branch") + + assert exported[0].name == "${prefix}metrics.daily_balance" + assert exported[0].query == ( + "SELECT SUM(balance) FROM ${prefix}transforms.accounts" + ) + assert ( + exported[0].reaggregate.rules[0].dimension + == "${prefix}dims.date.date_id[order]" + ) + def test_multiline_query_uses_literal_block_style(self): """multiline queries are serialized with |- literal block style""" spec = MetricSpec( diff --git a/datajunction-server/tests/internal/node_validation_test.py b/datajunction-server/tests/internal/node_validation_test.py index 5711a6ea7..7f765c7fe 100644 --- a/datajunction-server/tests/internal/node_validation_test.py +++ b/datajunction-server/tests/internal/node_validation_test.py @@ -1019,6 +1019,133 @@ async def test_validate_node_data_v2_flags_invalid_required_dimensions( ), [(e.code, e.message) for e in validator.errors] +@pytest.mark.asyncio +async def test_validate_node_data_v2_flags_invalid_reaggregate_dimensions( + session: AsyncSession, + user: User, +): + """reaggregate dimensions must resolve to columns on parent dimension nodes.""" + from datajunction_server.errors import ErrorCode + from datajunction_server.internal.validation import validate_node_data_v2 + + source = Node( + name="test.v2_reagg_dim_parent", + type=NodeType.SOURCE, + created_by_id=user.id, + current_version="v1.0", + ) + source_rev = NodeRevision( + name="test.v2_reagg_dim_parent", + display_name="reaggregate dim parent", + type=NodeType.SOURCE, + query=None, + status=NodeStatus.VALID, + version="v1.0", + node=source, + columns=[Column(name="id", type=ct.BigIntType(), order=0)], + created_by_id=user.id, + ) + dim = Node( + name="test.v2_reagg_dim", + type=NodeType.DIMENSION, + created_by_id=user.id, + current_version="v1.0", + ) + dim_rev = NodeRevision( + name="test.v2_reagg_dim", + display_name="tiny reaggregate dim", + type=NodeType.DIMENSION, + query="SELECT 1 AS id", + status=NodeStatus.VALID, + version="v1.0", + node=dim, + columns=[Column(name="id", type=ct.BigIntType(), order=0)], + created_by_id=user.id, + ) + session.add_all([source, source_rev, dim, dim_rev]) + await session.commit() + + child = NodeRevision( + name="test.v2_reagg_dim_child", + display_name="reaggregate dim child", + type=NodeType.METRIC, + query="SELECT SUM(id) FROM test.v2_reagg_dim_parent", + status=NodeStatus.VALID, + reaggregate={ + "rules": [ + { + "dimension": "test.v2_reagg_dim.ghost_col", + "fn": "last_value", + }, + ], + }, + ) + validator = await validate_node_data_v2(child, session) + + assert validator.status == NodeStatus.INVALID + assert any( + err.code == ErrorCode.INVALID_COLUMN and "reaggregate dimensions" in err.message + for err in validator.errors + ), [(e.code, e.message) for e in validator.errors] + + +@pytest.mark.asyncio +async def test_validate_node_data_v2_flags_unsupported_reaggregate_function( + session: AsyncSession, + user: User, +): + """dimension-specific reaggregate only accepts supported collapse functions.""" + from datajunction_server.errors import ErrorCode + from datajunction_server.internal.validation import validate_node_data_v2 + + source = Node( + name="test.v2_reagg_fn_parent", + type=NodeType.SOURCE, + created_by_id=user.id, + current_version="v1.0", + ) + source_rev = NodeRevision( + name="test.v2_reagg_fn_parent", + display_name="reaggregate function parent", + type=NodeType.SOURCE, + query=None, + status=NodeStatus.VALID, + version="v1.0", + node=source, + columns=[ + Column(name="id", type=ct.BigIntType(), order=0), + Column(name="order_date", type=ct.BigIntType(), order=1), + ], + created_by_id=user.id, + ) + session.add_all([source, source_rev]) + await session.commit() + + child = NodeRevision( + name="test.v2_reagg_fn_child", + display_name="reaggregate function child", + type=NodeType.METRIC, + query="SELECT SUM(id) FROM test.v2_reagg_fn_parent", + status=NodeStatus.VALID, + reaggregate={ + "rules": [ + { + "dimension": "order_date", + "fn": "sum", + }, + ], + }, + ) + validator = await validate_node_data_v2(child, session) + + assert validator.status == NodeStatus.INVALID + assert any( + err.code == ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION + and err.debug == {"invalid_reaggregate_functions": ["sum"]} + for err in validator.errors + ), [(e.code, e.message, e.debug) for e in validator.errors] + + @pytest.mark.asyncio async def test_validate_node_data_v2_cross_fact_metrics_no_shared_dims( session: AsyncSession, diff --git a/datajunction-server/tests/internal/nodes/derive_frozen_measures_test.py b/datajunction-server/tests/internal/nodes/derive_frozen_measures_test.py index 292be4521..a62de0e7e 100644 --- a/datajunction-server/tests/internal/nodes/derive_frozen_measures_test.py +++ b/datajunction-server/tests/internal/nodes/derive_frozen_measures_test.py @@ -16,7 +16,16 @@ from datajunction_server.database.measure import FrozenMeasure from datajunction_server.database.node import Node, NodeRevision from datajunction_server.database.user import OAuthProvider, User -from datajunction_server.internal.nodes import derive_frozen_measures_bulk +from datajunction_server.errors import DJInvalidInputException +from datajunction_server.internal.nodes import ( + _raise_if_frozen_measure_conflicts, + derive_frozen_measures_bulk, +) +from datajunction_server.models.decompose import ( + Aggregability, + AggregationRule, + MetricComponent, +) from datajunction_server.models.node import NodeStatus from datajunction_server.models.node_type import NodeType @@ -62,6 +71,7 @@ async def _make_metric( name: str, query: str, parents: list[Node], + reaggregate: dict[str, object] | None = None, ) -> Node: node = Node( name=name, @@ -75,6 +85,7 @@ async def _make_metric( type=NodeType.METRIC, version="v1.0", query=query, + reaggregate=reaggregate, status=NodeStatus.VALID, parents=parents, created_by_id=user.id, @@ -123,6 +134,189 @@ async def test_base_metric_populates_derived_expression_and_measure( assert any(fm.aggregation == "SUM" for fm in metric.current.frozen_measures) +@pytest.mark.asyncio +async def test_frozen_measure_name_collision_allows_reaggregate_only_difference( + session: AsyncSession, + user: User, +): + """Metric-level reaggregate metadata does not change measure identity.""" + src = await _make_source( + session, + user, + "src_reaggregate_collision", + [Column(name="semi_amount", type=ct.DoubleType(), order=0)], + ) + reaggregate = await _make_metric( + session, + user, + "m.semi_amount_eod", + "SELECT SUM(semi_amount) FROM src_reaggregate_collision", + [src], + reaggregate={ + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + }, + ) + additive = await _make_metric( + session, + user, + "m.semi_amount_total", + "SELECT SUM(semi_amount) FROM src_reaggregate_collision", + [src], + ) + await session.refresh(reaggregate, ["current"]) + await session.refresh(additive, ["current"]) + + await derive_frozen_measures_bulk(session, [additive.current.id]) + await session.commit() + + await session.refresh(additive.current, ["frozen_measures"]) + assert all(fm.rule.reaggregate is None for fm in additive.current.frozen_measures) + + await derive_frozen_measures_bulk(session, [reaggregate.current.id]) + await session.commit() + await session.refresh(reaggregate.current, ["frozen_measures"]) + + assert {fm.name for fm in additive.current.frozen_measures} == { + fm.name for fm in reaggregate.current.frozen_measures + } + + +@pytest.mark.asyncio +async def test_reaggregate_rule_is_not_persisted_on_shared_frozen_measure( + session: AsyncSession, + user: User, +): + """FrozenMeasure.rule stays metric-independent even for semi-additive metrics.""" + src = await _make_source( + session, + user, + "src_reaggregate_storage", + [Column(name="semi_amount", type=ct.DoubleType(), order=0)], + ) + reaggregate = await _make_metric( + session, + user, + "m.semi_amount_snapshot", + "SELECT SUM(semi_amount) FROM src_reaggregate_storage", + [src], + reaggregate={ + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + }, + ) + await session.refresh(reaggregate, ["current"]) + + await derive_frozen_measures_bulk(session, [reaggregate.current.id]) + await session.commit() + + await session.refresh(reaggregate.current, ["frozen_measures"]) + assert all( + fm.rule.reaggregate is None for fm in reaggregate.current.frozen_measures + ) + + +def test_frozen_measure_conflict_rejects_different_measure_identity(): + """FrozenMeasure name collisions fail when the metric-independent rule differs.""" + frozen_measure = FrozenMeasure( + name="amount_sum", + upstream_revision_id=1, + expression="amount", + aggregation="SUM", + rule=AggregationRule(type=Aggregability.FULL), + ) + measure = MetricComponent( + name="amount_sum", + expression="discounted_amount", + aggregation="SUM", + rule=AggregationRule(type=Aggregability.FULL), + ) + + with pytest.raises(DJInvalidInputException, match="already exists"): + _raise_if_frozen_measure_conflicts(frozen_measure, measure) + + +def test_frozen_measure_conflict_rejects_different_tuning_params(): + """ + A name collision fails when only the sketch tuning parameters differ. + + Component names are hashed from the expression and its source, not from + params, so a p95 at compression=200 and one at compression=1000 over the + same column collide on name. Without this the second metric would silently + bind the frozen measure -- and the materialized sketch -- built at the + first one's accuracy. + """ + frozen_measure = FrozenMeasure( + name="latency_tdigest", + upstream_revision_id=1, + expression="latency_ms", + aggregation="TDIGEST_AGG", + params={"compression": 200}, + rule=AggregationRule(type=Aggregability.FULL), + ) + measure = MetricComponent( + name="latency_tdigest", + expression="latency_ms", + aggregation="TDIGEST_AGG", + params={"compression": 1000}, + rule=AggregationRule(type=Aggregability.FULL), + ) + + with pytest.raises(DJInvalidInputException, match="tuning parameters"): + _raise_if_frozen_measure_conflicts(frozen_measure, measure) + + +def test_frozen_measure_reuse_allows_matching_params(): + """ + Identical params are reusable, and absent-vs-empty is not a difference. + + Almost every measure has no params at all; a stored ``None`` meeting a + freshly-extracted ``{}`` must not read as a conflict and start rejecting + ordinary metrics. + """ + frozen_measure = FrozenMeasure( + name="latency_tdigest", + upstream_revision_id=1, + expression="latency_ms", + aggregation="TDIGEST_AGG", + params={"compression": 200}, + rule=AggregationRule(type=Aggregability.FULL), + ) + measure = MetricComponent( + name="latency_tdigest", + expression="latency_ms", + aggregation="TDIGEST_AGG", + params={"compression": 200}, + rule=AggregationRule(type=Aggregability.FULL), + ) + _raise_if_frozen_measure_conflicts(frozen_measure, measure) + + unparameterized = FrozenMeasure( + name="amount_sum", + upstream_revision_id=1, + expression="amount", + aggregation="SUM", + params=None, + rule=AggregationRule(type=Aggregability.FULL), + ) + empty_params = MetricComponent( + name="amount_sum", + expression="amount", + aggregation="SUM", + params={}, + rule=AggregationRule(type=Aggregability.FULL), + ) + _raise_if_frozen_measure_conflicts(unparameterized, empty_params) + + @pytest.mark.asyncio async def test_derived_metric_expands_parent_cache( session: AsyncSession, diff --git a/datajunction-server/tests/models/deployment_test.py b/datajunction-server/tests/models/deployment_test.py index 91507b62d..a371e2f61 100644 --- a/datajunction-server/tests/models/deployment_test.py +++ b/datajunction-server/tests/models/deployment_test.py @@ -1706,7 +1706,7 @@ def fingerprint(spec: NodeSpec) -> SemanticFingerprint: "source": "71dcbc388988c2bdd850670427710384687b58565ee38ca392dc220adfed868d", "transform": "978e692880c7bcfb1bd78ece85895a1ec1558e85377b064a8dac3f3719cff2a5", "dimension": "f7b3c87a61fdadf9997432fd9334befdf43f2488874ef555e3f7d4c4ba86e3e1", - "metric": "a3fde7af5dbd00d194805af33fc213bca52244c7cdedf1f7363ec52d2f6d4116", + "metric": "f0b5356d75b1f6a39a2809581a9167b9e1298a41980ee8e83040cf6bd0996851", "cube": "9b0a56d974d1e3769bc2db94e2cfbae7a6a4839f664eebd2a4387ef112ceea81", } @@ -1853,6 +1853,38 @@ def test_semantic_fingerprint_renders_prefixes_and_normalizes_sql(): assert fingerprint(explicit) != fingerprint(implicit) +def test_metric_spec_equality_compares_rendered_reaggregate(): + """Parameterized reaggregate dimensions should not cause perpetual deploys.""" + parameterized = MetricSpec( + namespace="analytics", + name="daily_balance", + query="SELECT SUM(balance) FROM analytics.daily_balances", + reaggregate={ + "rules": [ + { + "dimension": "${prefix}date.date_id[order]", + "fn": "last_value", + }, + ], + }, + ) + rendered = MetricSpec( + namespace="analytics", + name="daily_balance", + query="SELECT SUM(balance) FROM analytics.daily_balances", + reaggregate={ + "rules": [ + { + "dimension": "analytics.date.date_id[order]", + "fn": "last_value", + }, + ], + }, + ) + + assert parameterized == rendered + + def test_semantic_diff_and_fingerprint_share_change_rules(): original = TransformSpec(name="node", query="SELECT id AS value FROM source") formatted = TransformSpec( @@ -1958,6 +1990,7 @@ def test_metric_presentation_fields_preserve_semantic_fingerprint(): "query", "columns", "required_dimensions", + "reaggregate", } ) assert all( diff --git a/datajunction-server/tests/models/node_test.py b/datajunction-server/tests/models/node_test.py index 1489d3928..3a630811c 100644 --- a/datajunction-server/tests/models/node_test.py +++ b/datajunction-server/tests/models/node_test.py @@ -227,6 +227,30 @@ def test_extra_validation() -> None: "bound dimensions which are only for metrics." ) + node = Node(name="A", type=NodeType.TRANSFORM, current_version="1") + node_revision = NodeRevision( + name=node.name, + type=node.type, + node=node, + version="1", + query="SELECT * FROM B", + reaggregate={ + "rules": [ + { + "dimension": "B.date_id", + "fn": "last_value", + }, + ], + }, + ) + with pytest.raises(Exception) as excinfo: + node_revision.extra_validation() + + assert str(excinfo.value) == ( + "Node A of type transform cannot have " + "reaggregate settings which are only for metrics." + ) + def test_merging_availability_simple_no_partitions() -> None: """ diff --git a/datajunction-server/tests/models/reaggregate_test.py b/datajunction-server/tests/models/reaggregate_test.py new file mode 100644 index 000000000..b1ce51c28 --- /dev/null +++ b/datajunction-server/tests/models/reaggregate_test.py @@ -0,0 +1,186 @@ +"""Tests for reaggregation models.""" + +import pytest +from pydantic import ValidationError + +from datajunction_server.models import reaggregate as reaggregate_module +from datajunction_server.models.reaggregate import ( + DimensionReaggregateRule, + ReaggregateSpec, + ReaggregationFunction, + dimension_reaggregate_rules, + dump_reaggregate_spec, + is_parameterized_reaggregate_function, + parse_reaggregate_spec, + unsupported_dimension_reaggregate_functions, +) + + +def test_dump_reaggregate_spec_from_dict(): + """ + Reaggregate specs passed as dictionaries are validated and serialized. + """ + assert dump_reaggregate_spec( + { + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + }, + ) == { + "fn": None, + "weight": None, + "params": None, + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + } + + +def test_dump_reaggregate_spec_from_model(): + """ + Reaggregate spec models are serialized to JSON-compatible dictionaries. + """ + assert dump_reaggregate_spec( + ReaggregateSpec( + rules=[ + DimensionReaggregateRule( + dimension="default.date_dim.date", + fn=ReaggregationFunction.LAST_VALUE, + ), + ], + ), + ) == { + "fn": None, + "weight": None, + "params": None, + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + } + + +def test_dump_reaggregate_spec_none(): + """ + Missing reaggregate specs pass through unchanged. + """ + assert dump_reaggregate_spec(None) is None + + +def test_parse_reaggregate_spec_from_model(): + """ + Parsed model inputs are returned unchanged. + """ + spec = ReaggregateSpec( + rules=[ + DimensionReaggregateRule( + dimension="default.date_dim.date", + fn=ReaggregationFunction.LAST_VALUE, + ), + ], + ) + + assert parse_reaggregate_spec(spec) is spec + + +def test_dimension_reaggregate_rules_empty_for_none(): + """ + Empty reaggregate specs have no dimension-specific rules. + """ + assert dimension_reaggregate_rules(None) == [] + + +def test_unsupported_dimension_reaggregate_functions_handles_empty_and_invalid(): + """ + Unsupported dimension collapse functions are reported from parsed specs. + """ + assert unsupported_dimension_reaggregate_functions(None) == [] + assert unsupported_dimension_reaggregate_functions( + { + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "sum", + }, + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + }, + ) == ["sum"] + + +def test_params_rejected_for_unparameterized_function(): + """ + Functions without tuneable parameters reject `params`. + """ + with pytest.raises(ValidationError) as excinfo: + ReaggregateSpec(fn=ReaggregationFunction.SUM, params={"compression": 200}) + assert "`sum` does not accept parameters" in str(excinfo.value) + assert "compression" in str(excinfo.value) + + +def test_params_rejected_when_no_function_given(): + """ + `params` without an `fn` is rejected. + """ + with pytest.raises(ValidationError) as excinfo: + ReaggregateSpec(params={"k": 256}) + assert "no reaggregation function does not accept parameters" in str(excinfo.value) + + +def test_empty_params_allowed(): + """ + An absent or empty `params` is fine on any function. + """ + assert ReaggregateSpec(fn=ReaggregationFunction.SUM).params is None + assert ReaggregateSpec(fn=ReaggregationFunction.SUM, params={}).params == {} + + +def test_params_accepted_for_parameterized_function(monkeypatch): + """ + Registering a function as parameterized enables `params` assignment. + """ + monkeypatch.setattr( + reaggregate_module, + "PARAMETERIZED_REAGGREGATE_FUNCTIONS", + frozenset({ReaggregationFunction.AVG}), + ) + spec = ReaggregateSpec(fn=ReaggregationFunction.AVG, params={"compression": 200}) + assert spec.params == {"compression": 200} + assert dump_reaggregate_spec(spec)["params"] == {"compression": 200} + + +def test_no_function_is_parameterized_today(): + """ + Every currently supported function is fully specified by its name. + """ + assert not any( + is_parameterized_reaggregate_function(fn) for fn in ReaggregationFunction + ) + + +def test_params_round_trip_through_parse(): + """ + `params` survives dict -> model -> dict without loss. + """ + monkeypatched = frozenset({ReaggregationFunction.AVG}) + original = reaggregate_module.PARAMETERIZED_REAGGREGATE_FUNCTIONS + reaggregate_module.PARAMETERIZED_REAGGREGATE_FUNCTIONS = monkeypatched + try: + spec = parse_reaggregate_spec({"fn": "avg", "params": {"compression": 200}}) + assert spec.params == {"compression": 200} + assert parse_reaggregate_spec(dump_reaggregate_spec(spec)).params == { + "compression": 200, + } + finally: + reaggregate_module.PARAMETERIZED_REAGGREGATE_FUNCTIONS = original diff --git a/datajunction-server/tests/sql/decompose_test.py b/datajunction-server/tests/sql/decompose_test.py index b2c93644b..d3afdbe7c 100644 --- a/datajunction-server/tests/sql/decompose_test.py +++ b/datajunction-server/tests/sql/decompose_test.py @@ -2,11 +2,14 @@ Tests for ``datajunction_server.sql.decompose``. """ +from types import SimpleNamespace + import pytest import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession from datajunction_server.database.node import Node, NodeRelationship, NodeRevision +from datajunction_server.errors import DJInvalidInputException from datajunction_server.models.cube_materialization import ( Aggregability, AggregationRule, @@ -14,6 +17,11 @@ ) from datajunction_server.models.engine import Dialect from datajunction_server.models.node_type import NodeType +from datajunction_server.models.reaggregate import ( + DimensionReaggregateRule, + ReaggregateSpec, + ReaggregationFunction, +) from datajunction_server.sql import functions as dj_functions from datajunction_server.sql.decompose import ( DUPLICATION_INVARIANT_AGGREGATIONS, @@ -62,7 +70,12 @@ async def create_metric(session: AsyncSession, current_user, parent_node): """Fixture to create a metric node with a query.""" created_metrics: list[NodeRevision] = [] - async def _create(query: str, name: str | None = None, parent=None): + async def _create( + query: str, + name: str | None = None, + parent=None, + reaggregate: dict[str, object] | None = None, + ): parent_to_use = parent if parent else parent_node metric_name = name or f"test_metric_{len(created_metrics)}" @@ -81,6 +94,7 @@ async def _create(query: str, name: str | None = None, parent=None): name=metric_name, type=NodeType.METRIC, query=query, + reaggregate=reaggregate, created_by_id=current_user.id, ) session.add(metric_rev) @@ -120,6 +134,105 @@ async def test_simple_sum(session: AsyncSession, create_metric): ) +@pytest.mark.asyncio +async def test_reaggregate_sum_attaches_spec(session: AsyncSession, create_metric): + """ + Semi-additive declarations attach to the single extracted base measure. + """ + metric_rev = await create_metric( + "SELECT SUM(account_balance) FROM parent_node", + reaggregate={ + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + }, + ) + + extractor = MetricComponentExtractor(metric_rev.id) + measures, derived_sql = await extractor.extract(session) + + expected_spec = DimensionReaggregateRule( + dimension="default.date_dim.date", + fn=ReaggregationFunction.LAST_VALUE, + ) + assert measures == [ + MetricComponent( + name="account_balance_sum_8e611a76", + expression="account_balance", + aggregation="SUM", + merge="SUM", + rule=AggregationRule( + type=Aggregability.FULL, + reaggregate=expected_spec, + ), + ), + ] + assert_sql_equal( + str(derived_sql), + "SELECT SUM(account_balance_sum_8e611a76) FROM parent_node", + ) + + +@pytest.mark.asyncio +async def test_reaggregate_multiple_rules_not_supported( + session: AsyncSession, + create_metric, +): + """ + V1 semi-additive metrics support one dimension-specific rule. + """ + metric_rev = await create_metric( + "SELECT SUM(account_balance) FROM parent_node", + reaggregate={ + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + { + "dimension": "default.region_dim.region", + "fn": "first_value", + }, + ], + }, + ) + + extractor = MetricComponentExtractor(metric_rev.id) + with pytest.raises(DJInvalidInputException, match="exactly one rule"): + await extractor.extract(session) + + +@pytest.mark.asyncio +async def test_reaggregate_unsupported_dimension_function_not_supported( + session: AsyncSession, + create_metric, +): + """ + Dimension-specific reaggregate accepts only collapse-safe functions. + """ + metric_rev = await create_metric( + "SELECT SUM(account_balance) FROM parent_node", + reaggregate={ + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "sum", + }, + ], + }, + ) + + extractor = MetricComponentExtractor(metric_rev.id) + with pytest.raises( + DJInvalidInputException, + match="unsupported dimension reaggregation function", + ): + await extractor.extract(session) + + @pytest.mark.asyncio async def test_sum_with_cast(session: AsyncSession, create_metric): """ @@ -854,6 +967,50 @@ async def test_unsupported_aggregation_function(session: AsyncSession, create_me ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("query", "match"), + [ + ( + "SELECT AVG(sales_amount) FROM parent_node", + "exactly one component", + ), + ( + "SELECT COUNT(DISTINCT user_id) FROM parent_node", + "one non-distinct fully-aggregatable component", + ), + ( + "SELECT MEDIAN(sales_amount) FROM parent_node", + "decomposable aggregation", + ), + ], +) +async def test_unsupported_reaggregate_shapes( + session: AsyncSession, + create_metric, + query: str, + match: str, +): + """ + V1 semi-additive metrics only support one ordinary decomposable component. + """ + metric_rev = await create_metric( + query, + reaggregate={ + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + }, + ) + + extractor = MetricComponentExtractor(metric_rev.id) + with pytest.raises(DJInvalidInputException, match=match): + await extractor.extract(session) + + @pytest.mark.asyncio async def test_count_if(session: AsyncSession, create_metric): """ @@ -1539,7 +1696,12 @@ async def create_derived_metric(clean_session: AsyncSession, clean_current_user) session = clean_session current_user = clean_current_user - async def _create(name: str, query: str, base_metric_nodes: list[Node]): + async def _create( + name: str, + query: str, + base_metric_nodes: list[Node], + reaggregate: dict[str, object] | None = None, + ): metric_node = Node( name=name, type=NodeType.METRIC, @@ -1555,6 +1717,7 @@ async def _create(name: str, query: str, base_metric_nodes: list[Node]): name=name, type=NodeType.METRIC, query=query, + reaggregate=reaggregate, created_by_id=current_user.id, ) session.add(metric_rev) @@ -1571,6 +1734,116 @@ async def _create(name: str, query: str, base_metric_nodes: list[Node]): return _create +@pytest.mark.asyncio +async def test_derived_metric_reaggregate_not_supported_db_load( + clean_session: AsyncSession, + create_base_metric, + create_derived_metric, +): + """ + Semi-additive declarations are currently limited to base metrics. + """ + session = clean_session + balance_node, _ = await create_base_metric( + "default.balance", + "SELECT SUM(account_balance) FROM parent_node", + ) + _, derived_rev = await create_derived_metric( + "default.double_balance", + "SELECT default.balance * 2", + [balance_node], + reaggregate={ + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + }, + ) + + extractor = MetricComponentExtractor(derived_rev.id) + with pytest.raises( + DJInvalidInputException, + match="Derived metric `default.double_balance` declares reaggregate", + ): + await extractor.extract(session) + + +def test_derived_metric_reaggregate_not_supported_cache_load(): + """ + Cache-based extraction enforces the same base-metric-only constraint. + """ + reaggregate = { + "rules": [ + { + "dimension": "default.date_dim.date", + "fn": "last_value", + }, + ], + } + base_node = SimpleNamespace( + name="default.balance", + type=NodeType.METRIC, + current=SimpleNamespace( + query="SELECT SUM(account_balance) FROM parent_node", + reaggregate=None, + ), + ) + derived_node = SimpleNamespace( + name="default.double_balance", + type=NodeType.METRIC, + current=SimpleNamespace( + query="SELECT default.balance * 2", + reaggregate=reaggregate, + ), + ) + + extractor = MetricComponentExtractor(0) + with pytest.raises( + DJInvalidInputException, + match="Derived metric `default.double_balance` declares reaggregate", + ): + extractor._build_metric_data_from_cache( + derived_node, + { + "default.balance": base_node, + "default.double_balance": derived_node, + }, + {"default.double_balance": ["default.balance"]}, + ) + + +def test_normalize_aliases_leaves_other_namespaces_qualified(): + """ + Alias normalization only strips the primary parent alias from column refs. + """ + extractor = MetricComponentExtractor(0) + query = parse("SELECT p.value, other.value FROM parent_node p") + + normalized = extractor._normalize_aliases(query) + + assert_sql_equal(str(normalized), "SELECT value, other.value FROM parent_node") + + +def test_substitute_metric_references_ignores_non_metric_columns(): + """ + Derived metric substitution leaves ordinary columns in place. + """ + extractor = MetricComponentExtractor(0) + query = parse("SELECT default.balance + raw_value FROM parent_node") + + substituted = extractor._substitute_metric_references( + query, + {"default.balance": ([], "SUM(balance)")}, + ) + + assert_sql_equal( + str(substituted), + "SELECT SUM(balance) + raw_value FROM parent_node", + ) + + @pytest.mark.asyncio async def test_extract_derived_metric_revenue_per_order( clean_session: AsyncSession, @@ -2404,3 +2677,74 @@ async def test_sum_abs_decomposes(session: AsyncSession, create_metric): assert comp.rule.type == Aggregability.FULL assert "ABS" in comp.expression assert_sql_equal(str(derived_sql), f"SELECT SUM({comp.name}) FROM parent_node") + + +class TestReaggregateParams: + """ + Tests that sketch tuning parameters flow through to components. + """ + + @staticmethod + def _spec(params): + """A spec carrying params.""" + return ReaggregateSpec.model_construct( + fn=ReaggregationFunction.SUM, + weight=None, + rules=[], + params=params, + ) + + def test_params_reach_components(self): + extractor = MetricComponentExtractor(1) + components, _ = extractor._extract_base( + parse("SELECT SUM(latency_ms) FROM t"), + self._spec({"compression": 200}), + ) + assert [component.params for component in components] == [ + {"compression": 200}, + ] + + def test_params_reach_every_component_of_a_multi_component_metric(self): + """AVG decomposes to a sum and a count; both are the same sketch family.""" + extractor = MetricComponentExtractor(1) + components, _ = extractor._extract_base( + parse("SELECT AVG(latency_ms) FROM t"), + self._spec({"compression": 200}), + ) + assert len(components) == 2 + assert all(c.params == {"compression": 200} for c in components) + + def test_params_are_copied_not_shared(self): + """ + Each component receives an independent copy of its params dict. + """ + extractor = MetricComponentExtractor(1) + params = {"compression": 200} + components, _ = extractor._extract_base( + parse("SELECT AVG(latency_ms) FROM t"), + self._spec(params), + ) + params["compression"] = 999 + assert all(c.params == {"compression": 200} for c in components) + assert components[0].params is not components[1].params + + def test_no_params_leaves_components_untouched(self): + """Omitting params applies no configuration to components.""" + extractor = MetricComponentExtractor(1) + components, _ = extractor._extract_base( + parse("SELECT SUM(latency_ms) FROM t"), + self._spec(None), + ) + assert [component.params for component in components] == [None] + + def test_params_without_an_aggregating_component_is_rejected(self): + """ + Declaring params on a non-aggregating metric is rejected. + """ + extractor = MetricComponentExtractor(1) + with pytest.raises(DJInvalidInputException) as excinfo: + extractor._extract_base( + parse("SELECT COUNT(DISTINCT order_id) FROM t"), + self._spec({"compression": 200}), + ) + assert "requires an aggregating component" in str(excinfo.value) diff --git a/datajunction-server/tests/sql/functions_test.py b/datajunction-server/tests/sql/functions_test.py index 10a1290df..964d2815b 100644 --- a/datajunction-server/tests/sql/functions_test.py +++ b/datajunction-server/tests/sql/functions_test.py @@ -288,6 +288,33 @@ async def test_approx_percentile(session: AsyncSession): assert not exc.errors assert query_with_list.select.projection[0].type == ct.FloatType() # type: ignore + # The two-argument form -- `accuracy` is optional in Spark, and this is the + # spelling metric authors actually use. + query = parse("SELECT approx_percentile(10.0, 0.5)") + exc = DJException() + ctx = ast.CompileContext(session=session, exception=exc) + await query.compile(ctx) + assert not exc.errors + assert query.select.projection[0].type == ct.FloatType() # type: ignore + + query = parse("SELECT approx_percentile(10.0, array(0.5, 0.9))") + exc = DJException() + ctx = ast.CompileContext(session=session, exception=exc) + await query.compile(ctx) + assert not exc.errors + assert query.select.projection[0].type == ct.ListType( # type: ignore + element_type=ct.FloatType(), + ) + + # A double-typed percentage dispatches too: DoubleType and FloatType are + # siblings under FloatingBase, so a FloatType-only registration missed it. + query = parse("SELECT approx_percentile(10.0, CAST(0.5 AS DOUBLE))") + exc = DJException() + ctx = ast.CompileContext(session=session, exception=exc) + await query.compile(ctx) + assert not exc.errors + assert query.select.projection[0].type == ct.FloatType() # type: ignore + @pytest.mark.asyncio async def test_array(session: AsyncSession): diff --git a/datajunction-ui/src/app/pages/AddEditNodePage/ReaggregateFields.jsx b/datajunction-ui/src/app/pages/AddEditNodePage/ReaggregateFields.jsx new file mode 100644 index 000000000..d163361b9 --- /dev/null +++ b/datajunction-ui/src/app/pages/AddEditNodePage/ReaggregateFields.jsx @@ -0,0 +1,79 @@ +/** + * Semi-additive metric controls. + */ +import { ErrorMessage, Field, useField, useFormikContext } from 'formik'; +import { useContext, useEffect, useMemo, useState } from 'react'; +import DJClientContext from '../../providers/djclient'; +import { labelize } from '../../../utils/form'; +import { FormikSelect } from './FormikSelect'; + +const SEMI_ADDITIVE_FUNCTIONS = ['last_value', 'first_value', 'min', 'max']; + +export const ReaggregateFields = () => { + const djClient = useContext(DJClientContext).DataJunctionAPI; + const { values } = useFormikContext(); + const [dimensionField] = useField('reaggregate_dimension'); + const [dimensionOptions, setDimensionOptions] = useState([]); + + useEffect(() => { + const fetchData = async () => { + if (values.upstream_node) { + const data = await djClient.node(values.upstream_node); + setDimensionOptions( + data.columns.map(col => ({ + value: col.name, + label: col.name, + })), + ); + } else { + setDimensionOptions([]); + } + }; + fetchData().catch(console.error); + }, [djClient, values.upstream_node]); + + const selectOptions = useMemo(() => { + if ( + !dimensionField.value || + dimensionOptions.some(option => option.value === dimensionField.value) + ) { + return dimensionOptions; + } + return [ + { value: dimensionField.value, label: dimensionField.value }, + ...dimensionOptions, + ]; + }, [dimensionField.value, dimensionOptions]); + + return ( +
+ {firstReaggregateRule(metricInfo?.reaggregate) ? ( + <> + {reaggregateFunctionLabel( + firstReaggregateRule(metricInfo.reaggregate).fn, + )} + {' on '} + {dimensionNodePath( + firstReaggregateRule(metricInfo.reaggregate).dimension, + ) ? ( + + {firstReaggregateRule(metricInfo.reaggregate).dimension} + + ) : ( + firstReaggregateRule(metricInfo.reaggregate).dimension + )} + > + ) : ( + 'None' + )} +
+