Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/kumo-api/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ for HTTP-compatible clients to interact with Kumo AI's cloud services. Resource
schemas are defined as Pydantic dataclasses.

While it is possible to interface with Kumo directly using these datamodels,
users are recommended to install the Python SDK (via `pip install kumoai-studio`)
for a smoother experience.
most users will want a higher-level Python library built on them, such as
`kumo-engine` (`pip install kumo-engine`).

## Development

Expand Down
29 changes: 9 additions & 20 deletions packages/kumo-bigquery/src/kumo_bigquery/backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,10 @@ class BigQuerySourceBackend(_BigQuerySessions):

Args:
identity: A fallback identity, used only for connectors that name none
themselves. Accepts an :class:`~kumoai_studio.identity.Identity`, an
themselves. Accepts an identity object, an
identity ID, or the name it was created under. Leaving it unset is
the common case: a bound
:class:`~kumoai_studio.connector.BigQueryConnector` records the identity
it was created with.
the common case: a connector records the identity it was created
with.
mode: The read mode every table is read under, or ``None`` to take
whichever the reader offers first.
staging_uri: A ``gs://`` prefix this run may export Parquet to. Without
Expand All @@ -71,23 +70,13 @@ class BigQuerySourceBackend(_BigQuerySessions):
an error: no bucket is a complete configuration.

Example:
>>> import kumoai_studio as kumo # doctest: +SKIP
>>> from kumoapi.data_source import ( # doctest: +SKIP
... BigQueryIdentityConfig, CreateIdentityArgs,
... GoogleWorkloadIdentityAuthentication)
>>> identity = kumo.create_identity( # doctest: +SKIP
... CreateIdentityArgs(
... display_name='prod-bq',
... config=BigQueryIdentityConfig(
... authentication=GoogleWorkloadIdentityAuthentication(
... service_account_email='kumo@data-project'
... '.iam.gserviceaccount.com'))))
>>> kumo.set_backend(kumo.LocalBackend( # doctest: +SKIP
... sources={kumo.DataSourceType.BIGQUERY: BigQuerySourceBackend(
... staging_uri='gs://bucket/kumo')}))
>>> import kumo.engine as kumo # doctest: +SKIP
>>> identity = kumo.BigQueryIdentity( # doctest: +SKIP
... workload_service_account_email='kumo@data-project'
... '.iam.gserviceaccount.com')
>>> events = kumo.BigQueryConnector( # doctest: +SKIP
... name='events', project_id='data-project',
... dataset_id='analytics', identity=identity)
... project_id='data-project', dataset_id='analytics',
... identity=identity, staging_uri='gs://bucket/kumo')
"""

source_type = DataSourceType.BIGQUERY
Expand Down
13 changes: 6 additions & 7 deletions packages/kumo-core/src/kumo_core/sources/principals.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,8 @@ def create(self, args: CreateIdentityArgs) -> Registration:
if name in self._by_name:
raise ValueError(
f"Identity {name!r} already exists in this process. "
f"Delete it with kumoai_studio.delete_identity, or rotate its "
f"credentials with kumoai_studio.rotate_identity_credentials.")
f"Use a different name, or delete the existing identity or "
f"rotate its credentials.")
registration = Registration(
principal=Principal(name=name, config=args.config,
credentials=args.credentials),
Expand Down Expand Up @@ -306,10 +306,9 @@ def resolve(
return None
wanted = ", ".join(repr(reference) for reference in named) or "no"
known = ", ".join(repr(name) for name in store.names()) or "none"
bind = f"; {connector_example}" if connector_example else ""
bind = f", as in {connector_example}" if connector_example else ""
raise ValueError(
f"Connector {getattr(connector, 'id', connector)!r} names {wanted} "
f"identity, and none exists in this process. Create one and name it "
f"on the connector: identity = kumoai_studio.create_identity("
f"CreateIdentityArgs(display_name='<name>', "
f"config={config_example})){bind}. {fallback} Existing: {known}.")
f"identity, and none exists in this process. Create the identity "
f"first (config={config_example}), or pass an Identity object on the "
f"connector{bind}. {fallback} Existing: {known}.")
34 changes: 10 additions & 24 deletions packages/kumo-databricks/src/kumo_databricks/backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,10 @@ class DatabricksSourceBackend(_DatabricksSessions):

Args:
identity: A fallback identity, used only for connectors that name none
themselves. Accepts an :class:`~kumoai_studio.identity.Identity`, an
themselves. Accepts an identity object, an
identity ID, or the name it was created under. Leaving it unset is the
common case: a bound
:class:`~kumoai_studio.connector.DatabricksConnector` records the
identity it was created with.
common case: a connector records the identity it was created
with.
mode: A :class:`~kumo_core.sources.types.ReadMode` to force, when a
run needs to know which path it took rather than only that the
table was read. ``None`` takes the cheapest the reader offers.
Expand All @@ -72,27 +71,14 @@ class DatabricksSourceBackend(_DatabricksSessions):
``SELECT`` the table already required.

Example:
>>> import kumoai_studio as kumo # doctest: +SKIP
>>> from kumoapi.data_source import ( # doctest: +SKIP
... CreateIdentityArgs, DatabricksIdentityConfig,
... DatabricksOAuthCredentials)
>>> from pydantic import SecretStr # doctest: +SKIP
>>> identity = kumo.create_identity( # doctest: +SKIP
... CreateIdentityArgs(
... display_name='prod-dbx',
... config=DatabricksIdentityConfig(
... host='https://workspace.cloud.databricks.com',
... client_id='service-principal-id'),
... credentials=DatabricksOAuthCredentials(
... client_secret=SecretStr('...'))))
>>> kumo.set_backend(kumo.LocalBackend( # doctest: +SKIP
... sources={kumo.DataSourceType.DATABRICKS:
... DatabricksSourceBackend(
... staging_volume_path='/Volumes/main/kumo/staging')}
... ))
>>> import kumo.engine as kumo # doctest: +SKIP
>>> identity = kumo.DatabricksIdentity( # doctest: +SKIP
... host='https://my-workspace.example.com',
... client_id='service-principal-id', client_secret='...')
>>> bricks = kumo.DatabricksConnector( # doctest: +SKIP
... name='analytics', catalog='main', schema_name='features',
... warehouse_id='warehouse-id', identity=identity)
... catalog='main', schema_name='features',
... warehouse_id='warehouse-id', identity=identity,
... staging_volume_path='/Volumes/main/kumo/staging')
"""

source_type = DataSourceType.DATABRICKS
Expand Down
37 changes: 13 additions & 24 deletions packages/kumo-engine/kumo/engine/_infer.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,15 @@
r"""Table schema inference: dtype/stype per column, from source metadata and,
where the dtype alone doesn't settle it, a sample of the actual data.

Ported from `kumoai/backend/local/table.py`'s `_LocalTableBackend.
infer_metadata`/`_stype_for_column`/`_sample`/`_statistics`/
`_column_statistics` in the cloud SDK. Sample decoding and statistics now
have one implementation in `kumo_core`; this adapter calls
Sample decoding and statistics have one implementation in `kumo_core`;
this adapter calls
`connector.source_backend` directly (no `Backend`/connector-id-lookup
indirection -- `Table.connector` already *is* the live connector) and threads a
Custom SQL table's query to the reader as `sql=` (the reader applies the
sample gating).

Two entry points, split the same way the cloud SDK splits `SourceTable.
column_dict` (source dtype + a pure dtype->stype rule)
from `Table.infer_metadata()` (samples the source to resolve whatever the
cheap pass left ambiguous):
Two entry points: a cheap pass (source dtype + a pure dtype->stype rule)
and a sampled pass that resolves whatever the cheap pass left ambiguous:

* `infer_source_columns` -- source-schema lookup. Filesystem CSV sources
read rows to infer dtypes; this is not necessarily a catalog-only call.
Expand All @@ -26,14 +22,12 @@
allows `stype=None`, exactly the "not yet resolved" state a cheap catalog
read can leave a numeric/string column in.
* `resolve_table_columns` -- takes that cheap mapping and finishes the job:
primary-key/time-column stype override, then a sampled read
(`kumoai/backend/local/table.py`'s `_sample`) for anything still
ambiguous. Backs `Table.from_source_table`, which is where this package's
primary-key/time-column stype override, then a sampled read for
anything still ambiguous. Backs `Table.from_source_table`, which is where this package's
cheap/expensive split necessarily ends -- `kumoapi.table.TableDefinition.
cols` is a `List[Column]`, and `kumoapi.table.Column` (unlike the cloud
SDK's own mutable wrapper) requires `stype`/`dtype` set, so a `Table`
cannot hold a partially-resolved schema the way a cloud-SDK `Table` can
between `from_source_table` and a later `infer_metadata()` call. All the
cols` is a `List[Column]`, and `kumoapi.table.Column` requires
`stype`/`dtype` set, so a `Table` cannot hold a partially-resolved
schema. All the
sampling this package ever needs, for a given `Table`, therefore has to
happen inside `from_source_table` itself, not deferred to a later no-arg
call.
Expand All @@ -50,16 +44,14 @@

from kumo.engine.connector.base import Connector

#: Rows sampled for stats-based stype inference, matching the cloud SDK's
#: `kumoai/backend/local/_common.py`'s `_SAMPLE_ROWS`.
#: Rows sampled for stats-based stype inference.
_SAMPLE_ROWS = 10_000


def _stype_from_dtype(dtype: Dtype) -> Optional[Stype]:
r"""Returns the stype a data type settles on its own, if any -- `None`
for the numeric and string families, which need column statistics to
separate (`_stype_from_stats`). Ported from
`kumoai/backend/local/table.py`'s `_stype_from_dtype`.
separate (`_stype_from_stats`).
"""
from kumo_core.stype_inference import infer_stype_from_dtype

Expand All @@ -69,8 +61,7 @@ def _stype_from_dtype(dtype: Dtype) -> Optional[Stype]:
def _stype_from_stats(dtype: Dtype, num_unique: int, unique_fraction: float,
num_unique_multi: Optional[int] = None,
multicategorical_proven: bool = False) -> Optional[Stype]:
r"""Returns the stype a column's statistics imply. Ported from
`kumoai/backend/local/table.py`'s `_stype_from_stats`.
r"""Returns the stype a column's statistics imply.
"""
from kumo_core.stype_inference import infer_stype_from_stats

Expand All @@ -90,7 +81,6 @@ def _stype_for_column(
sampled statistics. Identifier and numerical names use the shared rules;
explicit primary-key/time-column roles are applied in
`resolve_table_columns`.
Ported from `kumoai/backend/local/table.py`'s `_stype_for_column`.
"""
if dtype is None:
return None
Expand Down Expand Up @@ -127,8 +117,7 @@ def _sample(
statistics are an enhancement: a column whose stype follows from its
dtype must still resolve with no read at all, and only a column that
genuinely needed the data should fail (the caller's own error then names
it). Ported from `kumoai/backend/local/table.py`'s `_sample`/
`_statistics`.
it).
"""
try:
response = source_backend.get_sample(connector, table_name,
Expand Down
14 changes: 4 additions & 10 deletions packages/kumo-engine/kumo/engine/_portable.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,8 @@
r"""Portable graph conversion: `kumo.engine.graph.Graph` -> the dict `kumo_core`
training/prediction consume as `graph_config`.

Ported from `kumoai/backend/local/portable.py` in the cloud SDK, adapted to
this package's `Graph` shape: `Graph.edges` are already resolved
`kumoapi.graph.Edge` objects (built directly by the caller), not the
column-group form the cloud SDK's `GraphDefinition.col_groups` holds -- so
the `graph_api.resolve_foreign_keys` step `_to_portable_graph` does isn't
needed here. `kumoapi.graph.Edge`'s fields (`src_table`, `fkey`, `dst_table`)
`Graph.edges` are already resolved `kumoapi.graph.Edge` objects (built
directly by the caller), so no foreign-key resolution step is needed here. `kumoapi.graph.Edge`'s fields (`src_table`, `fkey`, `dst_table`)
already exactly match what `kumo_core.portable_graph._project_portable_graph`
expects an edge to look like, confirmed by reading both directly.

Expand Down Expand Up @@ -38,10 +34,8 @@
happens earlier and separately, via `SourceBackend.prepare_ingest(...)` ->
local Parquet -> `DiskgraphIngestSpec.source_paths` (`Graph.snapshot()`).
So `_source_table_name` below only needs to produce a well-formed, stable
identifier from fields the `Connector` already holds directly -- not to
replicate the cloud SDK's `_source_coordinate`/`_object_store_coordinate`/
`_custom_sql_coordinate` (`portable.py:107-258`), which exist there to drive
an actual `locate`-based connector lookup this package has no equivalent of.
identifier from fields the `Connector` already holds directly; nothing
looks a connector up from it.
"""

import warnings
Expand Down
4 changes: 2 additions & 2 deletions packages/kumo-engine/kumo/engine/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
r"""`configure(...)`: the one process-wide, opt-in setup call this package
has.

Everything else in `kumo.engine` is constructed and threaded through directly, no
`Backend`/`kumoai_studio.init(...)` needed. Scratch-directory placement is the
Everything else in `kumo.engine` is constructed and threaded through directly,
with no global init call. Scratch-directory placement is the
one exception: `Graph.snapshot()`'s DiskGraph ingest and `Trainer.fit`/`predict`
all read and write a shared local filesystem root
(`kumo.engine._scratch.get_scratch_dir()`, default `~/.cache/kumo-engine`)
Expand Down
30 changes: 11 additions & 19 deletions packages/kumo-engine/kumo/engine/connector/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,10 @@
directly from the existing first-party plugin packages (`kumo-snowflake`,
`kumo-s3`, `kumo-gcs`, `kumo-bigquery`, `kumo-databricks`).

Those mirror the *argument lists* of the equivalent cloud SDK classes
(`kumoai/connector/{snowflake,s3,gcs,bigquery,databricks}_connector.py`), but
drop everything that exists only to round-trip a server-side connector
registry: `name` (a project-scoped display name for `get_by_name` lookups),
`description`, `project_id`/`kumo_project_id`, and `_response`. In particular,
construction here never calls anything resembling the cloud SDK's
`IdentityBackedConnector._bind_connector` -> `get_backend().connector.
create_if_not_exist(...)` -- there is no `Backend`, so there is nothing to
round-trip through. Construction is pure Python: build the
`kumoapi.data_source.*ConnectorResourceConfig` and hold it.
Connectors carry no server-side registry state: no display name, no
description, no project. Construction never registers anything anywhere.
It is pure Python: build the `kumoapi.data_source.*ConnectorResourceConfig`
and hold it.
"""

from abc import ABC, abstractmethod
Expand Down Expand Up @@ -136,25 +130,23 @@ def __getitem__(self, table_name: str):

def sql(self, query: str):
r"""A `SQLSourceTable` whose rows come from the read-only `SELECT`
`query`, run in this connector's warehouse. Mirrors the cloud SDK's
`Connector.sql`. Only Snowflake, Databricks and BigQuery run SQL.
`query`, run in this connector's warehouse. Only Snowflake, Databricks
and BigQuery run SQL.
"""
from kumo.engine.connector.source_table import SQLSourceTable

return SQLSourceTable(query, self)

def table_names(self) -> List[str]:
r"""Source table names available through this connector. Mirrors
the cloud SDK's `Connector.table_names()` -- convenience over
`self.source_backend.list_tables(self).table_names`, which is what
this calls.
r"""Source table names available through this connector.
Convenience over `self.source_backend.list_tables(self).table_names`,
which is what this calls.
"""
return self.source_backend.list_tables(self).table_names

def has_table(self, table_name: str) -> bool:
r"""Whether `table_name` is visible through this connector. Mirrors
the cloud SDK's `Connector.has_table(name)` -- convenience over
`table_name in self.table_names()`.
r"""Whether `table_name` is visible through this connector.
Convenience over `table_name in self.table_names()`.
"""
return table_name in self.table_names()

Expand Down
4 changes: 1 addition & 3 deletions packages/kumo-engine/kumo/engine/connector/bigquery.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0
r"""`BigQueryConnector`, mirroring the argument list of the cloud SDK's
`kumoai.connector.bigquery_connector.BigQueryConnector`, minus everything
that exists only to round-trip a server-side connector registry. See
r"""`BigQueryConnector`: a connector with no server-side registry state. See
`kumo.engine.connector.base`.
"""

Expand Down
4 changes: 1 addition & 3 deletions packages/kumo-engine/kumo/engine/connector/databricks.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0
r"""`DatabricksConnector`, mirroring the argument list of the cloud SDK's
`kumoai.connector.databricks_connector.DatabricksConnector`, minus everything
that exists only to round-trip a server-side connector registry. See
r"""`DatabricksConnector`: a connector with no server-side registry state. See
`kumo.engine.connector.base`.
"""

Expand Down
10 changes: 4 additions & 6 deletions packages/kumo-engine/kumo/engine/connector/gcs.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0
r"""`GCSConnector`, mirroring the argument list of the cloud SDK's
`kumoai.connector.gcs_connector.GCSConnector`, minus everything that exists
only to round-trip a server-side connector registry. See
r"""`GCSConnector`: a connector with no server-side registry state. See
`kumo.engine.connector.base`.

TODO(local-sdk): same caveat as `kumo.engine.connector.s3` -- the cloud SDK's
`GCSURI`-based `root_dir` validation isn't ported yet.
TODO(local-sdk): same caveat as `kumo.engine.connector.s3` -- `root_dir`
does not yet reject ambiguous/percent-encoded path separators.
"""

from functools import cached_property
Expand Down Expand Up @@ -42,7 +40,7 @@ class GCSConnector(Connector):
Reads and writes through `kumo_gcs.GCSSourceBackend`/`GCSSinkBackend` --
install the `gcs` extra of this package to use it.

Two target modes are available, exactly as in the cloud SDK:
Two target modes are available:

* Set `root_dir` to constrain the connector to one bucket and optional
prefix. Table names are relative paths below the root.
Expand Down
Loading
Loading