diff --git a/packages/kumo-api/README.md b/packages/kumo-api/README.md index 75e257814..9f63361e1 100644 --- a/packages/kumo-api/README.md +++ b/packages/kumo-api/README.md @@ -5,8 +5,8 @@ for HTTP-compatible clients to interact with Kumo AI's cloud services. Resource schemas are defined as Pydantic dataclasses. While it is possible to interface with Kumo directly using these datamodels, -users are recommended to install the Python SDK (via `pip install kumoai-studio`) -for a smoother experience. +most users will want a higher-level Python library built on them, such as +`kumo-engine` (`pip install kumo-engine`). ## Development diff --git a/packages/kumo-bigquery/src/kumo_bigquery/backend.py b/packages/kumo-bigquery/src/kumo_bigquery/backend.py index cbc881a29..a2b79b682 100644 --- a/packages/kumo-bigquery/src/kumo_bigquery/backend.py +++ b/packages/kumo-bigquery/src/kumo_bigquery/backend.py @@ -58,11 +58,10 @@ class BigQuerySourceBackend(_BigQuerySessions): Args: identity: A fallback identity, used only for connectors that name none - themselves. Accepts an :class:`~kumoai_studio.identity.Identity`, an + themselves. Accepts an identity object, an identity ID, or the name it was created under. Leaving it unset is - the common case: a bound - :class:`~kumoai_studio.connector.BigQueryConnector` records the identity - it was created with. + the common case: a connector records the identity it was created + with. mode: The read mode every table is read under, or ``None`` to take whichever the reader offers first. staging_uri: A ``gs://`` prefix this run may export Parquet to. Without @@ -71,23 +70,13 @@ class BigQuerySourceBackend(_BigQuerySessions): an error: no bucket is a complete configuration. Example: - >>> import kumoai_studio as kumo # doctest: +SKIP - >>> from kumoapi.data_source import ( # doctest: +SKIP - ... BigQueryIdentityConfig, CreateIdentityArgs, - ... GoogleWorkloadIdentityAuthentication) - >>> identity = kumo.create_identity( # doctest: +SKIP - ... CreateIdentityArgs( - ... display_name='prod-bq', - ... config=BigQueryIdentityConfig( - ... authentication=GoogleWorkloadIdentityAuthentication( - ... service_account_email='kumo@data-project' - ... '.iam.gserviceaccount.com')))) - >>> kumo.set_backend(kumo.LocalBackend( # doctest: +SKIP - ... sources={kumo.DataSourceType.BIGQUERY: BigQuerySourceBackend( - ... staging_uri='gs://bucket/kumo')})) + >>> import kumo.engine as kumo # doctest: +SKIP + >>> identity = kumo.BigQueryIdentity( # doctest: +SKIP + ... workload_service_account_email='kumo@data-project' + ... '.iam.gserviceaccount.com') >>> events = kumo.BigQueryConnector( # doctest: +SKIP - ... name='events', project_id='data-project', - ... dataset_id='analytics', identity=identity) + ... project_id='data-project', dataset_id='analytics', + ... identity=identity, staging_uri='gs://bucket/kumo') """ source_type = DataSourceType.BIGQUERY diff --git a/packages/kumo-core/src/kumo_core/sources/principals.py b/packages/kumo-core/src/kumo_core/sources/principals.py index 63b4223db..e28a849af 100644 --- a/packages/kumo-core/src/kumo_core/sources/principals.py +++ b/packages/kumo-core/src/kumo_core/sources/principals.py @@ -158,8 +158,8 @@ def create(self, args: CreateIdentityArgs) -> Registration: if name in self._by_name: raise ValueError( f"Identity {name!r} already exists in this process. " - f"Delete it with kumoai_studio.delete_identity, or rotate its " - f"credentials with kumoai_studio.rotate_identity_credentials.") + f"Use a different name, or delete the existing identity or " + f"rotate its credentials.") registration = Registration( principal=Principal(name=name, config=args.config, credentials=args.credentials), @@ -306,10 +306,9 @@ def resolve( return None wanted = ", ".join(repr(reference) for reference in named) or "no" known = ", ".join(repr(name) for name in store.names()) or "none" - bind = f"; {connector_example}" if connector_example else "" + bind = f", as in {connector_example}" if connector_example else "" raise ValueError( f"Connector {getattr(connector, 'id', connector)!r} names {wanted} " - f"identity, and none exists in this process. Create one and name it " - f"on the connector: identity = kumoai_studio.create_identity(" - f"CreateIdentityArgs(display_name='', " - f"config={config_example})){bind}. {fallback} Existing: {known}.") + f"identity, and none exists in this process. Create the identity " + f"first (config={config_example}), or pass an Identity object on the " + f"connector{bind}. {fallback} Existing: {known}.") diff --git a/packages/kumo-databricks/src/kumo_databricks/backend.py b/packages/kumo-databricks/src/kumo_databricks/backend.py index 8f3b8e759..20bdd76f4 100644 --- a/packages/kumo-databricks/src/kumo_databricks/backend.py +++ b/packages/kumo-databricks/src/kumo_databricks/backend.py @@ -57,11 +57,10 @@ class DatabricksSourceBackend(_DatabricksSessions): Args: identity: A fallback identity, used only for connectors that name none - themselves. Accepts an :class:`~kumoai_studio.identity.Identity`, an + themselves. Accepts an identity object, an identity ID, or the name it was created under. Leaving it unset is the - common case: a bound - :class:`~kumoai_studio.connector.DatabricksConnector` records the - identity it was created with. + common case: a connector records the identity it was created + with. mode: A :class:`~kumo_core.sources.types.ReadMode` to force, when a run needs to know which path it took rather than only that the table was read. ``None`` takes the cheapest the reader offers. @@ -72,27 +71,14 @@ class DatabricksSourceBackend(_DatabricksSessions): ``SELECT`` the table already required. Example: - >>> import kumoai_studio as kumo # doctest: +SKIP - >>> from kumoapi.data_source import ( # doctest: +SKIP - ... CreateIdentityArgs, DatabricksIdentityConfig, - ... DatabricksOAuthCredentials) - >>> from pydantic import SecretStr # doctest: +SKIP - >>> identity = kumo.create_identity( # doctest: +SKIP - ... CreateIdentityArgs( - ... display_name='prod-dbx', - ... config=DatabricksIdentityConfig( - ... host='https://workspace.cloud.databricks.com', - ... client_id='service-principal-id'), - ... credentials=DatabricksOAuthCredentials( - ... client_secret=SecretStr('...')))) - >>> kumo.set_backend(kumo.LocalBackend( # doctest: +SKIP - ... sources={kumo.DataSourceType.DATABRICKS: - ... DatabricksSourceBackend( - ... staging_volume_path='/Volumes/main/kumo/staging')} - ... )) + >>> import kumo.engine as kumo # doctest: +SKIP + >>> identity = kumo.DatabricksIdentity( # doctest: +SKIP + ... host='https://my-workspace.example.com', + ... client_id='service-principal-id', client_secret='...') >>> bricks = kumo.DatabricksConnector( # doctest: +SKIP - ... name='analytics', catalog='main', schema_name='features', - ... warehouse_id='warehouse-id', identity=identity) + ... catalog='main', schema_name='features', + ... warehouse_id='warehouse-id', identity=identity, + ... staging_volume_path='/Volumes/main/kumo/staging') """ source_type = DataSourceType.DATABRICKS diff --git a/packages/kumo-engine/kumo/engine/_infer.py b/packages/kumo-engine/kumo/engine/_infer.py index 2882cc9cf..c65100830 100644 --- a/packages/kumo-engine/kumo/engine/_infer.py +++ b/packages/kumo-engine/kumo/engine/_infer.py @@ -4,19 +4,15 @@ r"""Table schema inference: dtype/stype per column, from source metadata and, where the dtype alone doesn't settle it, a sample of the actual data. -Ported from `kumoai/backend/local/table.py`'s `_LocalTableBackend. -infer_metadata`/`_stype_for_column`/`_sample`/`_statistics`/ -`_column_statistics` in the cloud SDK. Sample decoding and statistics now -have one implementation in `kumo_core`; this adapter calls +Sample decoding and statistics have one implementation in `kumo_core`; +this adapter calls `connector.source_backend` directly (no `Backend`/connector-id-lookup indirection -- `Table.connector` already *is* the live connector) and threads a Custom SQL table's query to the reader as `sql=` (the reader applies the sample gating). -Two entry points, split the same way the cloud SDK splits `SourceTable. -column_dict` (source dtype + a pure dtype->stype rule) -from `Table.infer_metadata()` (samples the source to resolve whatever the -cheap pass left ambiguous): +Two entry points: a cheap pass (source dtype + a pure dtype->stype rule) +and a sampled pass that resolves whatever the cheap pass left ambiguous: * `infer_source_columns` -- source-schema lookup. Filesystem CSV sources read rows to infer dtypes; this is not necessarily a catalog-only call. @@ -26,14 +22,12 @@ allows `stype=None`, exactly the "not yet resolved" state a cheap catalog read can leave a numeric/string column in. * `resolve_table_columns` -- takes that cheap mapping and finishes the job: - primary-key/time-column stype override, then a sampled read - (`kumoai/backend/local/table.py`'s `_sample`) for anything still - ambiguous. Backs `Table.from_source_table`, which is where this package's + primary-key/time-column stype override, then a sampled read for + anything still ambiguous. Backs `Table.from_source_table`, which is where this package's cheap/expensive split necessarily ends -- `kumoapi.table.TableDefinition. - cols` is a `List[Column]`, and `kumoapi.table.Column` (unlike the cloud - SDK's own mutable wrapper) requires `stype`/`dtype` set, so a `Table` - cannot hold a partially-resolved schema the way a cloud-SDK `Table` can - between `from_source_table` and a later `infer_metadata()` call. All the + cols` is a `List[Column]`, and `kumoapi.table.Column` requires + `stype`/`dtype` set, so a `Table` cannot hold a partially-resolved + schema. All the sampling this package ever needs, for a given `Table`, therefore has to happen inside `from_source_table` itself, not deferred to a later no-arg call. @@ -50,16 +44,14 @@ from kumo.engine.connector.base import Connector -#: Rows sampled for stats-based stype inference, matching the cloud SDK's -#: `kumoai/backend/local/_common.py`'s `_SAMPLE_ROWS`. +#: Rows sampled for stats-based stype inference. _SAMPLE_ROWS = 10_000 def _stype_from_dtype(dtype: Dtype) -> Optional[Stype]: r"""Returns the stype a data type settles on its own, if any -- `None` for the numeric and string families, which need column statistics to - separate (`_stype_from_stats`). Ported from - `kumoai/backend/local/table.py`'s `_stype_from_dtype`. + separate (`_stype_from_stats`). """ from kumo_core.stype_inference import infer_stype_from_dtype @@ -69,8 +61,7 @@ def _stype_from_dtype(dtype: Dtype) -> Optional[Stype]: def _stype_from_stats(dtype: Dtype, num_unique: int, unique_fraction: float, num_unique_multi: Optional[int] = None, multicategorical_proven: bool = False) -> Optional[Stype]: - r"""Returns the stype a column's statistics imply. Ported from - `kumoai/backend/local/table.py`'s `_stype_from_stats`. + r"""Returns the stype a column's statistics imply. """ from kumo_core.stype_inference import infer_stype_from_stats @@ -90,7 +81,6 @@ def _stype_for_column( sampled statistics. Identifier and numerical names use the shared rules; explicit primary-key/time-column roles are applied in `resolve_table_columns`. - Ported from `kumoai/backend/local/table.py`'s `_stype_for_column`. """ if dtype is None: return None @@ -127,8 +117,7 @@ def _sample( statistics are an enhancement: a column whose stype follows from its dtype must still resolve with no read at all, and only a column that genuinely needed the data should fail (the caller's own error then names - it). Ported from `kumoai/backend/local/table.py`'s `_sample`/ - `_statistics`. + it). """ try: response = source_backend.get_sample(connector, table_name, diff --git a/packages/kumo-engine/kumo/engine/_portable.py b/packages/kumo-engine/kumo/engine/_portable.py index a9c1f1238..3b4c50e66 100644 --- a/packages/kumo-engine/kumo/engine/_portable.py +++ b/packages/kumo-engine/kumo/engine/_portable.py @@ -4,12 +4,8 @@ r"""Portable graph conversion: `kumo.engine.graph.Graph` -> the dict `kumo_core` training/prediction consume as `graph_config`. -Ported from `kumoai/backend/local/portable.py` in the cloud SDK, adapted to -this package's `Graph` shape: `Graph.edges` are already resolved -`kumoapi.graph.Edge` objects (built directly by the caller), not the -column-group form the cloud SDK's `GraphDefinition.col_groups` holds -- so -the `graph_api.resolve_foreign_keys` step `_to_portable_graph` does isn't -needed here. `kumoapi.graph.Edge`'s fields (`src_table`, `fkey`, `dst_table`) +`Graph.edges` are already resolved `kumoapi.graph.Edge` objects (built +directly by the caller), so no foreign-key resolution step is needed here. `kumoapi.graph.Edge`'s fields (`src_table`, `fkey`, `dst_table`) already exactly match what `kumo_core.portable_graph._project_portable_graph` expects an edge to look like, confirmed by reading both directly. @@ -38,10 +34,8 @@ happens earlier and separately, via `SourceBackend.prepare_ingest(...)` -> local Parquet -> `DiskgraphIngestSpec.source_paths` (`Graph.snapshot()`). So `_source_table_name` below only needs to produce a well-formed, stable -identifier from fields the `Connector` already holds directly -- not to -replicate the cloud SDK's `_source_coordinate`/`_object_store_coordinate`/ -`_custom_sql_coordinate` (`portable.py:107-258`), which exist there to drive -an actual `locate`-based connector lookup this package has no equivalent of. +identifier from fields the `Connector` already holds directly; nothing +looks a connector up from it. """ import warnings diff --git a/packages/kumo-engine/kumo/engine/config.py b/packages/kumo-engine/kumo/engine/config.py index 67b302c50..34fc6c78a 100644 --- a/packages/kumo-engine/kumo/engine/config.py +++ b/packages/kumo-engine/kumo/engine/config.py @@ -4,8 +4,8 @@ r"""`configure(...)`: the one process-wide, opt-in setup call this package has. -Everything else in `kumo.engine` is constructed and threaded through directly, no -`Backend`/`kumoai_studio.init(...)` needed. Scratch-directory placement is the +Everything else in `kumo.engine` is constructed and threaded through directly, +with no global init call. Scratch-directory placement is the one exception: `Graph.snapshot()`'s DiskGraph ingest and `Trainer.fit`/`predict` all read and write a shared local filesystem root (`kumo.engine._scratch.get_scratch_dir()`, default `~/.cache/kumo-engine`) diff --git a/packages/kumo-engine/kumo/engine/connector/base.py b/packages/kumo-engine/kumo/engine/connector/base.py index 08bf43525..0c1799970 100644 --- a/packages/kumo-engine/kumo/engine/connector/base.py +++ b/packages/kumo-engine/kumo/engine/connector/base.py @@ -13,16 +13,10 @@ directly from the existing first-party plugin packages (`kumo-snowflake`, `kumo-s3`, `kumo-gcs`, `kumo-bigquery`, `kumo-databricks`). -Those mirror the *argument lists* of the equivalent cloud SDK classes -(`kumoai/connector/{snowflake,s3,gcs,bigquery,databricks}_connector.py`), but -drop everything that exists only to round-trip a server-side connector -registry: `name` (a project-scoped display name for `get_by_name` lookups), -`description`, `project_id`/`kumo_project_id`, and `_response`. In particular, -construction here never calls anything resembling the cloud SDK's -`IdentityBackedConnector._bind_connector` -> `get_backend().connector. -create_if_not_exist(...)` -- there is no `Backend`, so there is nothing to -round-trip through. Construction is pure Python: build the -`kumoapi.data_source.*ConnectorResourceConfig` and hold it. +Connectors carry no server-side registry state: no display name, no +description, no project. Construction never registers anything anywhere. +It is pure Python: build the `kumoapi.data_source.*ConnectorResourceConfig` +and hold it. """ from abc import ABC, abstractmethod @@ -136,25 +130,23 @@ def __getitem__(self, table_name: str): def sql(self, query: str): r"""A `SQLSourceTable` whose rows come from the read-only `SELECT` - `query`, run in this connector's warehouse. Mirrors the cloud SDK's - `Connector.sql`. Only Snowflake, Databricks and BigQuery run SQL. + `query`, run in this connector's warehouse. Only Snowflake, Databricks + and BigQuery run SQL. """ from kumo.engine.connector.source_table import SQLSourceTable return SQLSourceTable(query, self) def table_names(self) -> List[str]: - r"""Source table names available through this connector. Mirrors - the cloud SDK's `Connector.table_names()` -- convenience over - `self.source_backend.list_tables(self).table_names`, which is what - this calls. + r"""Source table names available through this connector. + Convenience over `self.source_backend.list_tables(self).table_names`, + which is what this calls. """ return self.source_backend.list_tables(self).table_names def has_table(self, table_name: str) -> bool: - r"""Whether `table_name` is visible through this connector. Mirrors - the cloud SDK's `Connector.has_table(name)` -- convenience over - `table_name in self.table_names()`. + r"""Whether `table_name` is visible through this connector. + Convenience over `table_name in self.table_names()`. """ return table_name in self.table_names() diff --git a/packages/kumo-engine/kumo/engine/connector/bigquery.py b/packages/kumo-engine/kumo/engine/connector/bigquery.py index 7f6181474..eecfe3ad7 100644 --- a/packages/kumo-engine/kumo/engine/connector/bigquery.py +++ b/packages/kumo-engine/kumo/engine/connector/bigquery.py @@ -1,9 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -r"""`BigQueryConnector`, mirroring the argument list of the cloud SDK's -`kumoai.connector.bigquery_connector.BigQueryConnector`, minus everything -that exists only to round-trip a server-side connector registry. See +r"""`BigQueryConnector`: a connector with no server-side registry state. See `kumo.engine.connector.base`. """ diff --git a/packages/kumo-engine/kumo/engine/connector/databricks.py b/packages/kumo-engine/kumo/engine/connector/databricks.py index b7f52f892..b2ccaa556 100644 --- a/packages/kumo-engine/kumo/engine/connector/databricks.py +++ b/packages/kumo-engine/kumo/engine/connector/databricks.py @@ -1,9 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -r"""`DatabricksConnector`, mirroring the argument list of the cloud SDK's -`kumoai.connector.databricks_connector.DatabricksConnector`, minus everything -that exists only to round-trip a server-side connector registry. See +r"""`DatabricksConnector`: a connector with no server-side registry state. See `kumo.engine.connector.base`. """ diff --git a/packages/kumo-engine/kumo/engine/connector/gcs.py b/packages/kumo-engine/kumo/engine/connector/gcs.py index 501fb9af8..a7749ac38 100644 --- a/packages/kumo-engine/kumo/engine/connector/gcs.py +++ b/packages/kumo-engine/kumo/engine/connector/gcs.py @@ -1,13 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -r"""`GCSConnector`, mirroring the argument list of the cloud SDK's -`kumoai.connector.gcs_connector.GCSConnector`, minus everything that exists -only to round-trip a server-side connector registry. See +r"""`GCSConnector`: a connector with no server-side registry state. See `kumo.engine.connector.base`. -TODO(local-sdk): same caveat as `kumo.engine.connector.s3` -- the cloud SDK's -`GCSURI`-based `root_dir` validation isn't ported yet. +TODO(local-sdk): same caveat as `kumo.engine.connector.s3` -- `root_dir` +does not yet reject ambiguous/percent-encoded path separators. """ from functools import cached_property @@ -42,7 +40,7 @@ class GCSConnector(Connector): Reads and writes through `kumo_gcs.GCSSourceBackend`/`GCSSinkBackend` -- install the `gcs` extra of this package to use it. - Two target modes are available, exactly as in the cloud SDK: + Two target modes are available: * Set `root_dir` to constrain the connector to one bucket and optional prefix. Table names are relative paths below the root. diff --git a/packages/kumo-engine/kumo/engine/connector/s3.py b/packages/kumo-engine/kumo/engine/connector/s3.py index 3788f3e77..6c15c6fe9 100644 --- a/packages/kumo-engine/kumo/engine/connector/s3.py +++ b/packages/kumo-engine/kumo/engine/connector/s3.py @@ -1,17 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -r"""`S3Connector`, mirroring the argument list of the cloud SDK's -`kumoai.connector.s3_connector.S3Connector`, minus everything that exists -only to round-trip a server-side connector registry. See +r"""`S3Connector`: a connector with no server-side registry state. See `kumo.engine.connector.base`. -TODO(local-sdk): the cloud SDK's `root_dir` parsing goes through `S3URI` -(`kumoai/connector/s3_connector.py`, `kumoai/connector/file_connector.py`), -which additionally rejects ambiguous/percent-encoded path separators. That -class lives in the cloud SDK package, which this package does not depend on; -port the stricter validation here (or factor it out into `kumoapi`) rather -than relying on the bare `urlsplit` below long-term. +TODO(local-sdk): `root_dir` is parsed with a bare `urlsplit` and does not yet +reject ambiguous/percent-encoded path separators. """ from functools import cached_property @@ -48,7 +42,7 @@ class S3Connector(Connector): Reads and writes through `kumo_s3.S3SourceBackend`/`S3SinkBackend` -- install the `s3` extra of this package to use it. - Two target modes are available, exactly as in the cloud SDK: + Two target modes are available: * Set `root_dir` to constrain the connector to one bucket and optional prefix. Table names are relative paths below the root. @@ -63,7 +57,7 @@ class S3Connector(Connector): `aws configure`, environment) -- there is no server-side workload identity in this package. region: Optional bucket region for rooted mode. Invalid in any-URI - mode, matching the cloud SDK's validation. + mode. """ def __init__( self, diff --git a/packages/kumo-engine/kumo/engine/connector/source_table.py b/packages/kumo-engine/kumo/engine/connector/source_table.py index afd5d3b6c..9fa8fa068 100644 --- a/packages/kumo-engine/kumo/engine/connector/source_table.py +++ b/packages/kumo-engine/kumo/engine/connector/source_table.py @@ -3,9 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 r"""`SourceTable`: a reference to a table living behind a `Connector`. -Mirrors the cloud SDK's `kumoai.connector.SourceTable` (`connector[name]` -> -`SourceTable`), minus what only exists to round-trip a server-side project: -construction here only stores the connector and table name. +`connector[name]` returns one. Construction only stores the connector and +table name. `SQLSourceTable` (`connector.sql(query)`) is the Custom SQL variant. `column_dict`/`columns` use the source-schema pass described in `kumo.engine._infer`. @@ -14,11 +13,10 @@ `stype=None`; resolving that via a sampled read is `Table. from_source_table`'s job, not this class's -- see `kumo.engine._infer`'s module docstring for why the split lands there rather than in a later no-arg -`infer_metadata()` call the way it does in the cloud SDK. +`infer_metadata()` call. Cached (`functools.cached_property`) per `SourceTable` instance, computed -once on first access -- unlike the cloud SDK, which round-trips to the -backend on every access. +once on first access. """ from functools import cached_property @@ -105,8 +103,7 @@ def checked_sql(connector: Connector, sql: str) -> CustomSQLSourceTable: class SQLSourceTable(SourceTable): r"""A source table whose rows come from a read-only `SELECT` run in the - connector's warehouse. Mirrors the cloud SDK's `kumoai_studio.connector. - SQLSourceTable`; constructed via `connector.sql(query)`. + connector's warehouse; constructed via `connector.sql(query)`. The query is checked up front with the same `kumo_core.custom_sql` gates hosted applies, and `table_name` is hosted's synthetic name for diff --git a/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb b/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb index 003b9428f..ff4f5f2d5 100644 --- a/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb +++ b/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb @@ -6,11 +6,10 @@ "source": [ "# Kumo GNN on the Netflix 5k slice \u2014 `kumo-engine`\n", "\n", - "This is `demo_netflix_gnn.ipynb` rewritten against **`kumo-engine`**, the new\n", - "local-only package (`packages/kumo-engine`) that replaces the cloud SDK's\n", - "`kumoai.LocalBackend`. Same dataset, same predictive query, same GNN pipeline --\n", - "but no `Backend`/`set_backend`, no `project_id`, and connectors/identities are\n", - "threaded through as plain Python objects instead of going through a registry.\n", + "This notebook runs a Kumo GNN end to end with **`kumo-engine`**\n", + "(`packages/kumo-engine`), in this process. There is no backend to set and no\n", + "`project_id`: connectors/identities are threaded through as plain Python\n", + "objects instead of going through a registry.\n", "\n", "See `docs/kumo-engine-sdk-split-design.md` for the design this notebook\n", "exercises end to end." @@ -60,8 +59,7 @@ " `[connections.]` entry in Snowflake's own `connections.toml`, the\n", " file most people already have working before they reach Kumo.\n", "\n", - "Both mirror the cloud SDK's own methods of the same names, minus the parts\n", - "that only exist to round-trip a server-side identity registry. A key pair\n", + "Neither registers anything server-side. A key pair\n", "is preferred over a password where a `connections.toml` entry has both." ] }, @@ -104,8 +102,8 @@ "source": [ "## 3. Read the catalog\n", "\n", - "`connector['table_name']` (`Connector.__getitem__`, same as the cloud SDK's\n", - "own `connector[table_name]`) returns a `SourceTable` -- a reference to that\n", + "`connector['table_name']` (`Connector.__getitem__`) returns a\n", + "`SourceTable` -- a reference to that\n", "table, with its schema resolved lazily (and cached) from the actual source\n", "data on first access via `.column_dict`/`.columns`." ] @@ -130,15 +128,14 @@ "source": [ "## 4. Define the tables\n", "\n", - "`Table.from_source_table(connector[table_name], ...)` resolves dtype/stype\n", - "the same way the cloud SDK's own `Table.from_source_table(source_table,\n", - "...)` does: cheap dtype-implied stypes first (`SourceTable.column_dict`),\n", + "`Table.from_source_table(connector[table_name], ...)` resolves dtype/stype:\n", + "cheap dtype-implied stypes first (`SourceTable.column_dict`),\n", "then a sampled scan for anything still ambiguous. `infer_metadata()` takes\n", "no arguments -- it re-resolves using whatever column selection/primary\n", "key/time columns the table already carries, so it chains after\n", "`from_source_table` (e.g. to pick up fresh source stats) and before\n", - "`with_column(...)`, the immutable equivalent of the cloud SDK's\n", - "`table[name].stype = ...` -- it returns a new `Table` rather than mutating." + "`with_column(...)`, which overrides a column's stype/dtype -- it returns a\n", + "new `Table` rather than mutating." ] }, { @@ -187,8 +184,8 @@ "## 5. Assemble the graph\n", "\n", "Edges are declared in the SDK's direction -- the table holding the foreign key\n", - "points at the table holding the primary key. `Graph.validate()` runs the same\n", - "structural check the cloud SDK's local backend does (portable-graph conversion\n", + "points at the table holding the primary key. `Graph.validate()` runs a\n", + "structural check (portable-graph conversion\n", "+ `kumo_core.pquery.normalize_graph`), needing only table *schemas* -- no\n", "materialized snapshot yet." ] @@ -219,8 +216,7 @@ "## 6. The predictive query\n", "\n", "Unchanged from the original demo. `PredictiveQuery(graph=graph, query=QUERY)`\n", - "binds the graph at construction -- matching the cloud SDK's own constructor\n", - "shape -- and parses lazily (cached on first access to `validate()`/\n", + "binds the graph at construction and parses lazily (cached on first access to `validate()`/\n", "`get_task_type()`, shared with `graph.native_graph`)." ] }, @@ -279,11 +275,9 @@ "source": [ "## 8. Train\n", "\n", - "`Trainer.fit(train_table)` runs the same three steps the cloud SDK's `fit`\n", - "does, in this process: materialize the training task against the snapshot,\n", - "run AutoML/GNN training, and (if `export_config` is given) publish the\n", - "trained model -- the local replacement for the cloud SDK's standalone\n", - "`export_model()`." + "`Trainer.fit(train_table)` runs three steps in this process: materialize\n", + "the training task against the snapshot, run AutoML/GNN training, and (if\n", + "`export_config` is given) publish the trained model." ] }, { @@ -323,9 +317,8 @@ "train_table_path`.\n", "\n", "`Trainer.predict` reuses the just-trained model (`Trainer` carries\n", - "`_training_result` after `fit`, the same way the cloud SDK's `Trainer` carries\n", - "`_training_job_id`) rather than naming a job id. `prediction_time` is the\n", - "local replacement for the cloud SDK's `Trainer.predict(prediction_time=...)`:\n", + "`_training_result` after `fit`) rather than naming a job id. Here\n", + "`prediction_time` is passed explicitly:\n", "there is no `connector.sql(...)` yet to compute `MAX(date)` directly, so a\n", "known in-range date stands in for \"the last rating in the data\"." ] @@ -378,9 +371,8 @@ "source": [ "## 10. What is still partial\n", "\n", - "* `PredictiveQuery.suggest_model_plan` isn't implemented -- matches the\n", - " reference cloud LocalBackend's own limitation (`kumo_core` can only\n", - " resolve a plan after ingest; pass model plan overrides to `Trainer`\n", + "* `PredictiveQuery.suggest_model_plan` isn't implemented (`kumo_core` can\n", + " only resolve a plan after ingest; pass model plan overrides to `Trainer`\n", " directly instead).\n", "* `Trainer.load` (reload a previously published model) isn't implemented yet.\n", "* `connector.sql(...)` (arbitrary read-only SQL) isn't ported -- used above\n", diff --git a/packages/kumo-gcs/src/kumo_gcs/backend.py b/packages/kumo-gcs/src/kumo_gcs/backend.py index c7d350004..0c0cb16d4 100644 --- a/packages/kumo-gcs/src/kumo_gcs/backend.py +++ b/packages/kumo-gcs/src/kumo_gcs/backend.py @@ -117,13 +117,13 @@ class GCSSourceBackend(GCSSessions, ObjectStoreSourceBackend): r"""Resolves source tables by reading Google Cloud Storage in this process. - Install with ``pip install 'kumoai-studio[gcs]'``. Without a named identity, + Install with ``pip install kumo-gcs``. Without a named identity, reads use Application Default Credentials. Arrow serves metadata and the shared downloader handles native and SDK transfers. Args: identity: A fallback identity, used only for connectors that name none - themselves. Accepts an :class:`~kumoai_studio.identity.Identity`, an + themselves. Accepts an identity object, an identity ID, or the name it was created under. storage_namespace: The storage system this process reaches, recorded in every graph coordinate a GCS table contributes. Defaults to @@ -132,12 +132,9 @@ class GCSSourceBackend(GCSSessions, ObjectStoreSourceBackend): store offers only ``streamed``, so this exists to assert that. Example: - >>> import kumoai_studio as kumo # doctest: +SKIP - >>> from kumo_gcs import GCSSourceBackend # doctest: +SKIP - >>> kumo.set_backend(kumo.LocalBackend( # doctest: +SKIP - ... sources={kumo.DataSourceType.GCS: GCSSourceBackend()})) + >>> import kumo.engine as kumo # doctest: +SKIP >>> events = kumo.GCSConnector( # doctest: +SKIP - ... name='events', root_dir='gs://company-data/events/') + ... root_dir='gs://company-data/events/') """ source_type = DataSourceType.GCS diff --git a/packages/kumo-s3/src/kumo_s3/backend.py b/packages/kumo-s3/src/kumo_s3/backend.py index dfd31df9d..11cd52fa3 100644 --- a/packages/kumo-s3/src/kumo_s3/backend.py +++ b/packages/kumo-s3/src/kumo_s3/backend.py @@ -152,14 +152,14 @@ def _filesystem(self, connector: Any, bucket: str, class S3SourceBackend(S3Sessions, ObjectStoreSourceBackend): r"""Resolves source tables by reading Amazon S3 in this process. - Installed as ``pip install 'kumoai-studio[s3]'``. With no identity named + Installed as ``pip install kumo-s3``. With no identity named the reader signs with the ambient AWS credential chain, which is what makes an instance profile or ``aws configure`` work with no further setup. Args: identity: A fallback identity, used only for connectors that name none - themselves. Accepts an :class:`~kumoai_studio.identity.Identity`, an + themselves. Accepts an identity object, an identity ID, or the name it was created under. storage_namespace: The storage system this process reaches, recorded in every graph coordinate an S3 table contributes. Defaults to @@ -169,12 +169,9 @@ class S3SourceBackend(S3Sessions, ObjectStoreSourceBackend): store offers only ``streamed``, so this exists to assert that. Example: - >>> import kumoai_studio as kumo # doctest: +SKIP - >>> from kumo_s3 import S3SourceBackend # doctest: +SKIP - >>> kumo.set_backend(kumo.LocalBackend( # doctest: +SKIP - ... sources={kumo.DataSourceType.S3: S3SourceBackend()})) + >>> import kumo.engine as kumo # doctest: +SKIP >>> events = kumo.S3Connector( # doctest: +SKIP - ... name='events', root_dir='s3://company-data/events/') + ... root_dir='s3://company-data/events/') """ source_type = DataSourceType.S3 diff --git a/packages/kumo-snowflake/src/kumo_snowflake/backend.py b/packages/kumo-snowflake/src/kumo_snowflake/backend.py index 8f6fee652..fdb52e1d5 100644 --- a/packages/kumo-snowflake/src/kumo_snowflake/backend.py +++ b/packages/kumo-snowflake/src/kumo_snowflake/backend.py @@ -57,30 +57,18 @@ class SnowflakeSourceBackend(_SnowflakeSessions): Args: identity: A fallback identity, used only for connectors that name none - themselves. Accepts an :class:`~kumoai_studio.identity.Identity`, an + themselves. Accepts an identity object, an identity ID, or the name it was created under. Leaving it unset is the - common case: a bound - :class:`~kumoai_studio.connector.SnowflakeConnector` records the identity - it was created with, which is what makes this reader stateless - enough for :class:`~kumoai_studio.backend.LocalBackend` to register by - default. + common case: a connector records the identity it was created + with, which is what makes this reader stateless enough to register + by default. Example: - >>> import kumoai_studio as kumo # doctest: +SKIP - >>> from kumoapi.data_source import ( # doctest: +SKIP - ... CreateIdentityArgs, SnowflakeAuthenticationKind, - ... SnowflakeIdentityConfig) - >>> identity = kumo.create_identity( # doctest: +SKIP - ... CreateIdentityArgs( - ... display_name='prod-sf', - ... config=SnowflakeIdentityConfig( - ... account='xy12345', - ... authentication=( - ... SnowflakeAuthenticationKind.OAUTH_TOKEN_FILE)))) - >>> kumo.set_backend(kumo.LocalBackend()) # doctest: +SKIP + >>> import kumo.engine as kumo # doctest: +SKIP + >>> identity = kumo.SnowflakeIdentity(account='xy12345') # doctest: +SKIP >>> snow = kumo.SnowflakeConnector( # doctest: +SKIP - ... name='analytics', warehouse='WH', database='ANALYTICS', - ... schema_name='PUBLIC', identity=identity) + ... warehouse='WH', database='ANALYTICS', schema_name='PUBLIC', + ... identity=identity) """ source_type = DataSourceType.SNOWFLAKE diff --git a/packages/kumo-snowflake/src/kumo_snowflake/spcs.py b/packages/kumo-snowflake/src/kumo_snowflake/spcs.py index 52d37a515..8a9051f1f 100644 --- a/packages/kumo-snowflake/src/kumo_snowflake/spcs.py +++ b/packages/kumo-snowflake/src/kumo_snowflake/spcs.py @@ -8,14 +8,14 @@ therefore stores no secret at all -- only where to read the token, which is why it pairs with no credentials. -The arguments are *built* here rather than the identity itself: constructing -one means calling ``kumoai_studio.create_identity``, and this package does not import -the SDK. The caller passes what comes back: +The arguments are *built* here rather than the identity itself, because this +package does not create identities. The caller passes what comes back to +whatever registers identities in its process. With kumo-engine, the connector +builds the identity from the same environment: - import kumoai_studio - from kumo_snowflake import spcs_identity_create_args + import kumo.engine as kumo - identity = kumoai_studio.create_identity(spcs_identity_create_args('prod-sf')) + connector = kumo.SnowflakeConnector.from_spcs_env() """ import os