From 8b4cd62d4e21363b84415f691e986a76a54cf5f9 Mon Sep 17 00:00:00 2001 From: "hotdata-automation[bot]" <267177015+hotdata-automation[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:49:27 +0000 Subject: [PATCH] chore(indexes): clarify vector index constraints and algorithms --- CHANGELOG.md | 1 + docs/CreateIndexRequest.md | 6 +++--- docs/IndexesApi.md | 3 ++- hotdata/api/indexes_api.py | 9 ++++++--- hotdata/models/create_index_request.py | 6 +++--- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d67e5f4..701b00f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- chore(indexes): clarify vector index constraints and algorithms - chore(indexes): clarify vector precision documentation - chore: clarify sorted table and index creation constraints - feat(indexes): add nlist field to index responses diff --git a/docs/CreateIndexRequest.md b/docs/CreateIndexRequest.md index 2619faa..e815f05 100644 --- a/docs/CreateIndexRequest.md +++ b/docs/CreateIndexRequest.md @@ -1,18 +1,18 @@ # CreateIndexRequest -Request body for POST .../indexes Two constraints span the whole table rather than this request alone. A table holds at most one vector index, whichever algorithm it uses. A second vector index is refused until the first is dropped. And a vector index that generates its own embeddings — that is, one created with `embedding_provider_id` — has to be the only index on its table. So a table that already carries any index (sorted, full-text, or vector) will not accept an embedding-backed vector index, and a table that already carries an embedding-backed vector index will not accept any further index of any type. To move between the two arrangements, drop what is there first. Plan for it when designing a table: combining full-text search with generated embeddings on one table is not possible, so use a separate table for the second index, or supply the embeddings yourself. A vector index over a column that already holds vectors — no `embedding_provider_id` — is not affected by that second rule and coexists with sorted and full-text indexes normally. Embedding generation also rewrites the table to add its generated column, and that rewrite cannot preserve a declared partition. An embedding-backed vector index is therefore refused on a partitioned table. No index of any type can be created on a table that declares a sort order (`sorted_by`). Such a request is refused with 400. +Request body for POST .../indexes Two constraints span the whole table rather than this request alone. A table holds at most one vector index, whichever algorithm it uses. A second vector index is refused until the first is dropped. And a vector index that generates its own embeddings — that is, one created with `embedding_provider_id` — has to be the only index on its table. So a table that already carries any index (sorted, full-text, or vector) will not accept an embedding-backed vector index, and a table that already carries an embedding-backed vector index will not accept any further index of any type. This applies to both vector algorithms. To move between the two arrangements, drop what is there first. Plan for it when designing a table: combining full-text search with generated embeddings on one table is not possible, so use a separate table for the second index, or supply the embeddings yourself. A vector index over a column that already holds vectors — no `embedding_provider_id` — is not affected by that second rule and coexists with sorted and full-text indexes normally. Embedding generation also rewrites the table to add its generated column, and that rewrite cannot preserve a declared partition. An embedding-backed vector index is therefore refused on a partitioned table. No index of any type can be created on a table that declares a sort order (`sorted_by`). Such a request is refused with 400. ## Properties Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**algorithm** | **str** | How a vector index organises the vectors it searches. Omit this field for `hnsw`, which is the default. `hnsw` — builds a graph of the vectors and keeps it in memory. Searches are very fast, and the memory a search needs grows with the whole table, so a large enough table cannot be served at all. `ivf` — groups the vectors into clusters and reads only the clusters nearest the search. Searches are considerably slower than `hnsw`, and the memory a search needs follows how much of the index it reads rather than the size of the table, so a table far too large for `hnsw` can still be searched. It keeps a copy of the table's rows beside the vectors so a search is answered without reading the table; that copy is extra storage, and how much depends on `vector_precision`, which decides how compactly the copied vectors are held. Available for columns that already hold vectors, with the `l2` and `cosine` metrics. | [optional] +**algorithm** | **str** | How a vector index organises the vectors it searches. Omit this field for `hnsw`, which is the default. `hnsw` — builds a graph of the vectors and keeps it in memory. Searches are very fast, and the memory a search needs grows with the whole table, so a large enough table cannot be served at all. `ivf` — groups the vectors into clusters and reads only the clusters nearest the search. Searches are considerably slower than `hnsw`, and the memory a search needs follows how much of the index it reads rather than the size of the table, so a table far too large for `hnsw` can still be searched. It keeps a copy of the table's rows beside the vectors so a search is answered without reading the table; that copy is extra storage, and how much depends on `vector_precision`, which decides how compactly the copied vectors are held. Available with the `l2` and `cosine` metrics, for columns that already hold vectors and for text columns with an embedding provider, which generates the vectors. Search an `ivf` index with `ORDER BY vector_distance(column, '<query>') LIMIT <k>` (or the matching distance function and a query vector), naming the columns to return rather than selecting `*`; `vector_search` works with `hnsw` indexes only. Generating embeddings for a very large table (tens of millions of rows) is not supported yet. | [optional] **var_async** | **bool** | When true, create the index as a background job and return a job ID for polling. | [optional] [default to False] **async_after_ms** | **int** | If set (requires `async` = true), wait up to this many milliseconds for the index build to finish: if it completes in time the index is returned (201), otherwise a 202 with a job ID to poll. Must be between 1000 and the server maximum; a value out of that range, or set without `async` = true, is rejected with 400. | [optional] **columns** | **List[str]** | Columns to index. Required for all index types. | **description** | **str** | User-facing description of the embedding (e.g., \"product descriptions\"). | [optional] **dimensions** | **int** | Output vector dimensions. Some models support multiple dimension sizes (e.g., OpenAI text-embedding-3-small supports 512 or 1536). If omitted, the model's default dimensions are used | [optional] -**embedding_provider_id** | **str** | Embedding provider ID. When set for a vector index, the source column is treated as text and embeddings are generated automatically. The vector index is then built on the generated embedding column (`{column}_embedding` by default). | [optional] +**embedding_provider_id** | **str** | Embedding provider ID. When set for a vector index, the source column is treated as text and embeddings are generated automatically. The vector index is then built on the generated embedding column (`{column}_embedding` by default), with either algorithm. For an `ivf` index the metric must be `l2` or `cosine`: a provider whose metric is `dot` is refused unless `metric` names one of those. Every later load and text search embeds with the same provider, and is refused if the provider's model or endpoint has changed since the index was created; changing its API key is fine. Dropping the index keeps the generated column as an ordinary column, which loads may leave out (its rows get no vector) and which a new embedding index can generate into again. | [optional] **index_name** | **str** | | **index_type** | **str** | Index type. `sorted` supports range queries, `bm25` full-text search, and `vector` similarity search. | [optional] [default to 'sorted'] **metric** | **str** | Distance metric for vector indexes: \"l2\", \"cosine\", or \"dot\". When omitted, defaults to \"l2\" for float array columns or the provider's preferred metric for text columns with auto-embedding. | [optional] diff --git a/docs/IndexesApi.md b/docs/IndexesApi.md index 42eab8c..2c560e1 100644 --- a/docs/IndexesApi.md +++ b/docs/IndexesApi.md @@ -111,7 +111,7 @@ Name | Type | Description | Notes Delete an index -Delete a specific index from a table. +Delete a specific index from a table. An index cannot be deleted while a load into its table is running; that request gets 409 and can be retried once the load finishes. Deleting a vector index that generated its own embeddings keeps the generated column as an ordinary column of the table. ### Example @@ -192,6 +192,7 @@ void (empty response body) |-------------|-------------|------------------| **204** | Index deleted | - | **404** | Index not found | - | +**409** | A load into the table is running; retry once it finishes | - | **500** | Internal server error | - | [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) diff --git a/hotdata/api/indexes_api.py b/hotdata/api/indexes_api.py index a5b6cf4..965c6fb 100644 --- a/hotdata/api/indexes_api.py +++ b/hotdata/api/indexes_api.py @@ -396,7 +396,7 @@ def delete_index( ) -> None: """Delete an index - Delete a specific index from a table. + Delete a specific index from a table. An index cannot be deleted while a load into its table is running; that request gets 409 and can be retried once the load finishes. Deleting a vector index that generated its own embeddings keeps the generated column as an ordinary column of the table. :param connection_id: The database's `default_connection_id`, as returned when the database is created or fetched (required) :type connection_id: str @@ -442,6 +442,7 @@ def delete_index( _response_types_map: Dict[str, Optional[str]] = { '204': None, '404': "ApiErrorResponse", + '409': "ApiErrorResponse", '500': "ApiErrorResponse", } response_data = self.api_client.call_api( @@ -477,7 +478,7 @@ def delete_index_with_http_info( ) -> ApiResponse[None]: """Delete an index - Delete a specific index from a table. + Delete a specific index from a table. An index cannot be deleted while a load into its table is running; that request gets 409 and can be retried once the load finishes. Deleting a vector index that generated its own embeddings keeps the generated column as an ordinary column of the table. :param connection_id: The database's `default_connection_id`, as returned when the database is created or fetched (required) :type connection_id: str @@ -523,6 +524,7 @@ def delete_index_with_http_info( _response_types_map: Dict[str, Optional[str]] = { '204': None, '404': "ApiErrorResponse", + '409': "ApiErrorResponse", '500': "ApiErrorResponse", } response_data = self.api_client.call_api( @@ -558,7 +560,7 @@ def delete_index_without_preload_content( ) -> RESTResponseType: """Delete an index - Delete a specific index from a table. + Delete a specific index from a table. An index cannot be deleted while a load into its table is running; that request gets 409 and can be retried once the load finishes. Deleting a vector index that generated its own embeddings keeps the generated column as an ordinary column of the table. :param connection_id: The database's `default_connection_id`, as returned when the database is created or fetched (required) :type connection_id: str @@ -604,6 +606,7 @@ def delete_index_without_preload_content( _response_types_map: Dict[str, Optional[str]] = { '204': None, '404': "ApiErrorResponse", + '409': "ApiErrorResponse", '500': "ApiErrorResponse", } response_data = self.api_client.call_api( diff --git a/hotdata/models/create_index_request.py b/hotdata/models/create_index_request.py index fbfcc8f..475e040 100644 --- a/hotdata/models/create_index_request.py +++ b/hotdata/models/create_index_request.py @@ -26,15 +26,15 @@ class CreateIndexRequest(BaseModel): """ - Request body for POST .../indexes Two constraints span the whole table rather than this request alone. A table holds at most one vector index, whichever algorithm it uses. A second vector index is refused until the first is dropped. And a vector index that generates its own embeddings — that is, one created with `embedding_provider_id` — has to be the only index on its table. So a table that already carries any index (sorted, full-text, or vector) will not accept an embedding-backed vector index, and a table that already carries an embedding-backed vector index will not accept any further index of any type. To move between the two arrangements, drop what is there first. Plan for it when designing a table: combining full-text search with generated embeddings on one table is not possible, so use a separate table for the second index, or supply the embeddings yourself. A vector index over a column that already holds vectors — no `embedding_provider_id` — is not affected by that second rule and coexists with sorted and full-text indexes normally. Embedding generation also rewrites the table to add its generated column, and that rewrite cannot preserve a declared partition. An embedding-backed vector index is therefore refused on a partitioned table. No index of any type can be created on a table that declares a sort order (`sorted_by`). Such a request is refused with 400. + Request body for POST .../indexes Two constraints span the whole table rather than this request alone. A table holds at most one vector index, whichever algorithm it uses. A second vector index is refused until the first is dropped. And a vector index that generates its own embeddings — that is, one created with `embedding_provider_id` — has to be the only index on its table. So a table that already carries any index (sorted, full-text, or vector) will not accept an embedding-backed vector index, and a table that already carries an embedding-backed vector index will not accept any further index of any type. This applies to both vector algorithms. To move between the two arrangements, drop what is there first. Plan for it when designing a table: combining full-text search with generated embeddings on one table is not possible, so use a separate table for the second index, or supply the embeddings yourself. A vector index over a column that already holds vectors — no `embedding_provider_id` — is not affected by that second rule and coexists with sorted and full-text indexes normally. Embedding generation also rewrites the table to add its generated column, and that rewrite cannot preserve a declared partition. An embedding-backed vector index is therefore refused on a partitioned table. No index of any type can be created on a table that declares a sort order (`sorted_by`). Such a request is refused with 400. """ # noqa: E501 - algorithm: Optional[StrictStr] = Field(default=None, description="How a vector index organises the vectors it searches. Omit this field for `hnsw`, which is the default. `hnsw` — builds a graph of the vectors and keeps it in memory. Searches are very fast, and the memory a search needs grows with the whole table, so a large enough table cannot be served at all. `ivf` — groups the vectors into clusters and reads only the clusters nearest the search. Searches are considerably slower than `hnsw`, and the memory a search needs follows how much of the index it reads rather than the size of the table, so a table far too large for `hnsw` can still be searched. It keeps a copy of the table's rows beside the vectors so a search is answered without reading the table; that copy is extra storage, and how much depends on `vector_precision`, which decides how compactly the copied vectors are held. Available for columns that already hold vectors, with the `l2` and `cosine` metrics.") + algorithm: Optional[StrictStr] = Field(default=None, description="How a vector index organises the vectors it searches. Omit this field for `hnsw`, which is the default. `hnsw` — builds a graph of the vectors and keeps it in memory. Searches are very fast, and the memory a search needs grows with the whole table, so a large enough table cannot be served at all. `ivf` — groups the vectors into clusters and reads only the clusters nearest the search. Searches are considerably slower than `hnsw`, and the memory a search needs follows how much of the index it reads rather than the size of the table, so a table far too large for `hnsw` can still be searched. It keeps a copy of the table's rows beside the vectors so a search is answered without reading the table; that copy is extra storage, and how much depends on `vector_precision`, which decides how compactly the copied vectors are held. Available with the `l2` and `cosine` metrics, for columns that already hold vectors and for text columns with an embedding provider, which generates the vectors. Search an `ivf` index with `ORDER BY vector_distance(column, '') LIMIT ` (or the matching distance function and a query vector), naming the columns to return rather than selecting `*`; `vector_search` works with `hnsw` indexes only. Generating embeddings for a very large table (tens of millions of rows) is not supported yet.") var_async: Optional[StrictBool] = Field(default=False, description="When true, create the index as a background job and return a job ID for polling.", alias="async") async_after_ms: Optional[Annotated[int, Field(strict=True, ge=1000)]] = Field(default=None, description="If set (requires `async` = true), wait up to this many milliseconds for the index build to finish: if it completes in time the index is returned (201), otherwise a 202 with a job ID to poll. Must be between 1000 and the server maximum; a value out of that range, or set without `async` = true, is rejected with 400.") columns: List[StrictStr] = Field(description="Columns to index. Required for all index types.") description: Optional[StrictStr] = Field(default=None, description="User-facing description of the embedding (e.g., \"product descriptions\").") dimensions: Optional[Annotated[int, Field(strict=True, ge=0)]] = Field(default=None, description="Output vector dimensions. Some models support multiple dimension sizes (e.g., OpenAI text-embedding-3-small supports 512 or 1536). If omitted, the model's default dimensions are used") - embedding_provider_id: Optional[StrictStr] = Field(default=None, description="Embedding provider ID. When set for a vector index, the source column is treated as text and embeddings are generated automatically. The vector index is then built on the generated embedding column (`{column}_embedding` by default).") + embedding_provider_id: Optional[StrictStr] = Field(default=None, description="Embedding provider ID. When set for a vector index, the source column is treated as text and embeddings are generated automatically. The vector index is then built on the generated embedding column (`{column}_embedding` by default), with either algorithm. For an `ivf` index the metric must be `l2` or `cosine`: a provider whose metric is `dot` is refused unless `metric` names one of those. Every later load and text search embeds with the same provider, and is refused if the provider's model or endpoint has changed since the index was created; changing its API key is fine. Dropping the index keeps the generated column as an ordinary column, which loads may leave out (its rows get no vector) and which a new embedding index can generate into again.") index_name: StrictStr index_type: Optional[StrictStr] = Field(default='sorted', description="Index type. `sorted` supports range queries, `bm25` full-text search, and `vector` similarity search.") metric: Optional[StrictStr] = Field(default=None, description="Distance metric for vector indexes: \"l2\", \"cosine\", or \"dot\". When omitted, defaults to \"l2\" for float array columns or the provider's preferred metric for text columns with auto-embedding.")