diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index 76439bcfc4..046d2efa4a 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -1712,6 +1712,7 @@ def render( def render_seed(self) -> t.Iterator[QueryOrDF]: import numpy as np + import pandas as pd self._ensure_hydrated() @@ -1752,8 +1753,6 @@ def render_seed(self) -> t.Iterator[QueryOrDF]: # convert all date/time types to native pandas timestamp for column in [*date_columns, *datetime_columns]: - import pandas as pd - df[column] = pd.to_datetime(df[column], infer_datetime_format=True, errors="ignore") # type: ignore # extract datetime.date from pandas timestamp for DATE columns @@ -1769,7 +1768,7 @@ def render_seed(self) -> t.Iterator[QueryOrDF]: ) for column in bool_columns: - df[column] = df[column].apply(lambda i: str_to_bool(str(i))) + df[column] = df[column].apply(lambda i: None if pd.isna(i) else str_to_bool(str(i))) df.loc[:, string_columns] = df[string_columns].mask( cond=lambda x: x.notna(), # type: ignore diff --git a/tests/core/test_model.py b/tests/core/test_model.py index 6f286e309d..761f5703df 100644 --- a/tests/core/test_model.py +++ b/tests/core/test_model.py @@ -1541,6 +1541,44 @@ def test_seed_model_custom_types(tmp_path): assert df["empty_date"].iloc[0] is None +def test_seed_model_boolean_nulls_are_preserved(tmp_path): + model_csv_path = (tmp_path / "model.csv").absolute() + + with open(model_csv_path, "w", encoding="utf-8") as fd: + fd.write("id,test_ind\n") + fd.write("1,null\n") + fd.write("2,false\n") + fd.write("3,true\n") + fd.write("4,null\n") + + model = create_seed_model( + "test_db.test_model", + SeedKind(path=str(model_csv_path)), + columns={ + "id": "int", + "test_ind": "boolean", + }, + dialect="databricks", + ) + + df = next(model.render_seed()) + + assert df["test_ind"].to_list() == [None, False, True, None] + + context = Context( + config=Config( + default_connection=DuckDBConnectionConfig(), + model_defaults=ModelDefaultsConfig(dialect="databricks"), + ) + ) + context.upsert_model(model) + + rendered_sql = context.render(model).sql("databricks") + + assert "CAST(NULL AS BOOLEAN)" in rendered_sql + assert "(4, NULL)" in rendered_sql + + def test_seed_with_special_characters_in_column(tmp_path, assert_exp_eq): config = Config(model_defaults=ModelDefaultsConfig(dialect="duckdb")) context = Context(config=config)