Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions sqlmesh/core/model/definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -1712,6 +1712,7 @@ def render(

def render_seed(self) -> t.Iterator[QueryOrDF]:
import numpy as np
import pandas as pd

self._ensure_hydrated()

Expand Down Expand Up @@ -1752,8 +1753,6 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:

# convert all date/time types to native pandas timestamp
for column in [*date_columns, *datetime_columns]:
import pandas as pd

df[column] = pd.to_datetime(df[column], infer_datetime_format=True, errors="ignore") # type: ignore

# extract datetime.date from pandas timestamp for DATE columns
Expand All @@ -1769,7 +1768,7 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:
)

for column in bool_columns:
df[column] = df[column].apply(lambda i: str_to_bool(str(i)))
df[column] = df[column].apply(lambda i: None if pd.isna(i) else str_to_bool(str(i)))

df.loc[:, string_columns] = df[string_columns].mask(
cond=lambda x: x.notna(), # type: ignore
Expand Down
38 changes: 38 additions & 0 deletions tests/core/test_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -1541,6 +1541,44 @@ def test_seed_model_custom_types(tmp_path):
assert df["empty_date"].iloc[0] is None


def test_seed_model_boolean_nulls_are_preserved(tmp_path):
model_csv_path = (tmp_path / "model.csv").absolute()

with open(model_csv_path, "w", encoding="utf-8") as fd:
fd.write("id,test_ind\n")
fd.write("1,null\n")
fd.write("2,false\n")
fd.write("3,true\n")
fd.write("4,null\n")

model = create_seed_model(
"test_db.test_model",
SeedKind(path=str(model_csv_path)),
columns={
"id": "int",
"test_ind": "boolean",
},
dialect="databricks",
)

df = next(model.render_seed())

assert df["test_ind"].to_list() == [None, False, True, None]

context = Context(
config=Config(
default_connection=DuckDBConnectionConfig(),
model_defaults=ModelDefaultsConfig(dialect="databricks"),
)
)
context.upsert_model(model)

rendered_sql = context.render(model).sql("databricks")

assert "CAST(NULL AS BOOLEAN)" in rendered_sql
assert "(4, NULL)" in rendered_sql


def test_seed_with_special_characters_in_column(tmp_path, assert_exp_eq):
config = Config(model_defaults=ModelDefaultsConfig(dialect="duckdb"))
context = Context(config=config)
Expand Down