Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion haystack/components/converters/csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,10 @@ def _build_document_from_row(
"""
row_meta = dict(base_meta)

# Reserve the generated row number before merging the columns, so that a CSV column of the same name
# is kept under a "csv_" prefixed key by the collision handling below instead of being overwritten.
row_meta["row_number"] = row_index

# content (strict: content_column must exist; validated by caller)
content = self._safe_value(row.get(content_column))

Expand All @@ -235,5 +239,4 @@ def _build_document_from_row(
suffix += 1
row_meta[key_to_use] = self._safe_value(v)

row_meta["row_number"] = row_index
return Document(content=content, meta=row_meta)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
fixes:
- |
`CSVToDocument` in `row` conversion mode no longer drops a CSV column named `row_number`. The generated row
index is reserved before the columns are merged, so the original value is kept under `csv_row_number`, the
same collision handling the other columns already use.
41 changes: 41 additions & 0 deletions test/components/converters/test_csv_to_document.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,47 @@ def test_row_mode_meta_collision_prefixed(self, tmp_path):
assert d.meta["row_number"] == 0
assert d.content == "ok"

def test_row_mode_keeps_a_csv_column_named_row_number(self, tmp_path):
# The generated row number must not overwrite a CSV column of the same name.
csv_text = "text,row_number\r\nhello,record-42\r\n"
f = tmp_path / "rownum.csv"
f.write_text(csv_text, encoding="utf-8")

conv = CSVToDocument(conversion_mode="row")
d = conv.run(sources=[ByteStream.from_file_path(f)], content_column="text")["documents"][0]

assert d.content == "hello"
assert d.meta["row_number"] == 0
assert d.meta["csv_row_number"] == "record-42"

def test_row_mode_row_number_collision_uses_the_next_suffix(self, tmp_path):
csv_text = "text,row_number\r\nhello,record-42\r\n"
f = tmp_path / "rownum_multi.csv"
f.write_text(csv_text, encoding="utf-8")
extra_meta = {"csv_row_number": "existing0", "csv_row_number_1": "existing1"}

conv = CSVToDocument(conversion_mode="row")
d = conv.run(
sources=[ByteStream.from_file_path(f)], meta=[extra_meta], content_column="text"
)["documents"][0]

assert d.meta["row_number"] == 0
assert d.meta["csv_row_number"] == "existing0"
assert d.meta["csv_row_number_1"] == "existing1"
assert d.meta["csv_row_number_2"] == "record-42"

def test_row_mode_row_number_as_content_column(self, tmp_path):
# Control: selecting row_number as the content column still works.
csv_text = "row_number,comment\r\nrecord-42,ok\r\n"
f = tmp_path / "rownum_content.csv"
f.write_text(csv_text, encoding="utf-8")

conv = CSVToDocument(conversion_mode="row")
d = conv.run(sources=[ByteStream.from_file_path(f)], content_column="row_number")["documents"][0]

assert d.content == "record-42"
assert d.meta["row_number"] == 0

def test_row_mode_meta_collision_multiple_suffixes(self, tmp_path):
"""
If meta already has csv_file_path and csv_file_path_1, we should write the next as csv_file_path_2.
Expand Down