From 74ce1543eb6d8544730478f4b104204ed5cde819 Mon Sep 17 00:00:00 2001 From: Yoch Melka Date: Tue, 20 Oct 2026 08:45:32 +0300 Subject: [PATCH 1/2] Faster load_svmfiles, empty files, tabs, nnz overflow guard - load_svmfiles computed the number of columns with Python's builtin max() over every index: 2.7 s on rcv1_test (33M values), now ndarray.max(). load_svmfiles goes from 9.7 s to 7.3 s on that file. - An empty file, or one with labels but no values, failed to load ('unable to infer matrix dimensions', 'max() iterable argument is empty'); it now gives a matrix with 0 columns. - Tabs count as whitespace around separators and at the end of lines; a trailing tab was read as an index. - More than INT_MAX values raises ValueError instead of silently wrapping the C int indptr. - load_svmfile closes the file when parsing fails. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XUT7Yj2eYmEyHV5fiicMsP --- svmloader.pyx | 24 +++++++++++++--------- tests/test_svmloader.py | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/svmloader.pyx b/svmloader.pyx index aff14fc..a6dc796 100644 --- a/svmloader.pyx +++ b/svmloader.pyx @@ -91,7 +91,7 @@ cdef _load_svmfile(fp, dtype, ltype, bint zero_based, bint multilabels): last_idx = idx # ensure we have correct separator - while s[0] == ' ': + while s[0] == ' ' or s[0] == '\t': s += 1 if s[0] != ':': raise ValueError('invalid separator') @@ -102,10 +102,13 @@ cdef _load_svmfile(fp, dtype, ltype, bint zero_based, bint multilabels): if s == end: raise ValueError('invalid value') s = end - while s[0] == ' ': + while s[0] == ' ' or s[0] == '\t': s += 1 sz += 1 + if sz > INT_MAX: + # indptr holds C ints, which would silently wrap around + raise ValueError('too many non-zero values (more than %d)' % INT_MAX) array.resize_smart(indices, sz) array.resize_smart(data, sz) indices.data.as_ints[sz-1] = idx @@ -141,6 +144,11 @@ def _openfile(filename): return fp +def _nfeatures(indices): + """Number of columns implied by the indices: 0 for a file without values.""" + return int(indices.max()) + 1 if len(indices) else 0 + + def load_svmfile(filename, dtype='d', ltype='i', nfeatures=None, zero_based=True, multilabels=False): """\ Load a sparse CSR matrix from filename at svmlib format. @@ -165,14 +173,12 @@ def load_svmfile(filename, dtype='d', ltype='i', nfeatures=None, zero_based=True assert(dtype=='f' or dtype=='d'), 'dtype must be "d" or "f"' assert(ltype=='i' or ltype=='d'), 'ltype must be "i" or "d"' - fp = _openfile(filename) - data, indices, indptr, y = _load_svmfile(fp, dtype, ltype, zero_based, multilabels) - fp.close() + with _openfile(filename) as fp: + data, indices, indptr, y = _load_svmfile(fp, dtype, ltype, zero_based, multilabels) if nfeatures is None: - X = sp.csr_matrix((data, indices, indptr), copy=False) - else: - X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False) + nfeatures = _nfeatures(indices) + X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False) return X, y @@ -212,7 +218,7 @@ def load_svmfiles(filenames, dtype='d', ltype='i', zero_based=True, multilabels= Xlst.append((data, indices, indptr)) ylst.append(y) # get nfeatures as the maximum indice - nfeatures = max(max(indices) for _, indices, _ in Xlst) + 1 + nfeatures = max([_nfeatures(indices) for _, indices, _ in Xlst], default=0) lst = [] for (data, indices, indptr), y in zip(Xlst, ylst): X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False) diff --git a/tests/test_svmloader.py b/tests/test_svmloader.py index e63a888..2f4e826 100644 --- a/tests/test_svmloader.py +++ b/tests/test_svmloader.py @@ -181,3 +181,48 @@ def test_save_svmfile_writes_lf_line_endings(tmp_path): svmloader.save_svmfile(str(path), X, np.array([1, 0])) assert path.read_bytes() == b"1 2:1.5\n0 1:2\n" + + +def test_load_empty_file(tmp_path): + path = tmp_path / "empty.svm" + path.write_bytes(b"") + + X, y = svmloader.load_svmfile(str(path)) + assert X.shape == (0, 0) + assert len(y) == 0 + + X, y = svmloader.load_svmfiles([str(path)]) + assert X.shape == (0, 0) + assert len(y) == 0 + + +def test_load_rows_without_values(tmp_path): + path = tmp_path / "labels_only.svm" + path.write_bytes(b"1\n0\n") + + X, y = svmloader.load_svmfile(str(path)) + + assert X.shape == (2, 0) + assert y.tolist() == [1, 0] + + +def test_load_svmfiles_with_an_empty_file(tmp_path): + a = tmp_path / "a.svm" + a.write_bytes(b"1 1:2 4:1\n") + b = tmp_path / "b.svm" + b.write_bytes(b"") + + X0, y0, X1, y1 = svmloader.load_svmfiles([str(a), str(b)], zero_based=False) + + assert X0.shape == (1, 4) + assert X1.shape == (0, 4) + + +def test_load_tabs_as_whitespace(tmp_path): + path = tmp_path / "tabs.svm" + path.write_bytes(b"1\t1:0.5\t3 :\t2\t\n0 2:1 \t\n") + + X, y = svmloader.load_svmfile(str(path), zero_based=False) + + assert y.tolist() == [1, 0] + np.testing.assert_allclose(X.toarray(), [[0.5, 0.0, 2.0], [0.0, 1.0, 0.0]]) From e337f5b9fad9d971a051c5bdf12a84010bebb52f Mon Sep 17 00:00:00 2001 From: Yoch Melka Date: Tue, 20 Oct 2026 08:45:32 +0300 Subject: [PATCH 2/2] Prepare the 0.5.3 release Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XUT7Yj2eYmEyHV5fiicMsP --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4914350..1f2c7c4 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ setup( name = 'svmloader', - version = '0.5.2', + version = '0.5.3', description = DESCRIPTION, long_description = LONG_DESCRIPTION, long_description_content_type = 'text/markdown',