Faster load_svmfiles, empty files, tabs, nnz overflow guard (0.5.3)

setup.py

@@ -47,7 +47,7 @@
 
 setup(
     name = 'svmloader',
-    version = '0.5.2',
+    version = '0.5.3',
     description = DESCRIPTION,
     long_description = LONG_DESCRIPTION,
     long_description_content_type = 'text/markdown',

svmloader.pyx

@@ -91,7 +91,7 @@ cdef _load_svmfile(fp, dtype, ltype, bint zero_based, bint multilabels):
             last_idx = idx
 
             # ensure we have correct separator
-            while s[0] == ' ':
+            while s[0] == ' ' or s[0] == '\t':
                 s += 1
             if s[0] != ':':
                 raise ValueError('invalid separator')
@@ -102,10 +102,13 @@ cdef _load_svmfile(fp, dtype, ltype, bint zero_based, bint multilabels):
             if s == end:
                 raise ValueError('invalid value')
             s = end
-            while s[0] == ' ':
+            while s[0] == ' ' or s[0] == '\t':
                 s += 1
 
             sz += 1
+            if sz > INT_MAX:
+                # indptr holds C ints, which would silently wrap around
+                raise ValueError('too many non-zero values (more than %d)' % INT_MAX)
             array.resize_smart(indices, sz)
             array.resize_smart(data, sz)
             indices.data.as_ints[sz-1] = idx
@@ -141,6 +144,11 @@ def _openfile(filename):
     return fp
 
 
+def _nfeatures(indices):
+    """Number of columns implied by the indices: 0 for a file without values."""
+    return int(indices.max()) + 1 if len(indices) else 0
+
+
 def load_svmfile(filename, dtype='d', ltype='i', nfeatures=None, zero_based=True, multilabels=False):
     """\
     Load a sparse CSR matrix from filename at svmlib format.
@@ -165,14 +173,12 @@ def load_svmfile(filename, dtype='d', ltype='i', nfeatures=None, zero_based=True
     assert(dtype=='f' or dtype=='d'), 'dtype must be "d" or "f"'
     assert(ltype=='i' or ltype=='d'), 'ltype must be "i" or "d"'
 
-    fp = _openfile(filename)
-    data, indices, indptr, y = _load_svmfile(fp, dtype, ltype, zero_based, multilabels)
-    fp.close()
+    with _openfile(filename) as fp:
+        data, indices, indptr, y = _load_svmfile(fp, dtype, ltype, zero_based, multilabels)
 
     if nfeatures is None:
-        X = sp.csr_matrix((data, indices, indptr), copy=False)
-    else:
-        X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False)
+        nfeatures = _nfeatures(indices)
+    X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False)
 
     return X, y
 
@@ -212,7 +218,7 @@ def load_svmfiles(filenames, dtype='d', ltype='i', zero_based=True, multilabels=
             Xlst.append((data, indices, indptr))
             ylst.append(y)
     # get nfeatures as the maximum indice
-    nfeatures = max(max(indices) for _, indices, _ in Xlst) + 1
+    nfeatures = max([_nfeatures(indices) for _, indices, _ in Xlst], default=0)
     lst = []
     for (data, indices, indptr), y in zip(Xlst, ylst):
         X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False)    

tests/test_svmloader.py

@@ -181,3 +181,48 @@ def test_save_svmfile_writes_lf_line_endings(tmp_path):
     svmloader.save_svmfile(str(path), X, np.array([1, 0]))
 
     assert path.read_bytes() == b"1 2:1.5\n0 1:2\n"
+
+
+def test_load_empty_file(tmp_path):
+    path = tmp_path / "empty.svm"
+    path.write_bytes(b"")
+
+    X, y = svmloader.load_svmfile(str(path))
+    assert X.shape == (0, 0)
+    assert len(y) == 0
+
+    X, y = svmloader.load_svmfiles([str(path)])
+    assert X.shape == (0, 0)
+    assert len(y) == 0
+
+
+def test_load_rows_without_values(tmp_path):
+    path = tmp_path / "labels_only.svm"
+    path.write_bytes(b"1\n0\n")
+
+    X, y = svmloader.load_svmfile(str(path))
+
+    assert X.shape == (2, 0)
+    assert y.tolist() == [1, 0]
+
+
+def test_load_svmfiles_with_an_empty_file(tmp_path):
+    a = tmp_path / "a.svm"
+    a.write_bytes(b"1 1:2 4:1\n")
+    b = tmp_path / "b.svm"
+    b.write_bytes(b"")
+
+    X0, y0, X1, y1 = svmloader.load_svmfiles([str(a), str(b)], zero_based=False)
+
+    assert X0.shape == (1, 4)
+    assert X1.shape == (0, 4)
+
+
+def test_load_tabs_as_whitespace(tmp_path):
+    path = tmp_path / "tabs.svm"
+    path.write_bytes(b"1\t1:0.5\t3 :\t2\t\n0 2:1 \t\n")
+
+    X, y = svmloader.load_svmfile(str(path), zero_based=False)
+
+    assert y.tolist() == [1, 0]
+    np.testing.assert_allclose(X.toarray(), [[0.5, 0.0, 2.0], [0.0, 1.0, 0.0]])