diff --git a/setup.py b/setup.py index 4914350..1f2c7c4 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ setup( name = 'svmloader', - version = '0.5.2', + version = '0.5.3', description = DESCRIPTION, long_description = LONG_DESCRIPTION, long_description_content_type = 'text/markdown', diff --git a/svmloader.pyx b/svmloader.pyx index aff14fc..a6dc796 100644 --- a/svmloader.pyx +++ b/svmloader.pyx @@ -91,7 +91,7 @@ cdef _load_svmfile(fp, dtype, ltype, bint zero_based, bint multilabels): last_idx = idx # ensure we have correct separator - while s[0] == ' ': + while s[0] == ' ' or s[0] == '\t': s += 1 if s[0] != ':': raise ValueError('invalid separator') @@ -102,10 +102,13 @@ cdef _load_svmfile(fp, dtype, ltype, bint zero_based, bint multilabels): if s == end: raise ValueError('invalid value') s = end - while s[0] == ' ': + while s[0] == ' ' or s[0] == '\t': s += 1 sz += 1 + if sz > INT_MAX: + # indptr holds C ints, which would silently wrap around + raise ValueError('too many non-zero values (more than %d)' % INT_MAX) array.resize_smart(indices, sz) array.resize_smart(data, sz) indices.data.as_ints[sz-1] = idx @@ -141,6 +144,11 @@ def _openfile(filename): return fp +def _nfeatures(indices): + """Number of columns implied by the indices: 0 for a file without values.""" + return int(indices.max()) + 1 if len(indices) else 0 + + def load_svmfile(filename, dtype='d', ltype='i', nfeatures=None, zero_based=True, multilabels=False): """\ Load a sparse CSR matrix from filename at svmlib format. @@ -165,14 +173,12 @@ def load_svmfile(filename, dtype='d', ltype='i', nfeatures=None, zero_based=True assert(dtype=='f' or dtype=='d'), 'dtype must be "d" or "f"' assert(ltype=='i' or ltype=='d'), 'ltype must be "i" or "d"' - fp = _openfile(filename) - data, indices, indptr, y = _load_svmfile(fp, dtype, ltype, zero_based, multilabels) - fp.close() + with _openfile(filename) as fp: + data, indices, indptr, y = _load_svmfile(fp, dtype, ltype, zero_based, multilabels) if nfeatures is None: - X = sp.csr_matrix((data, indices, indptr), copy=False) - else: - X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False) + nfeatures = _nfeatures(indices) + X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False) return X, y @@ -212,7 +218,7 @@ def load_svmfiles(filenames, dtype='d', ltype='i', zero_based=True, multilabels= Xlst.append((data, indices, indptr)) ylst.append(y) # get nfeatures as the maximum indice - nfeatures = max(max(indices) for _, indices, _ in Xlst) + 1 + nfeatures = max([_nfeatures(indices) for _, indices, _ in Xlst], default=0) lst = [] for (data, indices, indptr), y in zip(Xlst, ylst): X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False) diff --git a/tests/test_svmloader.py b/tests/test_svmloader.py index e63a888..2f4e826 100644 --- a/tests/test_svmloader.py +++ b/tests/test_svmloader.py @@ -181,3 +181,48 @@ def test_save_svmfile_writes_lf_line_endings(tmp_path): svmloader.save_svmfile(str(path), X, np.array([1, 0])) assert path.read_bytes() == b"1 2:1.5\n0 1:2\n" + + +def test_load_empty_file(tmp_path): + path = tmp_path / "empty.svm" + path.write_bytes(b"") + + X, y = svmloader.load_svmfile(str(path)) + assert X.shape == (0, 0) + assert len(y) == 0 + + X, y = svmloader.load_svmfiles([str(path)]) + assert X.shape == (0, 0) + assert len(y) == 0 + + +def test_load_rows_without_values(tmp_path): + path = tmp_path / "labels_only.svm" + path.write_bytes(b"1\n0\n") + + X, y = svmloader.load_svmfile(str(path)) + + assert X.shape == (2, 0) + assert y.tolist() == [1, 0] + + +def test_load_svmfiles_with_an_empty_file(tmp_path): + a = tmp_path / "a.svm" + a.write_bytes(b"1 1:2 4:1\n") + b = tmp_path / "b.svm" + b.write_bytes(b"") + + X0, y0, X1, y1 = svmloader.load_svmfiles([str(a), str(b)], zero_based=False) + + assert X0.shape == (1, 4) + assert X1.shape == (0, 4) + + +def test_load_tabs_as_whitespace(tmp_path): + path = tmp_path / "tabs.svm" + path.write_bytes(b"1\t1:0.5\t3 :\t2\t\n0 2:1 \t\n") + + X, y = svmloader.load_svmfile(str(path), zero_based=False) + + assert y.tolist() == [1, 0] + np.testing.assert_allclose(X.toarray(), [[0.5, 0.0, 2.0], [0.0, 1.0, 0.0]])