diff --git a/README.md b/README.md index 3e2f886..b9617b1 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,8 @@ reader, _ := archives.OpenBytes("pkg.tgz", data) Filename mappings are used first. When a name has no supported extension, `Open` and `OpenBytes` detect ZIP, TAR, gzip, bzip2, xz, and zstd from the content. -Compressed content is opened as TAR and returns a parser error when it does not -contain a TAR archive. `Open` reads at most 512 bytes before rejecting an +Compressed content is inspected for TAR headers; other payloads become a single +member named `content`. `Open` reads at most 512 bytes before rejecting an unsupported stream with no recognised extension. ### Streaming browsing @@ -180,6 +180,13 @@ buffers that input within `MaxInputBytes`; `OpenStreamBytes(name, data, options) reuses an existing byte slice without copying it. Keep that slice unchanged until `Close`. TAR and gem can consume a forward-only input directly. +When content detection finds gzip, bzip2, xz or zstd, the decompressed prefix +determines whether to read TAR entries or expose one member named `content`. +The single member has `Size: -1` in streams and browser listings because its size +is unknown before reading. Entry and expansion limits apply while reading or +skipping it. Buffered readers report its actual size. Explicit TAR filenames +such as `.tar.gz` still require TAR content. + Hex packages use `.tar` filenames. Buffered readers, streams, and browsers detect the four regular outer files `VERSION`, `CHECKSUM`, `metadata.config`, and `contents.tar.gz` in any order, with `VERSION` equal to `3` and a 64-character @@ -289,6 +296,7 @@ for _, f := range result.Files { - `.zip`, `.jar`, `.whl`, `.nupkg`, `.egg`, `.vsix` (ZIP-based) - `.tar`, `.tar.gz`, `.tgz`, `.crate`, `.tar.bz2`, `.tar.xz`, `.tar.zst` +- gzip, bzip2, xz and zstd compressed single files, detected by content - `.gem` (Ruby gems with nested data.tar.gz) - `.tar` Hex packages (detected by outer markers, with nested contents.tar.gz) - `.conda` (v2 conda packages: zip of `pkg-*.tar.zst` and `info-*.tar.zst`, presented as one merged tar) diff --git a/archives.go b/archives.go index cf1feaf..3b94d14 100644 --- a/archives.go +++ b/archives.go @@ -48,7 +48,7 @@ const ( type FileInfo struct { Path string // Full path within archive Name string // Base name - Size int64 // Uncompressed size in bytes + Size int64 // Uncompressed bytes; -1 for an unknown streaming size ModTime time.Time // Modification time IsDir bool // Whether this is a directory Mode uint32 // fs.FileMode value: permission bits plus fs.ModeType bits @@ -152,6 +152,8 @@ func openRaw(format string, raw []byte) (Reader, error) { return openGem(raw) case formatConda: return openConda(raw) + case compressionGzip, compressionBzip2, compressionXZ, compressionZstd: + return openCompressed(raw) default: return nil, fmt.Errorf("unsupported format: %s", format) } @@ -171,14 +173,8 @@ func archiveFormat(detected string) string { return formatZIP case "tar": return formatTAR - case compressionGzip: - return formatTarGzip - case compressionBzip2: - return formatTarBzip2 - case compressionXZ: - return formatTarXZ - case compressionZstd: - return formatTarZstd + case compressionGzip, compressionBzip2, compressionXZ, compressionZstd: + return detected default: return "" } diff --git a/archives_test.go b/archives_test.go index 4a7ef71..0da7470 100644 --- a/archives_test.go +++ b/archives_test.go @@ -89,8 +89,8 @@ func TestIsInDir(t *testing.T) { {"dir/subdir/file.txt", "dir/", false}, {"dir/subdir/file.txt", "dir/subdir/", true}, {"other/file.txt", "dir/", false}, - {"dir/", "", true}, // dir entry is in root - {"dir/", "dir/", true}, // explicit entry for the directory itself + {"dir/", "", true}, // dir entry is in root + {"dir/", "dir/", true}, // explicit entry for the directory itself {"dirX/file", "dir/", false}, } @@ -460,13 +460,13 @@ func TestOpenDoesNotInferGem(t *testing.T) { } } -func TestOpenCompressedNonTar(t *testing.T) { +func TestExplicitCompressedTarRejectsNonTar(t *testing.T) { buf := new(bytes.Buffer) w := gzip.NewWriter(buf) _, _ = w.Write([]byte("not a tar archive")) _ = w.Close() - _, err := OpenBytes("artifact", buf.Bytes()) + _, err := OpenBytes("artifact.tar.gz", buf.Bytes()) if err == nil { t.Fatal("compressed non-TAR content was accepted") } diff --git a/compressed.go b/compressed.go new file mode 100644 index 0000000..6f89bc6 --- /dev/null +++ b/compressed.go @@ -0,0 +1,93 @@ +package archives + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "io" + + "github.com/git-pkgs/magic" +) + +const compressedMemberName = "content" + +type compressedStream struct { + reader io.Reader + closer io.Closer + started bool +} + +//nolint:ireturn // decompressed content may contain TAR entries or one file +func newCompressedStream(ctx context.Context, content io.Reader, compression string) (streamSource, error) { + reader, closer, err := tarContentReader(contextReader{ctx: ctx, reader: content}, compression) + if err != nil { + return nil, err + } + buffered := bufio.NewReaderSize(contextReader{ctx: ctx, reader: reader}, contentSniffSize) + prefix, err := buffered.Peek(contentSniffSize) + if err != nil && err != io.EOF { + if closer != nil { + _ = closer.Close() + } + return nil, err + } + // The TAR parser also accepts v7 headers without a ustar signature. + _, headerErr := tar.NewReader(bytes.NewReader(prefix)).Next() + if magic.DetectPrefix(prefix).Format == magic.FormatTAR || headerErr == nil { + return &tarStream{reader: tar.NewReader(buffered), closer: closer}, nil + } + return &compressedStream{reader: buffered, closer: closer}, nil +} + +func (s *compressedStream) Next() (*StreamEntry, error) { + if s.started { + return nil, io.EOF + } + s.started = true + return &StreamEntry{FileInfo: FileInfo{Path: compressedMemberName, Name: compressedMemberName, Size: -1}}, nil +} + +func (s *compressedStream) Read(p []byte) (int, error) { + if !s.started { + return 0, io.EOF + } + return s.reader.Read(p) +} + +func (s *compressedStream) Close() error { + if s.closer != nil { + return s.closer.Close() + } + return nil +} + +func openCompressed(raw []byte) (*tarReader, error) { + stream, err := OpenStreamBytes("", raw, StreamOptions{}) + if err != nil { + return nil, err + } + defer func() { _ = stream.Close() }() + result := &tarReader{raw: raw, index: make(map[string]int)} + for { + entry, err := stream.Next() + if err == io.EOF { + return result, nil + } + if err != nil { + return nil, err + } + data, err := io.ReadAll(stream) + if err != nil { + return nil, err + } + info := entry.FileInfo + if info.Size < 0 { + info.Size = int64(len(data)) + } + if _, exists := result.index[info.Path]; !exists { + result.index[info.Path] = len(result.files) + } + result.files = append(result.files, tarFileEntry{info: info, data: data}) + } +} diff --git a/compressed_test.go b/compressed_test.go new file mode 100644 index 0000000..86a9ef1 --- /dev/null +++ b/compressed_test.go @@ -0,0 +1,249 @@ +package archives + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + "io" + "math" + "strings" + "testing" +) + +func TestCompressedV7TAR(t *testing.T) { + raw := tarWithPayloads(t, []byte("hello")) + clear(raw[257:512]) + copy(raw[148:156], " ") + var checksum int + for _, value := range raw[:512] { + checksum += int(value) + } + copy(raw[148:156], fmt.Sprintf("%06o\x00 ", checksum)) + data := gzipTar(t, raw) + r, err := OpenBytes("download", data) + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + files, err := r.List() + if err != nil || len(files) != 1 || files[0].Path != "package/lib/file0" || files[0].Size != 5 { + t.Fatalf("v7 listing: %+v %v", files, err) + } + assertTarPayload(t, r, "package/lib/file0", []byte("hello")) + s, err := OpenStreamBytes("download", data, StreamOptions{}) + if err != nil { + t.Fatal(err) + } + assertStreamMatches(t, s, r, files) +} + +func TestStreamRejectsNegativeZIP64Size(t *testing.T) { + for _, size := range []uint64{math.MaxUint64, math.MaxUint64 - 1, math.MaxInt64 + 1} { + var buf bytes.Buffer + w := zip.NewWriter(&buf) + member, err := w.CreateRaw(&zip.FileHeader{Name: "oversized", Method: zip.Store, UncompressedSize64: size, CompressedSize64: 1}) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(member, "x"); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + s, err := OpenStreamBytes("oversized.zip", buf.Bytes(), StreamOptions{}) + if err != nil { + t.Fatal(err) + } + if _, err := s.Next(); !errors.Is(err, ErrEntrySizeLimit) { + t.Fatalf("size %d: %v", size, err) + } + if err := s.Recover(); !errors.Is(err, ErrEntrySizeLimit) { + t.Fatalf("size error must remain terminal: %v", err) + } + _ = s.Close() + } +} + +func TestCompressedSingleFileListingLimits(t *testing.T) { + for _, test := range []struct { + limits StreamOptions + want error + }{ + {StreamOptions{MaxEntryBytes: 5}, ErrEntrySizeLimit}, + {StreamOptions{MaxExpandedBytes: 5}, ErrDecompressLimit}, + } { + browser, err := OpenBrowser("bundle.gz", bytes.NewReader(gzipTar(t, []byte(singleFilePayload))), nil, BrowseOptions{Limits: test.limits}) + if err != nil { + t.Fatal(err) + } + _, err = browser.List() + if !errors.Is(err, test.want) || !strings.Contains(err.Error(), "content exceeds 5") { + t.Fatalf("listing limit must include member path: %v", err) + } + } +} + +const singleFilePayload = "certificate content\n" + +func singleFileFormats(t *testing.T) map[string][]byte { + t.Helper() + bzip, err := base64.StdEncoding.DecodeString("QlpoOTFBWSZTWSxCGs8AAAJRgAAQQAArIZQAIAAxANNNBCNtUzSaFoISuvM63zfi7kinChIFiENZ4A==") + if err != nil { + t.Fatal(err) + } + return map[string][]byte{ + "gz": gzipTar(t, []byte(singleFilePayload)), "bz2": bzip, + "xz": compressTar(t, "test.tar.xz", []byte(singleFilePayload)), + "zst": compressTar(t, "test.tar.zst", []byte(singleFilePayload)), + } +} + +func TestCompressedSingleFileReaders(t *testing.T) { + for format, data := range singleFileFormats(t) { + for _, filename := range []string{"bundle.pem." + format, "renamed"} { + for _, buffered := range []bool{false, true} { + var r Reader + var err error + if buffered { + r, err = OpenBytes(filename, data) + } else { + r, err = Open(filename, bytes.NewReader(data)) + } + if err != nil { + t.Fatal(err) + } + files, err := r.List() + if err != nil || len(files) != 1 || files[0].Path != "content" || files[0].Size != int64(len(singleFilePayload)) { + t.Fatalf("%s: %+v %v", filename, files, err) + } + assertTarPayload(t, r, "content", []byte(singleFilePayload)) + hash, err := r.Hash("SHA256") + if err != nil || hash != fmt.Sprintf("%x", sha256.Sum256(data)) { + t.Fatalf("compressed hash: %s %v", hash, err) + } + _ = r.Close() + } + } + } +} + +func TestCompressedSingleFileStreams(t *testing.T) { + for format, data := range singleFileFormats(t) { + for _, read := range []bool{false, true} { + s, err := OpenStream("bundle."+format, bytes.NewReader(data), StreamOptions{}) + if err != nil { + t.Fatal(err) + } + entry, err := s.Next() + if err != nil || entry.Path != "content" || entry.Size != -1 { + t.Fatalf("%s: %+v %v", format, entry, err) + } + if read { + body, err := io.ReadAll(s) + if err != nil || string(body) != singleFilePayload { + t.Fatalf("%s: %q %v", format, body, err) + } + } + if _, err := s.Next(); err != io.EOF { + t.Fatal(err) + } + _ = s.Close() + } + } +} + +func TestCompressedSingleFileZeroPrefixAndEmptyContent(t *testing.T) { + for _, payload := range [][]byte{nil, append(make([]byte, 1024), []byte(singleFilePayload)...)} { + s, err := OpenStreamBytes("renamed", gzipTar(t, payload), StreamOptions{}) + if err != nil { + t.Fatal(err) + } + if entry, err := s.Next(); err != nil || entry.Path != "content" { + t.Fatalf("%+v %v", entry, err) + } + body, err := io.ReadAll(s) + if err != nil || !bytes.Equal(body, payload) { + t.Fatalf("payload mismatch: %v", err) + } + _ = s.Close() + } +} + +func TestCompressedSingleFileLimits(t *testing.T) { + for format, data := range singleFileFormats(t) { + for _, test := range []struct { + options StreamOptions + want error + }{ + {StreamOptions{MaxEntryBytes: 5}, ErrEntrySizeLimit}, + {StreamOptions{MaxExpandedBytes: 5}, ErrDecompressLimit}, + {StreamOptions{MaxInputBytes: int64(len(data) - 1)}, ErrInputLimit}, + {StreamOptions{MaxEntryBytes: int64(len(singleFilePayload)), MaxExpandedBytes: int64(len(singleFilePayload))}, nil}, + } { + for _, read := range []bool{false, true} { + s, err := OpenStreamBytes("renamed", data, test.options) + if err == nil { + err = consumeStream(s, read) + _ = s.Close() + } + if !errors.Is(err, test.want) { + t.Fatalf("%s read=%v: %v, want %v", format, read, err, test.want) + } + } + } + } +} + +func TestCompressedSingleFileCancellationAndTrailer(t *testing.T) { + data := gzipTar(t, bytes.Repeat([]byte(singleFilePayload), 1000)) + ctx, cancel := context.WithCancel(context.Background()) + s, err := OpenStreamBytesContext(ctx, "data.gz", data, StreamOptions{}) + if err != nil { + t.Fatal(err) + } + if _, err := s.Next(); err != nil { + t.Fatal(err) + } + cancel() + if _, err := io.ReadAll(s); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + _ = s.Close() + data[len(data)-8] ^= 1 + s, err = OpenStreamBytes("bad.gz", data, StreamOptions{}) + if err != nil { + t.Fatal(err) + } + if err := consumeStream(s, false); err == nil { + t.Fatal("corrupt gzip trailer accepted") + } + _ = s.Close() +} + +func TestCompressedSingleFileBrowser(t *testing.T) { + data := gzipTar(t, []byte(singleFilePayload)) + browser, err := OpenBrowser("bundle.gz", bytes.NewReader(data), func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil + }, BrowseOptions{}) + if err != nil { + t.Fatal(err) + } + files, err := browser.List() + if err != nil || len(files) != 1 || files[0].Path != "content" || files[0].Size != -1 { + t.Fatalf("%+v %v", files, err) + } + reader, err := browser.Extract("content") + if err != nil { + t.Fatal(err) + } + defer func() { _ = reader.Close() }() + body, err := io.ReadAll(reader) + if err != nil || string(body) != singleFilePayload { + t.Fatalf("%q %v", body, err) + } +} diff --git a/stream.go b/stream.go index 8373fc5..30892d3 100644 --- a/stream.go +++ b/stream.go @@ -35,13 +35,16 @@ type StreamEntry struct { // Paths and duplicate entries are preserved. A Stream is not safe for // concurrent use. It does not implement the random-access Reader interface. type Stream struct { - ctx context.Context - source streamSource - options StreamOptions - expanded int64 - entries int - err error - closed bool + ctx context.Context + source streamSource + options StreamOptions + expanded int64 + entries int + err error + closed bool + unknownSize bool + entryBytes int64 + entryPath string } type streamSource interface { @@ -51,7 +54,8 @@ type streamSource interface { // OpenStream reads supported archives sequentially. ZIP and conda input is // buffered up to MaxInputBytes for random access. Hex detection buffers an outer -// TAR prefix; TAR, gem, and Hex entry bodies are streamed. +// TAR prefix; TAR, gem, and Hex entry bodies are streamed. Compression detected +// by content exposes a non-TAR payload as one member named "content", with size -1. // The caller owns content and must close it when needed. func OpenStream(filename string, content io.Reader, options StreamOptions) (*Stream, error) { return OpenStreamContext(context.Background(), filename, content, options) @@ -109,6 +113,8 @@ func openStream(ctx context.Context, filename string, content io.Reader, raw []b source = &nestedTarStream{outer: newContainerTar(ctx, content, options), member: "data.tar.gz", kind: "gem"} case formatTAR: source = &plainTarStream{ctx: ctx, content: content, options: options} + case compressionGzip, compressionBzip2, compressionXZ, compressionZstd: + source, err = newCompressedStream(ctx, content, format) default: compression, ok := streamCompression(format) if !ok { @@ -180,7 +186,8 @@ func streamCompression(format string) (string, bool) { // Next discards the unread body and returns the next entry, or io.EOF. // Header sizes, including sparse logical sizes, count against limits before -// the body is exposed. MaxEntries counts visible entries; nested containers +// the body is exposed. Unknown sizes count as bytes are read or skipped. +// MaxEntries counts visible entries; nested containers // have a separate MaxEntries budget. Hidden TAR extension headers are excluded. // Any iteration error is terminal. func (s *Stream) Next() (*StreamEntry, error) { @@ -194,6 +201,11 @@ func (s *Stream) Next() (*StreamEntry, error) { s.err = err return nil, err } + if s.unknownSize { + if _, err := io.Copy(io.Discard, s); err != nil { + return nil, err + } + } entry, err := s.source.Next() if err == nil { err = s.checkEntry(entry) @@ -203,7 +215,12 @@ func (s *Stream) Next() (*StreamEntry, error) { return nil, err } s.entries++ - s.expanded += entry.Size + s.unknownSize = entry.Size == -1 + s.entryBytes = 0 + s.entryPath = entry.Path + if !s.unknownSize { + s.expanded += entry.Size + } return entry, nil } @@ -211,7 +228,8 @@ func (s *Stream) checkEntry(entry *StreamEntry) error { if s.entries >= s.options.MaxEntries { return fmt.Errorf("%w: exceeds %d", ErrEntryLimit, s.options.MaxEntries) } - if entry.Size < 0 || entry.Size > s.options.MaxEntryBytes { + _, singleFile := s.source.(*compressedStream) + if (entry.Size < 0 && (!singleFile || entry.Size != -1)) || entry.Size > s.options.MaxEntryBytes { return fmt.Errorf("%w: %s exceeds %d bytes", ErrEntrySizeLimit, entry.Path, s.options.MaxEntryBytes) } if entry.Size > s.options.MaxExpandedBytes-s.expanded { @@ -233,13 +251,44 @@ func (s *Stream) Read(p []byte) (int, error) { s.err = err return 0, err } - n, err := s.source.Read(p) + var n int + var err error + if s.unknownSize { + n, err = s.readUnknownSize(p) + } else { + n, err = s.source.Read(p) + } if err != nil && err != io.EOF { s.err = err } return n, err } +func (s *Stream) readUnknownSize(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + remaining := min(s.options.MaxEntryBytes-s.entryBytes, s.options.MaxExpandedBytes-s.expanded) + if remaining == 0 { + var probe [1]byte + n, err := s.source.Read(probe[:]) + if n > 0 { + if s.entryBytes == s.options.MaxEntryBytes { + return 0, fmt.Errorf("%w: %s exceeds %d bytes", ErrEntrySizeLimit, s.entryPath, s.options.MaxEntryBytes) + } + return 0, fmt.Errorf("%w: %s exceeds %d expanded bytes", ErrDecompressLimit, s.entryPath, s.options.MaxExpandedBytes) + } + return 0, err + } + if int64(len(p)) > remaining { + p = p[:remaining] + } + n, err := s.source.Read(p) + s.entryBytes += int64(n) + s.expanded += int64(n) + return n, err +} + // Recover abandons a failed ZIP entry so the next call to Next can continue. // TAR-based formats, including conda, return errors.ErrUnsupported, as does a // stream with no pending error. Cancellation and resource-limit errors remain