diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index b89a8d71d..2db93afa9 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -20,6 +20,7 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/archive/testarchive" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" ) @@ -1235,7 +1236,7 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { c.Assert(info.Name, DeepEquals, test.pkg) c.Assert(info.Arch, DeepEquals, arch) - err = tarball.Extract(pkg, &tarball.ExtractOptions{ + err = tarball.Extract(deb.OpenTar(pkg), &tarball.ExtractOptions{ Package: test.pkg, TargetDir: extractDir, Extract: map[string][]tarball.ExtractInfo{ diff --git a/internal/bin/extract.go b/internal/bin/extract.go new file mode 100644 index 000000000..41533ac54 --- /dev/null +++ b/internal/bin/extract.go @@ -0,0 +1,27 @@ +package bin + +import ( + "io" + + "github.com/ulikunitz/xz" +) + +// DataReader takes a Reader for a bin package (an XZ-compressed tarball) and +// returns a Reader to the inner tarball. +func DataReader(pkgReader io.Reader) (io.ReadCloser, error) { + xzReader, err := xz.NewReader(pkgReader) + if err != nil { + return nil, err + } + return io.NopCloser(xzReader), nil +} + +func OpenTar(pkgReader io.ReadSeeker) func() (io.ReadCloser, error) { + return func() (io.ReadCloser, error) { + _, err := pkgReader.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + return DataReader(pkgReader) + } +} diff --git a/internal/bin/extract_test.go b/internal/bin/extract_test.go new file mode 100644 index 000000000..6b23020dd --- /dev/null +++ b/internal/bin/extract_test.go @@ -0,0 +1,41 @@ +package bin_test + +import ( + "archive/tar" + "bytes" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/bin" + "github.com/canonical/chisel/internal/testutil" +) + +func (s *S) TestDataReader(c *C) { + data := testutil.MustMakeBin([]testutil.TarEntry{ + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "content"), + }) + + dataReader, err := bin.DataReader(bytes.NewReader(data)) + c.Assert(err, IsNil) + tarReader := tar.NewReader(dataReader) + _, err = tarReader.Next() + c.Assert(err, IsNil) +} + +func (s *S) TestDataReaderInvalid(c *C) { + _, err := bin.DataReader(bytes.NewReader([]byte("not an xz stream"))) + c.Assert(err, ErrorMatches, "xz.*") +} + +func (s *S) TestOpenTar(c *C) { + data := testutil.MustMakeBin([]testutil.TarEntry{ + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "content"), + }) + pkgReader := testutil.ReadSeekNopCloser(bytes.NewReader(data)) + openTar := bin.OpenTar(pkgReader) + + // Each call must return a fresh stream over the same content. + testutil.CheckOpenTar(c, openTar) +} diff --git a/internal/bin/log.go b/internal/bin/log.go new file mode 100644 index 000000000..e0f72a8df --- /dev/null +++ b/internal/bin/log.go @@ -0,0 +1,53 @@ +package bin + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/bin/suite_test.go b/internal/bin/suite_test.go new file mode 100644 index 000000000..59d1c4660 --- /dev/null +++ b/internal/bin/suite_test.go @@ -0,0 +1,13 @@ +package bin_test + +import ( + "testing" + + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) diff --git a/internal/deb/extract.go b/internal/deb/extract.go index bf543c2d8..301343629 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -47,3 +47,13 @@ func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { return dataReader, nil } + +func OpenTar(pkgReader io.ReadSeeker) func() (io.ReadCloser, error) { + return func() (io.ReadCloser, error) { + _, err := pkgReader.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + return DataReader(pkgReader) + } +} diff --git a/internal/deb/extract_test.go b/internal/deb/extract_test.go new file mode 100644 index 000000000..d9fc2665b --- /dev/null +++ b/internal/deb/extract_test.go @@ -0,0 +1,18 @@ +package deb_test + +import ( + "bytes" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/deb" + "github.com/canonical/chisel/internal/testutil" +) + +func (s *S) TestOpenTar(c *C) { + pkgReader := testutil.ReadSeekNopCloser(bytes.NewReader(testutil.PackageData["test-package"])) + openTar := deb.OpenTar(pkgReader) + + // Each call must return a fresh stream over the same content. + testutil.CheckOpenTar(c, openTar) +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index ce7911621..6566692c7 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -16,6 +16,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" @@ -146,7 +147,7 @@ func Run(options *RunOptions) error { } // Fetch all packages, using the selection order. - packages := make(map[string]io.ReadSeekCloser) + packages := make(map[string]tarball.OpenTarFunc) var pkgInfos []manifestutil.PackageInfo for _, slice := range options.Selection.Slices { if packages[slice.Package] != nil { @@ -158,7 +159,7 @@ func Run(options *RunOptions) error { return err } defer reader.Close() - packages[slice.Package] = reader + packages[slice.Package] = deb.OpenTar(reader) pkgInfos = append(pkgInfos, info) } @@ -235,17 +236,16 @@ func Run(options *RunOptions) error { // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { - reader := packages[slice.Package] - if reader == nil { + openTar := packages[slice.Package] + if openTar == nil { continue } - err := tarball.Extract(reader, &tarball.ExtractOptions{ + err := tarball.Extract(openTar, &tarball.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], TargetDir: targetDir, Create: create, }) - reader.Close() packages[slice.Package] = nil if err != nil { return err diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index 40db2cbac..b33bb9dbf 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -12,7 +12,6 @@ import ( "strings" "syscall" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/strdist" ) @@ -58,7 +57,11 @@ func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { return options, nil } -func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { +// OpenTarFunc opens a fresh reader over a package's inner tar stream. +// Each call must return a stream from its start. +type OpenTarFunc func() (io.ReadCloser, error) + +func Extract(openTar OpenTarFunc, options *ExtractOptions) (err error) { defer func() { if err != nil { err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err) @@ -79,15 +82,15 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { return err } - return extractData(pkgReader, validOpts) + return extractEntries(openTar, validOpts) } -func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { - dataReader, err := deb.DataReader(pkgReader) +func extractEntries(openTar OpenTarFunc, options *ExtractOptions) error { + tarStream, err := openTar() if err != nil { return err } - defer dataReader.Close() + defer tarStream.Close() oldUmask := syscall.Umask(0) defer func() { @@ -117,10 +120,10 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { // create them with the permissions defined in the tarball. // // The assumption is that the tar entries of the parent directories appear - // before the entry for the file itself. This is the case for .deb files but - // not for all tarballs. + // before the entry for the file itself. This is the case for the tarballs + // produced by common packaging tools but not for all tarballs. tarDirMode := make(map[string]fs.FileMode) - tarReader := tar.NewReader(dataReader) + tarReader := tar.NewReader(tarStream) for { tarHeader, err := tarReader.Next() if err == io.EOF { @@ -261,11 +264,7 @@ func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { ExtractOptions: options, pendingLinks: pendingHardLinks, } - _, err := pkgReader.Seek(0, io.SeekStart) - if err != nil { - return err - } - err = extractHardLinks(pkgReader, extractHardLinkOptions) + err := extractHardLinks(openTar, extractHardLinkOptions) if err != nil { return err } @@ -299,14 +298,14 @@ type extractHardLinkOptions struct { // extractHardLinks iterates through the tarball a second time to extract the // hard links that were not extracted in the first pass. -func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { - dataReader, err := deb.DataReader(pkgReader) +func extractHardLinks(openTar OpenTarFunc, opts *extractHardLinkOptions) error { + tarStream, err := openTar() if err != nil { return err } - defer dataReader.Close() + defer tarStream.Close() - tarReader := tar.NewReader(dataReader) + tarReader := tar.NewReader(tarStream) for { tarHeader, err := tarReader.Next() if err == io.EOF { diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index fd5eb6147..a83d80a2b 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -1,7 +1,6 @@ package tarball_test import ( - "bytes" "os" "path" "path/filepath" @@ -17,7 +16,7 @@ import ( type extractTest struct { summary string - pkgdata []byte + pkg *testutil.TestPkg options tarball.ExtractOptions hackopt func(c *C, o *tarball.ExtractOptions) result map[string]string @@ -26,16 +25,18 @@ type extractTest struct { error string } +var testPkg = testutil.NewTestPkg(testutil.TestPackageEntries...) + var extractTests = []extractTest{{ summary: "Extract nothing", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: nil, }, result: map[string]string{}, }, { summary: "Extract a few entries", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/file": []tarball.ExtractInfo{{ @@ -69,7 +70,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Extract a few entries, nil Create closure", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/file": []tarball.ExtractInfo{{ @@ -105,7 +106,7 @@ var extractTests = []extractTest{{ }, }, { summary: "Copy a couple of entries elsewhere", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/file": []tarball.ExtractInfo{{ @@ -127,7 +128,7 @@ var extractTests = []extractTest{{ notCreated: []string{"/foo/", "/foo/bar/"}, }, { summary: "Copy same file twice", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/file": []tarball.ExtractInfo{{ @@ -147,7 +148,7 @@ var extractTests = []extractTest{{ notCreated: []string{"/dir/bar/", "/dir/foo/"}, }, { summary: "Globbing a single dir level", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/s*/": []tarball.ExtractInfo{{ @@ -162,7 +163,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Globbing for files with multiple levels at once", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/s**": []tarball.ExtractInfo{{ @@ -180,7 +181,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Globbing multiple paths", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/s**": []tarball.ExtractInfo{{ @@ -202,7 +203,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Globbing must have matching source and target", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/foo/b**": []tarball.ExtractInfo{{ @@ -213,7 +214,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": when using wildcards source and target paths must match: /foo/b\*\*`, }, { summary: "Globbing must also have a single target", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/foo/b**": []tarball.ExtractInfo{{ @@ -226,7 +227,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": when using wildcards source and target paths must match: /foo/b\*\*`, }, { summary: "Globbing cannot change modes", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/n**": []tarball.ExtractInfo{{ @@ -238,7 +239,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": when using wildcards source and target paths must match: /dir/n\*\*`, }, { summary: "Missing file", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/missing-file": []tarball.ExtractInfo{{ @@ -249,7 +250,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at /missing-file`, }, { summary: "Missing directory", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/missing-dir/": []tarball.ExtractInfo{{ @@ -260,7 +261,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at /missing-dir/`, }, { summary: "Missing glob", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/missing-dir/**": []tarball.ExtractInfo{{ @@ -271,7 +272,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at /missing-dir/\*\*`, }, { summary: "Missing multiple entries", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/missing-file": []tarball.ExtractInfo{{ @@ -285,7 +286,7 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at:\n- /missing-dir/\n- /missing-file`, }, { summary: "Optional entries may be missing", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/": []tarball.ExtractInfo{{ @@ -307,7 +308,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Optional entries mixed in cannot be missing", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/missing-file": []tarball.ExtractInfo{{ @@ -322,11 +323,11 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": no content at /dir/missing-file`, }, { summary: "Extract non-ASCII path and preserve parent directories permissions", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Dir(0766, "./日本/"), testutil.Reg(0644, "./日本/語", "whatever"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/日本/語": []tarball.ExtractInfo{{ @@ -341,7 +342,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Entries for same destination must have the same mode", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/": []tarball.ExtractInfo{{ @@ -356,11 +357,11 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": path /dir/ requested twice with diverging mode: 0777 != 0000`, }, { summary: "Single hard link entry can be extracted with the content entry", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Reg(0644, "./file", "text for file"), testutil.Hrd(0644, "./hardlink", "./file"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -375,11 +376,11 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Single hard link entry can be extracted without the content entry", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Reg(0644, "./file", "text for file"), testutil.Hrd(0644, "./hardlink", "./file"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/hardlink": []tarball.ExtractInfo{{ @@ -393,10 +394,10 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Dangling hard link", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink", "./non-existing-target"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/hardlink": []tarball.ExtractInfo{{ @@ -407,11 +408,11 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": cannot create hard link /hardlink: no content at /non-existing-target`, }, { summary: "Multiple dangling hard links", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink1", "./non-existing-target"), testutil.Hrd(0644, "./hardlink2", "./non-existing-target"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -422,11 +423,11 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": cannot create hard link /hardlink1: no content at /non-existing-target`, }, { summary: "Hard link does not follow the symlink", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Lnk(0644, "./symlink", "./file"), testutil.Hrd(0644, "./hardlink", "./symlink"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -441,7 +442,7 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Explicit extraction overrides existing file", - pkgdata: testutil.PackageData["test-package"], + pkg: testPkg, options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/": []tarball.ExtractInfo{{ @@ -460,10 +461,10 @@ var extractTests = []extractTest{{ notCreated: []string{}, }, { summary: "Hardlink cannot escape target directory", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink", "/etc/group"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -474,10 +475,10 @@ var extractTests = []extractTest{{ error: `cannot extract from package "test-package": invalid link target /etc/group`, }, { summary: "Cannot extract outside of target directory", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Reg(0644, "./../file", "hijacking system file"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/**": []tarball.ExtractInfo{{ @@ -511,7 +512,7 @@ func (s *S) TestExtract(c *C) { test.hackopt(c, &options) } - err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) + err := tarball.Extract(test.pkg.OpenTar(), &options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -538,16 +539,16 @@ func (s *S) TestExtract(c *C) { var extractCreateCallbackTests = []struct { summary string - pkgdata []byte + pkg *testutil.TestPkg options tarball.ExtractOptions calls map[string][]tarball.ExtractInfo }{{ summary: "Create is called with the set of ExtractInfo(s) that match the file", - pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + pkg: testutil.NewTestPkg( testutil.Dir(0755, "./"), testutil.Dir(0766, "./dir/"), testutil.Reg(0644, "./dir/file", "whatever"), - }), + ), options: tarball.ExtractOptions{ Extract: map[string][]tarball.ExtractInfo{ "/dir/": []tarball.ExtractInfo{{ @@ -622,7 +623,7 @@ func (s *S) TestExtractCreateCallback(c *C) { return nil } - err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) + err := tarball.Extract(test.pkg.OpenTar(), &options) c.Assert(err, IsNil) c.Assert(createExtractInfos, DeepEquals, test.calls) diff --git a/internal/testutil/opentar.go b/internal/testutil/opentar.go new file mode 100644 index 000000000..e2be98bf9 --- /dev/null +++ b/internal/testutil/opentar.go @@ -0,0 +1,47 @@ +package testutil + +import ( + "archive/tar" + "fmt" + "io" + + "gopkg.in/check.v1" +) + +func CheckOpenTar(c *check.C, openTar func() (io.ReadCloser, error)) { + first, err := openTar() + c.Assert(err, check.IsNil, check.Commentf("first call failed")) + entries, err := readTarEntries(first) + c.Assert(err, check.IsNil, check.Commentf("cannot read first stream")) + err = first.Close() + c.Assert(err, check.IsNil, check.Commentf("cannot close first stream")) + + second, err := openTar() + c.Assert(err, check.IsNil, check.Commentf("second call failed")) + entries2, err := readTarEntries(second) + c.Assert(err, check.IsNil, check.Commentf("cannot read second stream")) + err = second.Close() + c.Assert(err, check.IsNil, check.Commentf("cannot close second stream")) + + c.Assert(entries2, check.DeepEquals, entries, check.Commentf("second stream differs from the first")) +} + +// readTarEntries reads all entries of a tar stream into a comparable form. +func readTarEntries(tarStream io.Reader) ([]string, error) { + tarReader := tar.NewReader(tarStream) + var entries []string + for { + header, err := tarReader.Next() + if err == io.EOF { + return entries, nil + } + if err != nil { + return nil, err + } + content, err := io.ReadAll(tarReader) + if err != nil { + return nil, err + } + entries = append(entries, fmt.Sprintf("%c %s %o %q", header.Typeflag, header.Name, header.Mode, string(content))) + } +} diff --git a/internal/testutil/pkgdata.go b/internal/testutil/pkgdata.go index f901873d3..813dea16e 100644 --- a/internal/testutil/pkgdata.go +++ b/internal/testutil/pkgdata.go @@ -8,6 +8,7 @@ import ( "github.com/blakesmith/ar" "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" ) var PackageData = map[string][]byte{} @@ -107,6 +108,15 @@ func makeTar(entries []TarEntry) ([]byte, error) { return buf.Bytes(), nil } +// MustMakeTar returns the tar stream holding the given entries. +func MustMakeTar(entries []TarEntry) []byte { + data, err := makeTar(entries) + if err != nil { + panic(err) + } + return data +} + func compressBytesZstd(input []byte) ([]byte, error) { var buf bytes.Buffer writer, err := zstd.NewWriter(&buf) @@ -160,6 +170,37 @@ func MustMakeDeb(entries []TarEntry) []byte { return data } +func compressBytesXZ(input []byte) ([]byte, error) { + var buf bytes.Buffer + writer, err := xz.NewWriter(&buf) + if err != nil { + return nil, err + } + if _, err = writer.Write(input); err != nil { + return nil, err + } + if err = writer.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func MakeBin(entries []TarEntry) ([]byte, error) { + tarData, err := makeTar(entries) + if err != nil { + return nil, err + } + return compressBytesXZ(tarData) +} + +func MustMakeBin(entries []TarEntry) []byte { + data, err := MakeBin(entries) + if err != nil { + panic(err) + } + return data +} + // Reg is a shortcut for creating a regular file TarEntry structure (with // tar.Typeflag set tar.TypeReg). Reg stands for "REGular file". func Reg(mode int64, path, content string) TarEntry { diff --git a/internal/testutil/testpkg.go b/internal/testutil/testpkg.go new file mode 100644 index 000000000..fb0f9159f --- /dev/null +++ b/internal/testutil/testpkg.go @@ -0,0 +1,30 @@ +package testutil + +import ( + "bytes" + "io" +) + +// TestPkg is a package holding an in-memory tar stream, providing an +// OpenTarFunc over it. It is not tied to any package format. +type TestPkg struct { + tarData []byte +} + +// NewTestPkg returns a TestPkg holding a tar stream with the given entries. +func NewTestPkg(entries ...TarEntry) *TestPkg { + return &TestPkg{tarData: MustMakeTar(entries)} +} + +// NewTestPkgFromTar returns a TestPkg holding the given raw tar stream. +func NewTestPkgFromTar(tarData []byte) *TestPkg { + return &TestPkg{tarData: tarData} +} + +// OpenTar returns a function opening a fresh reader over the package's +// tar stream on every call, conforming to the OpenTarFunc contract. +func (p *TestPkg) OpenTar() func() (io.ReadCloser, error) { + return func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(p.tarData)), nil + } +}