feat: creating files logs information for use in chisel.db (#105)

When creating files as part of the cut command, use a proxy that
logs information about the hash, path, mode, etc. This will then be
ingrated into a report which will be part of chisel.db.

internal/archive/archive_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/canonical/chisel/internal/archive"
 	"github.com/canonical/chisel/internal/archive/testarchive"
 	"github.com/canonical/chisel/internal/deb"
+	"github.com/canonical/chisel/internal/fsutil"
 	"github.com/canonical/chisel/internal/testutil"
 )
 
@@ -500,6 +501,7 @@ func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) {
 				{Path: "/hostname"},
 			},
 		},
+		Creator: fsutil.NewCreator(),
 	})
 	c.Assert(err, IsNil)
 

internal/deb/extract.go

@@ -25,6 +25,7 @@ type ExtractOptions struct {
 	TargetDir string
 	Extract   map[string][]ExtractInfo
 	Globbed   map[string][]string
+	Creator   *fsutil.Creator
 }
 
 type ExtractInfo struct {
@@ -185,7 +186,7 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error {
 				// Base directory for extracted content. Relevant mainly to preserve
 				// the metadata, since the extracted content itself will also create
 				// any missing directories unaccounted for in the options.
-				err := fsutil.Create(&fsutil.CreateOptions{
+				err := options.Creator.Create(&fsutil.CreateOptions{
 					Path:        filepath.Join(options.TargetDir, sourcePath),
 					Mode:        tarHeader.FileInfo().Mode(),
 					MakeParents: true,
@@ -226,7 +227,7 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error {
 			if extractInfo.Mode != 0 {
 				tarHeader.Mode = int64(extractInfo.Mode)
 			}
-			err := fsutil.Create(&fsutil.CreateOptions{
+			err := options.Creator.Create(&fsutil.CreateOptions{
 				Path:        targetPath,
 				Mode:        tarHeader.FileInfo().Mode(),
 				Data:        pathReader,

internal/deb/extract_test.go

@@ -6,6 +6,7 @@ import (
 	. "gopkg.in/check.v1"
 
 	"github.com/canonical/chisel/internal/deb"
+	"github.com/canonical/chisel/internal/fsutil"
 	"github.com/canonical/chisel/internal/testutil"
 )
 
@@ -290,6 +291,7 @@ func (s *S) TestExtract(c *C) {
 		options := test.options
 		options.Package = "base-files"
 		options.TargetDir = dir
+		options.Creator = fsutil.NewCreator()
 
 		if test.globbed != nil {
 			options.Globbed = make(map[string][]string)

internal/fsutil/create.go

@@ -1,7 +1,10 @@
 package fsutil
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
+	"hash"
 	"io"
 	"io/fs"
 	"os"
@@ -18,8 +21,32 @@ type CreateOptions struct {
 	MakeParents bool
 }
 
-// Creates a filesystem entry according to the provided options.
-func Create(o *CreateOptions) error {
+type Info struct {
+	Path string
+	Mode fs.FileMode
+	Hash string
+	Size int
+	Link string
+}
+
+type Creator struct {
+	// Created keeps track of information about the filesystem entries created.
+	// If an entry is created several times it only tracks the latest one.
+	Created map[string]Info
+}
+
+func NewCreator() *Creator {
+	return &Creator{Created: make(map[string]Info)}
+}
+
+// Create creates a filesystem entry according to the provided options.
+func (c *Creator) Create(options *CreateOptions) error {
+	rp := &readerProxy{inner: options.Data, h: sha256.New()}
+	// Use the proxy instead of the raw Reader.
+	optsCopy := *options
+	optsCopy.Data = rp
+	o := &optsCopy
+
 	var err error
 	if o.MakeParents {
 		if err := os.MkdirAll(filepath.Dir(o.Path), 0755); err != nil {
@@ -36,7 +63,19 @@ func Create(o *CreateOptions) error {
 	default:
 		err = fmt.Errorf("unsupported file type: %s", o.Path)
 	}
-	return err
+	if err != nil {
+		return err
+	}
+
+	info := Info{
+		Path: o.Path,
+		Mode: o.Mode,
+		Hash: hex.EncodeToString(rp.h.Sum(nil)),
+		Size: rp.size,
+		Link: o.Link,
+	}
+	c.Created[o.Path] = info
+	return nil
 }
 
 func createDir(o *CreateOptions) error {
@@ -84,3 +123,20 @@ func createSymlink(o *CreateOptions) error {
 	}
 	return os.Symlink(o.Link, o.Path)
 }
+
+// readerProxy implements the io.Reader interface proxying the calls to its
+// inner io.Reader. On each read, the proxy keeps track of the file size and hash.
+type readerProxy struct {
+	inner io.Reader
+	h     hash.Hash
+	size  int
+}
+
+var _ io.Reader = (*readerProxy)(nil)
+
+func (rp *readerProxy) Read(p []byte) (n int, err error) {
+	n, err = rp.inner.Read(p)
+	rp.h.Write(p[:n])
+	rp.size += n
+	return n, err
+}

internal/fsutil/create_test.go

@@ -2,8 +2,10 @@ package fsutil_test
 
 import (
 	"bytes"
+	"fmt"
 	"io/fs"
 	"path/filepath"
+	"strings"
 	"syscall"
 
 	. "gopkg.in/check.v1"
@@ -73,18 +75,60 @@ func (s *S) TestCreate(c *C) {
 	}()
 
 	for _, test := range createTests {
+		if test.result == nil {
+			// Empty map for no files created.
+			test.result = make(map[string]string)
+		}
 		c.Logf("Options: %v", test.options)
 		dir := c.MkDir()
 		options := test.options
 		options.Path = filepath.Join(dir, options.Path)
-		err := fsutil.Create(&options)
+		creator := fsutil.NewCreator()
+		err := creator.Create(&options)
 		if test.error != "" {
 			c.Assert(err, ErrorMatches, test.error)
-			continue
 		} else {
 			c.Assert(err, IsNil)
 		}
-		result := testutil.TreeDump(dir)
-		c.Assert(result, DeepEquals, test.result)
+		c.Assert(testutil.TreeDump(dir), DeepEquals, test.result)
+		if test.options.MakeParents {
+			// The creator does not record the parent directories created
+			// implicitly.
+			for path, info := range treeDumpFSCreator(creator, dir) {
+				c.Assert(info, Equals, test.result[path])
+			}
+		} else {
+			c.Assert(treeDumpFSCreator(creator, dir), DeepEquals, test.result)
+		}
+	}
+}
+
+// treeDumpFSCreator dumps the contents stored in Creator about the filesystem
+// entries created using the same format as [testutil.TreeDump].
+func treeDumpFSCreator(cr *fsutil.Creator, root string) map[string]string {
+	result := make(map[string]string)
+	for _, file := range cr.Created {
+		path := strings.TrimPrefix(file.Path, root)
+		fperm := file.Mode.Perm()
+		if file.Mode&fs.ModeSticky != 0 {
+			fperm |= 01000
+		}
+		switch file.Mode.Type() {
+		case fs.ModeDir:
+			result[path+"/"] = fmt.Sprintf("dir %#o", fperm)
+		case fs.ModeSymlink:
+			result[path] = fmt.Sprintf("symlink %s", file.Link)
+		case 0: // Regular
+			var entry string
+			if file.Size == 0 {
+				entry = fmt.Sprintf("file %#o empty", file.Mode.Perm())
+			} else {
+				entry = fmt.Sprintf("file %#o %s", fperm, file.Hash[:8])
+			}
+			result[path] = entry
+		default:
+			panic(fmt.Errorf("unknown file type %d: %s", file.Mode.Type(), path))
+		}
 	}
+	return result
 }

internal/setup/fetch.go

@@ -137,7 +137,7 @@ func extractTar(dataReader io.Reader, targetDir string) error {
 
 		//debugf("Extracting header: %#v", tarHeader)
 
-		err = fsutil.Create(&fsutil.CreateOptions{
+		err = fsutil.NewCreator().Create(&fsutil.CreateOptions{
 			Path:        filepath.Join(targetDir, sourcePath),
 			Mode:        tarHeader.FileInfo().Mode(),
 			Data:        tarReader,

internal/slicer/slicer.go

@@ -153,11 +153,13 @@ func Run(options *RunOptions) error {
 		if reader == nil {
 			continue
 		}
+		creator := fsutil.NewCreator()
 		err := deb.Extract(reader, &deb.ExtractOptions{
 			Package:   slice.Package,
 			Extract:   extract[slice.Package],
 			TargetDir: targetDir,
 			Globbed:   globbedPaths,
+			Creator:   creator,
 		})
 		reader.Close()
 		packages[slice.Package] = nil
@@ -211,7 +213,8 @@ func Run(options *RunOptions) error {
 				return fmt.Errorf("internal error: cannot extract path of kind %q", pathInfo.Kind)
 			}
 
-			err := fsutil.Create(&fsutil.CreateOptions{
+			creator := fsutil.NewCreator()
+			err := creator.Create(&fsutil.CreateOptions{
 				Path:        targetPath,
 				Mode:        tarHeader.FileInfo().Mode(),
 				Data:        fileContent,