Merge pull request #5 from gagliardetto/cli-v0

Fix fallocate, add `version` cmd, add cross-compilation

Makefile

@@ -1,10 +1,26 @@
 DEFAULT:compile
 
 IPLD_SCHEMA_PATH := ledger.ipldsch
+LD_FLAGS := "-X main.GitCommit=`git rev-parse HEAD` -X main.GitTag=`git symbolic-ref -q --short HEAD || git describe --tags --exact-match || git rev-parse HEAD`"
 
 compile:
-	@echo "Compiling faithful-cli binary to ./bin/faithful-cli ..."
-	go build -o ./bin/faithful-cli .
+	@echo "\nCompiling faithful-cli binary for current platform ..."
+	go build -ldflags=$(LD_FLAGS) -o ./bin/faithful-cli .
+compile-all: compile-linux compile-mac compile-windows
+compile-linux:
+	@echo "\nCompiling faithful-cli binary for linux amd64 ..."
+	GOOS=linux GOARCH=amd64 go build -ldflags=$(LD_FLAGS) -o ./bin/linux/amd64/faithful-cli_linux_amd64 .
+compile-mac:
+	@echo "\nCompiling faithful-cli binary for mac amd64 ..."
+	GOOS=darwin GOARCH=amd64 go build -ldflags=$(LD_FLAGS) -o ./bin/darwin/amd64/faithful-cli_darwin_amd64 .
+
+	@echo "\nCompiling faithful-cli binary for mac arm64 ..."
+	GOOS=darwin GOARCH=arm64 go build -ldflags=$(LD_FLAGS) -o ./bin/darwin/arm64/faithful-cli_darwin_arm64 .
+compile-windows:
+	@echo "\nCompiling faithful-cli binary for windows amd64 ..."
+	GOOS=windows GOARCH=amd64 go build -ldflags=$(LD_FLAGS) -o ./bin/windows-amd64/faithful-cli_windows_amd64
+test:
+	go test -v ./...
 bindcode: install-deps
 	ipld schema codegen \
 		--generator=go-bindnode \

cmd-version.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+	"runtime/debug"
+
+	"github.com/urfave/cli/v2"
+)
+
+func newCmd_Version() *cli.Command {
+	return &cli.Command{
+		Name:        "version",
+		Description: "Print version information",
+		Before: func(c *cli.Context) error {
+			return nil
+		},
+		Flags: []cli.Flag{},
+		Action: func(c *cli.Context) error {
+			fmt.Println("YELLOWSTONE FAITHFUL CLI")
+			fmt.Printf("Tag/Branch: %s\n", GitTag)
+			fmt.Printf("Commit: %s\n", GitCommit)
+			if info, ok := debug.ReadBuildInfo(); ok {
+				fmt.Printf("More info:\n")
+				for _, setting := range info.Settings {
+					if isAnyOf(setting.Key,
+						"-compiler",
+						"GOARCH",
+						"GOOS",
+						"GOAMD64",
+						"vcs",
+						"vcs.revision",
+						"vcs.time",
+						"vcs.modified",
+					) {
+						fmt.Printf("  %s: %s\n", setting.Key, setting.Value)
+					}
+				}
+			}
+			return nil
+		},
+	}
+}
+
+var (
+	GitCommit string
+	GitTag    string
+)
+
+func isAnyOf(s string, anyOf ...string) bool {
+	for _, v := range anyOf {
+		if s == v {
+			return true
+		}
+	}
+	return false
+}

cmd-x-index-all.go

@@ -65,18 +65,19 @@ func newCmd_Index_all() *cli.Command {
 			{
 				startedAt := time.Now()
 				defer func() {
-					klog.Infof("Finished in %s", time.Since(startedAt))
+					klog.Infof("Took %s", time.Since(startedAt))
 				}()
 				klog.Infof("Creating all indexes for %s", carPath)
 				indexPaths, err := createAllIndexes(context.Background(), tmpDir, carPath, indexDir)
 				if err != nil {
 					return err
 				}
 				spew.Dump(indexPaths)
-				klog.Info("Index created")
+				klog.Info("Indexes created.")
 				if verify {
 					return verifyAllIndexes(context.Background(), carPath, indexPaths)
 				}
+				klog.Info("Skipping verification.")
 			}
 			return nil
 		},
@@ -281,6 +282,8 @@ func createAllIndexes(
 		humanize.Comma(int64(numIndexedTransactions)),
 	)
 
+	klog.Infof("Preparing to seal indexes...")
+
 	rootCID := rd.header.Roots[0]
 	paths := &IndexPaths{}
 
@@ -293,21 +296,21 @@ func createAllIndexes(
 		if err != nil {
 			return nil, fmt.Errorf("failed to seal cid_to_offset index: %w", err)
 		}
-		klog.Infof("Sealed cid_to_offset index: %s", paths.CidToOffset)
+		klog.Infof("Successfully sealed cid_to_offset index: %s", paths.CidToOffset)
 
 		klog.Infof("Sealing slot_to_cid index...")
 		paths.SlotToCid, err = slot_to_cid.Seal(ctx, carPath, rootCID)
 		if err != nil {
 			return nil, fmt.Errorf("failed to seal slot_to_cid index: %w", err)
 		}
-		klog.Infof("Sealed slot_to_cid index: %s", paths.SlotToCid)
+		klog.Infof("Successfully sealed slot_to_cid index: %s", paths.SlotToCid)
 
 		klog.Infof("Sealing sig_to_cid index...")
 		paths.SignatureToCid, err = sig_to_cid.Seal(ctx, carPath, rootCID)
 		if err != nil {
 			return nil, fmt.Errorf("failed to seal sig_to_cid index: %w", err)
 		}
-		klog.Infof("Sealed sig_to_cid index: %s", paths.SignatureToCid)
+		klog.Infof("Successfully sealed sig_to_cid index: %s", paths.SignatureToCid)
 	}
 
 	return paths, nil

compactindex/build.go

@@ -1,5 +1,3 @@
-//go:build unix
-
 package compactindex
 
 import (
@@ -13,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"syscall"
 )
 
 // Builder creates new compactindex files.
@@ -46,7 +45,7 @@ func NewBuilder(dir string, numItems uint, targetFileSize uint64) (*Builder, err
 	buckets := make([]tempBucket, numBuckets)
 	for i := range buckets {
 		name := filepath.Join(dir, fmt.Sprintf("keys-%d", i))
-		f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0666)
+		f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0o666)
 		if err != nil {
 			return nil, err
 		}
@@ -90,6 +89,13 @@ func (b *Builder) Seal(ctx context.Context, f *os.File) (err error) {
 	// Create hole to leave space for bucket header table.
 	bucketTableLen := int64(b.NumBuckets) * bucketHdrLen
 	err = fallocate(f, headerSize, bucketTableLen)
+	if errors.Is(err, syscall.EOPNOTSUPP) {
+		// The underlying file system may not support fallocate
+		err = fake_fallocate(f, headerSize, bucketTableLen)
+		if err != nil {
+			return fmt.Errorf("failed to fake fallocate() bucket table: %w", err)
+		}
+	}
 	if err != nil {
 		return fmt.Errorf("failed to fallocate() bucket table: %w", err)
 	}

compactindex/fallocate_fake.go

@@ -0,0 +1,27 @@
+package compactindex
+
+import (
+	"fmt"
+	"os"
+)
+
+func fake_fallocate(f *os.File, offset int64, size int64) error {
+	const blockSize = 4096
+	var zero [blockSize]byte
+
+	for size > 0 {
+		step := size
+		if step > blockSize {
+			step = blockSize
+		}
+
+		if _, err := f.Write(zero[:step]); err != nil {
+			return fmt.Errorf("failure while generic fallocate: %w", err)
+		}
+
+		offset += step
+		size -= step
+	}
+
+	return nil
+}

compactindex/fallocate_generic.go

@@ -7,22 +7,5 @@ import (
 )
 
 func fallocate(f *os.File, offset int64, size int64) error {
-	const blockSize = 4096
-	var zero [blockSize]byte
-
-	for size > 0 {
-		step := size
-		if step > blockSize {
-			step = blockSize
-		}
-
-		if _, err := f.Write(zero[:step]); err != nil {
-			return err
-		}
-
-		offset += step
-		size -= step
-	}
-
-	return nil
+	return fake_fallocate(f, offset, size)
 }

compactindex/fallocate_linux.go

@@ -1,10 +1,17 @@
+//go:build linux
+
 package compactindex
 
 import (
+	"fmt"
 	"os"
 	"syscall"
 )
 
 func fallocate(f *os.File, offset int64, size int64) error {
-	return syscall.Fallocate(int(f.Fd()), 0, offset, size)
+	err := syscall.Fallocate(int(f.Fd()), 0, offset, size)
+	if err != nil {
+		return fmt.Errorf("failure while linux fallocate: %w", err)
+	}
+	return nil
 }

compactindex36/build.go

@@ -1,5 +1,3 @@
-//go:build unix
-
 package compactindex36
 
 // This is a fork of the original project at https://github.com/firedancer-io/radiance/tree/main/pkg/compactindex
@@ -18,6 +16,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"syscall"
 )
 
 // Builder creates new compactindex files.
@@ -95,6 +94,13 @@ func (b *Builder) Seal(ctx context.Context, f *os.File) (err error) {
 	// Create hole to leave space for bucket header table.
 	bucketTableLen := int64(b.NumBuckets) * bucketHdrLen
 	err = fallocate(f, headerSize, bucketTableLen)
+	if errors.Is(err, syscall.EOPNOTSUPP) {
+		// The underlying file system may not support fallocate
+		err = fake_fallocate(f, headerSize, bucketTableLen)
+		if err != nil {
+			return fmt.Errorf("failed to fake fallocate() bucket table: %w", err)
+		}
+	}
 	if err != nil {
 		return fmt.Errorf("failed to fallocate() bucket table: %w", err)
 	}

compactindex36/fallocate_fake.go

@@ -0,0 +1,27 @@
+package compactindex36
+
+import (
+	"fmt"
+	"os"
+)
+
+func fake_fallocate(f *os.File, offset int64, size int64) error {
+	const blockSize = 4096
+	var zero [blockSize]byte
+
+	for size > 0 {
+		step := size
+		if step > blockSize {
+			step = blockSize
+		}
+
+		if _, err := f.Write(zero[:step]); err != nil {
+			return fmt.Errorf("failure while generic fallocate: %w", err)
+		}
+
+		offset += step
+		size -= step
+	}
+
+	return nil
+}

compactindex36/fallocate_generic.go

@@ -1,28 +1,11 @@
 //go:build !linux
 
-package compactindex
+package compactindex36
 
 import (
 	"os"
 )
 
 func fallocate(f *os.File, offset int64, size int64) error {
-	const blockSize = 4096
-	var zero [blockSize]byte
-
-	for size > 0 {
-		step := size
-		if step > blockSize {
-			step = blockSize
-		}
-
-		if _, err := f.Write(zero[:step]); err != nil {
-			return err
-		}
-
-		offset += step
-		size -= step
-	}
-
-	return nil
+	return fake_fallocate(f, offset, size)
 }

compactindex36/fallocate_linux.go

@@ -1,10 +1,17 @@
+//go:build linux
+
 package compactindex36
 
 import (
+	"fmt"
 	"os"
 	"syscall"
 )
 
 func fallocate(f *os.File, offset int64, size int64) error {
-	return syscall.Fallocate(int(f.Fd()), 0, offset, size)
+	err := syscall.Fallocate(int(f.Fd()), 0, offset, size)
+	if err != nil {
+		return fmt.Errorf("failure while linux fallocate: %w", err)
+	}
+	return nil
 }

main.go

@@ -56,6 +56,7 @@ func main() {
 			newCmd_VerifyIndex(),
 			newCmd_XTraverse(),
 			newCmd_rpcServerCar(),
+			newCmd_Version(),
 		},
 	}