Skip to content

Commit

Permalink
add new splitting command
Browse files Browse the repository at this point in the history
  • Loading branch information
anjor committed Jul 1, 2024
1 parent 7a528ee commit bddd2a4
Showing 1 changed file with 80 additions and 0 deletions.
80 changes: 80 additions & 0 deletions cmd-car-split.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package main

import (
"context"
"io/fs"
"os"

"github.com/rpcpool/yellowstone-faithful/accum"
"github.com/rpcpool/yellowstone-faithful/carreader"
"github.com/rpcpool/yellowstone-faithful/iplddecoders"
"github.com/urfave/cli/v2"
"k8s.io/klog/v2"
)

func newCmd_SplitCar() *cli.Command {
return &cli.Command{
Name: "split-car",
Description: "Splits an epoch car file into smaller chunks. Each chunk corresponds to a subset.",
ArgsUsage: "<epoch-car-path>",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "size",
Aliases: []string{"s"},
Value: 31 * 1024 * 1024 * 1024, // 31 GiB
Usage: "Target size in bytes to chunk CARs to.",
Required: false,
},
},
Action: func(c *cli.Context) error {
carPath := c.Args().First()
var file fs.File
var err error
if carPath == "-" {
file = os.Stdin
} else {
file, err = os.Open(carPath)
if err != nil {
klog.Exit(err.Error())
}
defer file.Close()
}

rd, err := carreader.New(file)
if err != nil {
klog.Exitf("Failed to open CAR: %s", err)
}
{
// print roots:
roots := rd.Header.Roots
klog.Infof("Roots: %d", len(roots))
for i, root := range roots {
if i == 0 && len(roots) == 1 {
klog.Infof("- %s (Epoch CID)", root.String())
} else {
klog.Infof("- %s", root.String())
}
}
}

maxSize := c.Int("size")

Check failure on line 60 in cmd-car-split.go

GitHub Actions / test (1.21.x, ubuntu-latest)

maxSize declared and not used
accum := accum.NewObjectAccumulator(
rd,
iplddecoders.KindBlock,
func(owm1 *accum.ObjectWithMetadata, owm2 []accum.ObjectWithMetadata) error {
size := 0

Check failure on line 65 in cmd-car-split.go

GitHub Actions / test (1.21.x, ubuntu-latest)

size declared and not used

return nil
},
iplddecoders.KindEpoch,
iplddecoders.KindSubset,
)

if err := accum.Run((context.Background())); err != nil {
klog.Exitf("error while accumulating objects: %w", err)
}

return nil
},
}
}

0 comments on commit bddd2a4

Please sign in to comment.