From 5dc77eeddef82fe119f85ca064797bbc6cfabb79 Mon Sep 17 00:00:00 2001 From: anjor Date: Thu, 11 Jul 2024 10:54:57 +0100 Subject: [PATCH 1/4] add merge car method --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index e75a044b..8708d788 100644 --- a/main.go +++ b/main.go @@ -57,6 +57,7 @@ func main() { newCmd_Version(), newCmd_rpc(), newCmd_check_deals(), + newCmd_MergeCars(), }, } From dde4f0f5471fd343649fd46c66416ad3d83a1741 Mon Sep 17 00:00:00 2001 From: anjor Date: Thu, 11 Jul 2024 10:55:04 +0100 Subject: [PATCH 2/4] add merge car method --- cmd-merge-cars.go | 91 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 cmd-merge-cars.go diff --git a/cmd-merge-cars.go b/cmd-merge-cars.go new file mode 100644 index 00000000..9abbf181 --- /dev/null +++ b/cmd-merge-cars.go @@ -0,0 +1,91 @@ +package main + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "os" + + "github.com/urfave/cli/v2" +) + +const varintSize = 10 + +func newCmd_MergeCars() *cli.Command { + var outputFile string + return &cli.Command{ + Name: "merge-cars", + Description: "Merges split car files into a single file", + Usage: "Merges split car files into a single file", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "output-file", + Aliases: []string{"o"}, + Usage: "Output file name", + Required: true, + Destination: &outputFile, + }, + }, + Action: func(c *cli.Context) error { + paths := c.Args().Slice() + + out, err := os.Create(outputFile) + if err != nil { + return fmt.Errorf("failed to create output file: %w", err) + } + defer out.Close() + + w := bufio.NewWriter(out) + + if _, err := io.WriteString(w, nulRootCarHeader); err != nil { + return fmt.Errorf("failed to write empty header: %w", err) + } + + for _, path := range paths { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("failed to open input file %s: %w", path, err) + } + defer f.Close() + + r := bufio.NewReader(f) + err = discardHeader(r) + if err != nil { + return fmt.Errorf("failed to discard header: %w", err) + } + + io.Copy(w, r) + + } + + return nil + }, + } +} + +func discardHeader(streamBuf *bufio.Reader) error { + maybeHeaderLen, err := streamBuf.Peek(varintSize) + if err != nil { + return fmt.Errorf("failed to read header: %s\n", err) + } + + hdrLen, viLen := binary.Uvarint(maybeHeaderLen) + if hdrLen <= 0 || viLen < 0 { + return fmt.Errorf("unexpected header len = %d, varint len = %d\n", hdrLen, viLen) + } + + _, err = io.CopyN(io.Discard, streamBuf, int64(viLen)) + if err != nil { + return fmt.Errorf("failed to discard header varint: %s\n", err) + } + + // ignoring header decoding for now + _, err = io.CopyN(io.Discard, streamBuf, int64(hdrLen)) + if err != nil { + return fmt.Errorf("failed to discard header header: %s\n", err) + } + + return nil +} From 638f3a1d7153683f6dfd7251e1446a1247a4dc0f Mon Sep 17 00:00:00 2001 From: anjor Date: Thu, 11 Jul 2024 14:34:03 +0100 Subject: [PATCH 3/4] header --- cmd-merge-cars.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cmd-merge-cars.go b/cmd-merge-cars.go index 9abbf181..6cf3b2fc 100644 --- a/cmd-merge-cars.go +++ b/cmd-merge-cars.go @@ -10,7 +10,26 @@ import ( "github.com/urfave/cli/v2" ) -const varintSize = 10 +const ( + varintSize = 10 + nulRootCarHeader = "\x19" + // 25 bytes of CBOR (encoded as varint :cryingbear: ) + // map with 2 keys + "\xA2" + + // text-key with length 5 + "\x65" + "roots" + + // 1 element array + "\x81" + + // tag 42 + "\xD8\x2A" + + // bytes with length 5 + "\x45" + + // nul-identity-cid prefixed with \x00 as required in DAG-CBOR: https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-cbor.md#links + "\x00\x01\x55\x00\x00" + + // text-key with length 7 + "\x67" + "version" + + // 1, we call this v0 due to the nul-identity CID being an open question: https://github.com/ipld/go-car/issues/26#issuecomment-604299576 + "\x01" +) func newCmd_MergeCars() *cli.Command { var outputFile string From d3efc7d5f0b6b19ca509535860937b749436a32a Mon Sep 17 00:00:00 2001 From: anjor Date: Mon, 15 Jul 2024 13:20:47 +0100 Subject: [PATCH 4/4] duplicate const --- cmd-merge-cars.go | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/cmd-merge-cars.go b/cmd-merge-cars.go index 6cf3b2fc..9abbf181 100644 --- a/cmd-merge-cars.go +++ b/cmd-merge-cars.go @@ -10,26 +10,7 @@ import ( "github.com/urfave/cli/v2" ) -const ( - varintSize = 10 - nulRootCarHeader = "\x19" + // 25 bytes of CBOR (encoded as varint :cryingbear: ) - // map with 2 keys - "\xA2" + - // text-key with length 5 - "\x65" + "roots" + - // 1 element array - "\x81" + - // tag 42 - "\xD8\x2A" + - // bytes with length 5 - "\x45" + - // nul-identity-cid prefixed with \x00 as required in DAG-CBOR: https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-cbor.md#links - "\x00\x01\x55\x00\x00" + - // text-key with length 7 - "\x67" + "version" + - // 1, we call this v0 due to the nul-identity CID being an open question: https://github.com/ipld/go-car/issues/26#issuecomment-604299576 - "\x01" -) +const varintSize = 10 func newCmd_MergeCars() *cli.Command { var outputFile string