protobuf: add vtproto to supplement protobuf marshaling

This adds the vtproto generator to generate code that results in faster
marshaling and unmarshaling. vtproto is similar to gogo but it generates
additional code on top of the existing protobuf structures rather than
changing the generator itself.

Signed-off-by: Jonathan A. Sternberg <[email protected]>

bench/go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v1.0.0-rc93 // indirect
+	github.com/planetscale/vtprotobuf v0.6.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	go.opencensus.io v0.22.3 // indirect
 	golang.org/x/sys v0.11.0 // indirect

bench/go.sum

@@ -468,6 +468,8 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA=
+github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/moby/patternmatcher v0.5.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/pkg/errors v0.9.1
+	github.com/planetscale/vtprotobuf v0.6.0
 	github.com/stretchr/testify v1.8.4
 	github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205
 	golang.org/x/sync v0.1.0

go.sum

@@ -18,6 +18,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA=
+github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=

hack/dockerfiles/generated-files.Dockerfile

@@ -18,23 +18,37 @@ RUN <<EOT
   rm -f /opt/protoc/readme.md
 EOT
 
+FROM base AS protoc-libs
+WORKDIR /app
+RUN --mount=type=bind,source=go.mod,target=/app/go.mod \
+    --mount=type=bind,source=go.sum,target=/app/go.sum \
+    --mount=type=cache,target=/root/.cache \
+    --mount=type=cache,target=/go/pkg/mod <<EOT
+  set -e
+  mkdir -p /opt/protoc
+  go mod download github.com/planetscale/vtprotobuf
+  cp -r $(go list -m -f='{{.Dir}}' github.com/planetscale/vtprotobuf)/include /opt/protoc
+EOT
+
 FROM base AS tools
 WORKDIR /app
 RUN --mount=type=bind,source=go.mod,target=/app/go.mod \
     --mount=type=bind,source=go.sum,target=/app/go.sum \
     --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/go/pkg/mod \
   go install \
+    github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto \
     google.golang.org/protobuf/cmd/protoc-gen-go
 COPY --link --from=protoc /opt/protoc /usr/local
+COPY --link --from=protoc-libs /opt/protoc /usr/local
 
 FROM tools AS generate
 RUN --mount=target=github.com/tonistiigi/fsutil \
     --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/go/pkg/mod <<EOT
   set -e
   mkdir /out
-  find github.com/tonistiigi/fsutil -name '*.proto' | xargs protoc --go_out=/out
+  find github.com/tonistiigi/fsutil -name '*.proto' | xargs protoc --go_out=/out --go-vtproto_out=/out
 EOT
 
 FROM scratch AS update

receive.go

@@ -197,7 +197,7 @@ func (r *receiver) run(ctx context.Context) error {
 		}
 		var p types.Packet
 		for {
-			p = types.Packet{Data: p.Data[:0]}
+			p.ResetVT()
 			if err := r.conn.RecvMsg(&p); err != nil {
 				return err
 			}

tools/tools.go

@@ -3,5 +3,6 @@
 package tools
 
 import (
+	_ "github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto"
 	_ "google.golang.org/protobuf/cmd/protoc-gen-go"
 )

types/stat.go

@@ -1,40 +1,19 @@
 package types
 
-import (
-	"os"
-
-	"google.golang.org/protobuf/proto"
-)
+import "os"
 
 func (s *Stat) IsDir() bool {
 	return os.FileMode(s.Mode).IsDir()
 }
 
 func (s *Stat) Marshal() ([]byte, error) {
-	return proto.MarshalOptions{Deterministic: true}.Marshal(s)
+	return s.MarshalVTStrict()
 }
 
 func (s *Stat) Unmarshal(dAtA []byte) error {
-	return proto.UnmarshalOptions{Merge: true}.Unmarshal(dAtA, s)
+	return s.UnmarshalVT(dAtA)
 }
 
 func (s *Stat) Clone() *Stat {
-	clone := &Stat{
-		Path:     s.Path,
-		Mode:     s.Mode,
-		Uid:      s.Uid,
-		Gid:      s.Gid,
-		Size:     s.Size,
-		ModTime:  s.ModTime,
-		Linkname: s.Linkname,
-		Devmajor: s.Devmajor,
-		Devminor: s.Devminor,
-	}
-	if s.Xattrs != nil {
-		s.Xattrs = make(map[string][]byte, len(s.Xattrs))
-		for k, v := range s.Xattrs {
-			clone.Xattrs[k] = v
-		}
-	}
-	return clone
+	return s.CloneVT()
 }

types/stat.proto

@@ -4,6 +4,8 @@ package fsutil.types;
 
 option go_package = "github.com/tonistiigi/fsutil/types";
 
+import "github.com/planetscale/vtprotobuf/vtproto/ext.proto";
+
 message Stat {
   string path = 1;
   uint32 mode = 2;