enhancements for cloud-init support for containers

- add support for gzip encoding for write_files
- add better validation for the user-data payload

Signed-off-by: Paul Gaiduk <[email protected]>

pkg/pillar/go.mod

@@ -12,8 +12,10 @@ require (
 	github.com/digitalocean/go-qemu v0.0.0-20220826173844-d5f5e3ceed89
 	github.com/docker/docker v20.10.24+incompatible
 	github.com/eriknordmark/ipinfo v0.0.0-20230728132417-2d8f4da903d7
+	github.com/eshard/uevent v1.0.1
 	github.com/fsnotify/fsnotify v1.5.1
 	github.com/go-chi/chi/v5 v5.0.10
+	github.com/go-playground/validator/v10 v10.15.5
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/protobuf v1.5.3
 	github.com/google/go-cmp v0.5.9
@@ -40,7 +42,7 @@ require (
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/shjala/gostats v0.0.0-20230215094906-69a32327600f
 	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.2
 	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e
 	github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868
 	golang.org/x/crypto v0.14.0
@@ -81,10 +83,12 @@ require (
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/eshard/uevent v1.0.1 // indirect
 	github.com/estesp/manifest-tool/v2 v2.0.6-0.20220728154431-89d791ab7966 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/godbus/dbus/v5 v5.0.6 // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -99,6 +103,7 @@ require (
 	github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531 // indirect
 	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/kr/fs v0.1.0 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/lithammer/shortuuid/v4 v4.0.0 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect

pkg/pillar/go.sum

@@ -662,6 +662,8 @@ github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWp
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
+github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
+github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
 github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
@@ -711,6 +713,13 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
+github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@@ -1118,6 +1127,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
+github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/lf-edge/edge-containers v0.0.0-20221025050409-93c34bebadd2 h1:ckxNk8MEdATh8ZsArR7puG9PI5izRzCT+/TE9dvuAwM=
 github.com/lf-edge/edge-containers v0.0.0-20221025050409-93c34bebadd2/go.mod h1:eA41YxPbZRVvewIYRzmqDB1PeLQXxCy9WQEc3AVCsPI=
 github.com/lf-edge/eve-api/go v0.0.0-20231011200019-cb3cb1275e0d h1:PVKqYtPsH5BAIYfOaKej/+lc7+GKcFZBGnzbS6JWbrE=
@@ -1555,8 +1566,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/surma/gocpio v1.0.2-0.20160926205914-fcb68777e7dc h1:iA3Eg1OVd2o0M4M+0PBsBBssMz98L8CUH7x0xVkuyUA=
 github.com/surma/gocpio v1.0.2-0.20160926205914-fcb68777e7dc/go.mod h1:zaLNaN+EDnfSnNdWPJJf9OZxWF817w5dt8JNzF9LCVI=

pkg/pillar/utils/cloudconfig/cloudconfig.go

@@ -1,14 +1,17 @@
 package cloudconfig
 
 import (
+	"compress/gzip"
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
+	"github.com/go-playground/validator/v10"
 	"github.com/lf-edge/eve/pkg/pillar/base"
 	fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file"
 	"gopkg.in/yaml.v2"
@@ -18,18 +21,24 @@ import (
 // Only supported fields are defined here. The rest is ignored.
 type CloudConfig struct {
 	RunCmd     []string       `yaml:"runcmd"`
-	WriteFiles []WritableFile `yaml:"write_files"` //nolint:tagliatelle // cloud-init standard uses snake_case
+	WriteFiles []WritableFile `yaml:"write_files" validate:"dive"` //nolint:tagliatelle // cloud-init standard uses snake_case
 }
 
 // WritableFile represents a file that can be written to disk with the specified content, permissions, encoding and owner.
 type WritableFile struct {
-	Path        string `yaml:"path"`
-	Content     string `yaml:"content"`
+	Path        string `yaml:"path" validate:"required"`
+	Content     string `yaml:"content" validate:"required"`
 	Permissions string `yaml:"permissions"`
 	Encoding    string `yaml:"encoding"`
 	Owner       string `yaml:"owner"`
 }
 
+var validate = validator.New()
+
+// MaxDecompressedContentSize is the maximum size of a file that can be written to disk after decompression.
+// This is to prevent a DoS attack by sending a compressed file that is too big to be decompressed.
+const MaxDecompressedContentSize = 10 * 1024 * 1024 // 10 MB
+
 // IsCloudConfig checks if the given string is a cloud-config file by checking if the first line starts with "#cloud-config".
 // It returns true if the first line starts with "#cloud-config", otherwise it returns false.
 func IsCloudConfig(ci string) bool {
@@ -48,6 +57,10 @@ func ParseCloudConfig(ci string) (*CloudConfig, error) {
 	if err != nil {
 		return nil, err
 	}
+	err = validate.Struct(cc)
+	if err != nil {
+		return nil, err
+	}
 	return &cc, nil
 }
 
@@ -68,16 +81,24 @@ func WriteFile(log *base.LogObject, file WritableFile, rootPath string) error {
 
 	var contentBytes []byte
 	switch file.Encoding {
-	case "b64":
-		// decode base64 content
+	case "b64", "base64":
 		contentBytes, err = base64.StdEncoding.DecodeString(file.Content)
-		if err != nil {
-			return err
-		}
-	case "plain":
+
+	case "plain", "":
 		contentBytes = []byte(file.Content)
+
+	case "gz+base64", "gzip+base64", "gz+b64", "gzip+b64":
+		contentBytes, err = decodeGzipBase64Content(file.Content)
+
+	case "gzip", "gz":
+		contentBytes, err = decodeGzipContent(file.Content)
+
 	default:
-		return errors.New("unsupported encoding type. Only base64 and plain are supported")
+		return errors.New("unsupported encoding type. Only base64, plain and gzip are supported")
+	}
+
+	if err != nil {
+		return err
 	}
 
 	// check if the parent directory exists
@@ -105,3 +126,27 @@ func WriteFile(log *base.LogObject, file WritableFile, rootPath string) error {
 
 	return nil
 }
+
+func decodeGzipContent(content string) ([]byte, error) {
+	gzipReader, err := gzip.NewReader(strings.NewReader(content))
+	if err != nil {
+		return nil, err
+	}
+	defer gzipReader.Close()
+	contentBytes, err := io.ReadAll(io.LimitReader(gzipReader, MaxDecompressedContentSize+1))
+	if err != nil {
+		return nil, err
+	}
+	if len(contentBytes) > MaxDecompressedContentSize {
+		return nil, errors.New("maximum decompressed content size exceeded")
+	}
+	return contentBytes, nil
+}
+
+func decodeGzipBase64Content(content string) ([]byte, error) {
+	gzipBytes, err := base64.StdEncoding.DecodeString(content)
+	if err != nil {
+		return nil, err
+	}
+	return decodeGzipContent(string(gzipBytes))
+}