patch envelopes clear inline objects every time config recreates

We don't have a mechanism to update EdgeDevConfig by part,
we always send whole config to EVE. It is easier to delete
previous patch envelopes and create them from scratch, rather
than load each file, compare them with new and rewrite if it's
changed (since we are talking about files less than 1MB) or rewriting
them.

This, of course create a possible state where app instance tries
to access file which is not there anymore and new config has not
been propagated yet to zedrouter, but in that case metadata server
will return error and app instance can either retry or request
description.json once again

This commit also addresses some spelling issues and yetus fixes

Signed-off-by: Pavel Abramov <[email protected]>

pkg/pillar/cmd/zedagent/parseconfig_test.go

@@ -1322,14 +1322,14 @@ func TestParsePatchEnvelope(t *testing.T) {
 	shaBytes := sha256.Sum256([]byte(fileData))
 	g.Expect(pes.Get(appU1)).To(BeEquivalentTo([]types.PatchEnvelopeInfo{
 		{
-			PatchId:     patchID,
+			PatchID:     patchID,
 			AllowedApps: []string{appU1, appU2},
 			BinaryBlobs: []types.BinaryBlobCompleted{
 				{
 					FileName:     inlineFileName,
 					FileSha:      hex.EncodeToString(shaBytes[:]),
 					FileMetadata: fileMetadata,
-					Url:          filepath.Join(persistCacheFolder, inlineFileName),
+					URL:          filepath.Join(persistCacheFolder, inlineFileName),
 				},
 			},
 		},

pkg/pillar/cmd/zedagent/parsepatchenvelopes.go

@@ -6,12 +6,15 @@ package zedagent
 import (
 	"encoding/hex"
 	"fmt"
+	"os"
 
 	"crypto/sha256"
 
 	zconfig "github.com/lf-edge/eve-api/go/config"
 	"github.com/lf-edge/eve/pkg/pillar/persistcache"
 	"github.com/lf-edge/eve/pkg/pillar/types"
+
+	"github.com/lf-edge/eve/pkg/pillar/utils/generics"
 )
 
 func parsePatchEnvelopes(ctx *getconfigContext, config *zconfig.EdgeDevConfig) {
@@ -22,12 +25,28 @@ func parsePatchEnvelopesImpl(ctx *getconfigContext, config *zconfig.EdgeDevConfi
 	persistCacheFilepath string) {
 	log.Tracef("Parsing patchEnvelope from configuration")
 
+	// Remove previously created patch envelopes
+	// so that we will not have stale objects
+	if err := os.RemoveAll(persistCacheFilepath); err != nil {
+		log.Errorf("Failed to delete persistCacheFilepath %v", err)
+		return
+	}
+
+	// Store list of binary blobs which were created before
+	pc, err := persistcache.New(persistCacheFilepath)
+	if err != nil {
+		log.Errorf("Failed to load persistCache %v", err)
+		return
+	}
+	blobsBefore := pc.Objects()
+
+	var blobsAfter []string
 	patchEnvelopes := config.GetPatchEnvelopes()
 	result := types.PatchEnvelopes{}
 	for _, pe := range patchEnvelopes {
 		peInfo := types.PatchEnvelopeInfo{
 			AllowedApps: pe.GetAppInstIdsAllowed(),
-			PatchId:     pe.GetUuid(),
+			PatchID:     pe.GetUuid(),
 		}
 		for _, a := range pe.GetArtifacts() {
 			err := addBinaryBlobToPatchEnvelope(&peInfo, a, persistCacheFilepath)
@@ -38,9 +57,19 @@ func parsePatchEnvelopesImpl(ctx *getconfigContext, config *zconfig.EdgeDevConfi
 		}
 
 		result.Envelopes = append(result.Envelopes, peInfo)
+
+		for _, inlineBlob := range peInfo.BinaryBlobs {
+			blobsAfter = append(blobsAfter, inlineBlob.FileName)
+		}
 	}
 
 	publishPatchEnvelopes(ctx, result)
+
+	// Provide zedrouter with newest version for description.json and then delete files
+	blobsToDelete, _ := generics.DiffSets(blobsBefore, blobsAfter)
+	for _, blob := range blobsToDelete {
+		pc.Delete(blob)
+	}
 }
 
 func publishPatchEnvelopes(ctx *getconfigContext, patchEnvelopes types.PatchEnvelopes) {
@@ -107,7 +136,7 @@ func cacheInlineBase64Artifact(artifact *zconfig.InlineOpaqueBase64Data, persist
 		FileName:     artifact.GetFileNameToUse(),
 		FileSha:      hex.EncodeToString(shaBytes[:]),
 		FileMetadata: metadata,
-		Url:          url,
+		URL:          url,
 	}, nil
 }
 
@@ -119,6 +148,6 @@ func getBinaryBlobVolumeRef(artifact *zconfig.ExternalOpaqueBinaryBlob) (*types.
 		ImageName:    artifact.GetImageName(),
 		FileName:     artifact.GetFileNameToUse(),
 		FileMetadata: artifact.GetBlobMetaData(),
-		ImageId:      artifact.GetImageId(),
+		ImageID:      artifact.GetImageId(),
 	}, nil
 }

pkg/pillar/cmd/zedrouter/metadata_handlers.go

@@ -93,7 +93,11 @@ type AppCustomBlobsHandler struct {
 	zedrouter *zedrouter
 }
 
-const PatchEnvelopesContextKeyType = "patchEnvelopes"
+type middlewareKeys int
+
+const (
+	patchEnvelopesContextKey middlewareKeys = iota
+)
 
 // ServeHTTP for networkHandler provides a json return
 func (hdl networkHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -577,9 +581,9 @@ func (hdl AppCustomBlobsHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 func HandlePatchDescription(z *zedrouter) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// WithPatchEnvelopesByIP middleware returns envelopes which are more than 0
-		envelopes := r.Context().Value("patchEnvelopes").([]types.PatchEnvelopeInfo)
+		envelopes := r.Context().Value(patchEnvelopesContextKey).([]types.PatchEnvelopeInfo)
 
-		b, err := types.PatchEnvelopesJsonForAppInstance(envelopes)
+		b, err := types.PatchEnvelopesJSONForAppInstance(envelopes)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusUnprocessableEntity)
 			return
@@ -603,14 +607,14 @@ func sendError(w http.ResponseWriter, code int, msg string) {
 func HandlePatchDownload(z *zedrouter) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// WithPatchEnvelopesByIP middleware returns envelopes which are more than 0
-		envelopes := r.Context().Value(PatchEnvelopesContextKeyType).([]types.PatchEnvelopeInfo)
+		envelopes := r.Context().Value(patchEnvelopesContextKey).([]types.PatchEnvelopeInfo)
 
 		patchID := chi.URLParam(r, "patch")
 		if patchID == "" {
 			sendError(w, http.StatusNoContent, "patch in route is missing")
 			return
 		}
-		e := types.FindPatchEnvelopeById(envelopes, patchID)
+		e := types.FindPatchEnvelopeByID(envelopes, patchID)
 		if e != nil {
 			path, err := os.MkdirTemp("", "patchEnvelopeZip")
 			if err != nil {
@@ -646,7 +650,7 @@ func HandlePatchDownload(z *zedrouter) func(http.ResponseWriter, *http.Request)
 func HandlePatchFileDownload(z *zedrouter) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// WithPatchEnvelopesByIP middleware returns envelopes which are more than 0
-		envelopes := r.Context().Value(PatchEnvelopesContextKeyType).([]types.PatchEnvelopeInfo)
+		envelopes := r.Context().Value(patchEnvelopesContextKey).([]types.PatchEnvelopeInfo)
 
 		patchID := chi.URLParam(r, "patch")
 		if patchID == "" {
@@ -659,10 +663,10 @@ func HandlePatchFileDownload(z *zedrouter) func(http.ResponseWriter, *http.Reque
 			return
 		}
 
-		e := types.FindPatchEnvelopeById(envelopes, patchID)
+		e := types.FindPatchEnvelopeByID(envelopes, patchID)
 		if e != nil {
 			if idx := types.CompletedBinaryBlobIdxByName(e.BinaryBlobs, fileName); idx != -1 {
-				http.ServeFile(w, r, e.BinaryBlobs[idx].Url)
+				http.ServeFile(w, r, e.BinaryBlobs[idx].URL)
 				return
 			} else {
 				sendError(w, http.StatusNotFound, "file is not found")
@@ -677,7 +681,7 @@ func HandlePatchFileDownload(z *zedrouter) func(http.ResponseWriter, *http.Reque
 // WithPatchEnvelopesByIP is a middleware for Patch Envelopes which adds
 // to a context patchEnvelope variable containing available patch envelopes
 // for given IP address (it gets resolved to app instance UUID)
-// in case there is no patch envelopes avaiable it returns StatusNoContent
+// in case there is no patch envelopes available it returns StatusNoContent
 func WithPatchEnvelopesByIP(z *zedrouter) func(http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -705,7 +709,7 @@ func WithPatchEnvelopesByIP(z *zedrouter) func(http.Handler) http.Handler {
 				sendError(w, http.StatusOK, fmt.Sprintf("No envelopes for %s", appUUID.String()))
 			}
 
-			ctx := context.WithValue(r.Context(), PatchEnvelopesContextKeyType, envelopes)
+			ctx := context.WithValue(r.Context(), patchEnvelopesContextKey, envelopes)
 
 			r = r.WithContext(ctx)
 			next.ServeHTTP(w, r)

pkg/pillar/cmd/zedrouter/zedrouter.go

@@ -170,7 +170,7 @@ type zedrouter struct {
 	// volumemgr, therefore we need to be subscribed
 	// to volume status to know filepath and download status
 	// patchEnvelopeInfo is list of patchEnvelopes which
-	// is coming from EdgeDevConfig containing infromation
+	// is coming from EdgeDevConfig containing information
 	// about inline patch envelopes and volume uuid as reference
 	// for external patch envelopes
 	subPatchEnvelopeInfo pubsub.Subscription
@@ -810,7 +810,7 @@ func (z *zedrouter) initSubscriptions() (err error) {
 		return err
 	}
 
-	// Infromation about volumes reffered in external patch envelopes
+	// Information about volumes referred in external patch envelopes
 	z.subVolumeStatus, err = z.pubSub.NewSubscription(pubsub.SubscriptionOptions{
 		AgentName:   "volumemgr",
 		MyAgentName: agentName,