incus-simplestreams: Add prune command

Signed-off-by: Piotr Resztak <[email protected]>

cmd/incus-simplestreams/main.go

@@ -62,6 +62,9 @@ func main() {
 	verifyCmd := cmdVerify{global: &globalCmd}
 	app.AddCommand(verifyCmd.Command())
 
+	pruneCmd := cmdPrune{global: &globalCmd}
+	app.AddCommand(pruneCmd.Command())
+
 	// Run the main command and handle errors.
 	err := app.Execute()
 	if err != nil {

cmd/incus-simplestreams/main_prune.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"sort"
+
+	"github.com/spf13/cobra"
+
+	cli "github.com/lxc/incus/v6/internal/cmd"
+	"github.com/lxc/incus/v6/shared/simplestreams"
+)
+
+type cmdPrune struct {
+	global *cmdGlobal
+
+	flagDryRun        bool
+	flagPruneVersions bool
+	flagVerbose       bool
+}
+
+// Command generates the command definition.
+func (c *cmdPrune) Command() *cobra.Command {
+	cmd := &cobra.Command{}
+	cmd.Use = "prune"
+	cmd.Short = "Clean up obsolete files and data"
+	cmd.Long = cli.FormatSection("Description",
+		`Cleans up obsolote tarball files and removes outdated versions of a product
+
+The prune command scans the project directory for tarball files that do not have corresponding references
+in the 'images.json' file. Any tarball file that is not listed in images.json is considered orphaned
+and will be deleted.
+The prune command identifies and deletes older versions of product data,
+retaining only the most recent existing version.`)
+
+	cmd.RunE = c.Run
+	cmd.Flags().BoolVarP(&c.flagDryRun, "dry-run", "d", false, "Preview changes without executing actual operations")
+	cmd.Flags().BoolVarP(&c.flagPruneVersions, "prune-versions", "p", true, "Prune old versions of a product")
+	cmd.Flags().BoolVarP(&c.flagVerbose, "verbose", "v", false, "Show all information messages")
+
+	return cmd
+}
+
+// Run runs the actual command logic.
+func (c *cmdPrune) Run(cmd *cobra.Command, args []string) error {
+	// Quick checks.
+	exit, err := c.global.CheckArgs(cmd, args, 0, 0)
+	if exit {
+		return err
+	}
+
+	if c.flagDryRun {
+		c.flagVerbose = true
+	}
+
+	products, err := c.productsWithPrunePreviousVersions()
+	if err != nil {
+		return err
+	}
+
+	err = c.pruneImages(products)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *cmdPrune) pruneImages(products *simplestreams.Products) error {
+	filesToPreserve := []string{}
+	for _, product := range products.Products {
+		for _, version := range product.Versions {
+			for _, item := range version.Items {
+				filesToPreserve = append(filesToPreserve, item.Path)
+			}
+		}
+	}
+
+	err := filepath.WalkDir("./images", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Omit the path if it is a directory or if it exists in the images.json file.
+		if d.IsDir() || slices.Contains(filesToPreserve, path) {
+			return nil
+		}
+
+		if c.flagVerbose {
+			fmt.Printf("Removing: %s\n", path)
+		}
+
+		if !c.flagDryRun {
+			e := os.Remove(path)
+			if e != nil {
+				return e
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *cmdPrune) productsWithPrunePreviousVersions() (*simplestreams.Products, error) {
+	body, err := os.ReadFile("streams/v1/images.json")
+	if err != nil {
+		return nil, err
+	}
+
+	products := simplestreams.Products{}
+	err = json.Unmarshal(body, &products)
+	if err != nil {
+		return nil, err
+	}
+
+	if !c.flagPruneVersions {
+		return &products, nil
+	}
+
+	for kProduct, product := range products.Products {
+		// If there are fewer than two versions, then there is nothing to do here.
+		if len(product.Versions) < 2 {
+			continue
+		}
+
+		versionNames := []string{}
+		for kVersion := range product.Versions {
+			versionNames = append(versionNames, kVersion)
+		}
+
+		sort.Strings(versionNames)
+
+		recentVersion := versionNames[len(versionNames)-1]
+		if c.flagVerbose {
+			fmt.Printf("Leaving version %s for product %s\n", recentVersion, kProduct)
+		}
+
+		// Keep only recent 'version'.
+		p := products.Products[kProduct]
+		p.Versions = map[string]simplestreams.ProductVersion{recentVersion: p.Versions[recentVersion]}
+		products.Products[kProduct] = p
+	}
+
+	if !c.flagDryRun {
+		// Write back the images file.
+		body, err = json.Marshal(&products)
+		if err != nil {
+			return nil, err
+		}
+
+		err = os.WriteFile("streams/v1/images.json", body, 0644)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &products, nil
+}