add os codename fetcher/helper method (#433)

Signed-off-by: Alex Goodman <[email protected]>

.github/workflows/update-generated-code.yaml

@@ -0,0 +1,56 @@
+name: PR to update generated code
+on:
+  schedule:
+    - cron: "0 1 * * 1" # every monday at 1 AM
+
+  workflow_dispatch:
+
+  # TODO: delete me
+  push:
+
+permissions:
+  contents: read
+
+env:
+  SLACK_NOTIFICATIONS: true
+
+jobs:
+  run-code-gen:
+    name: "Run code generation"
+    runs-on: ubuntu-latest
+    if: github.repository == 'anchore/grype-db' # only run for main repo
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
+
+      - name: Bootstrap environment
+        uses: ./.github/actions/bootstrap
+
+      - run: |
+          make generate-processor-code
+
+      - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a #v2.1.0
+        id: generate-token
+        with:
+          app_id: ${{ secrets.TOKEN_APP_ID }}
+          private_key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }}
+
+      - uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
+        with:
+          signoff: true
+          delete-branch: true
+          branch: auto/latest-code-gen
+          labels: dependencies
+          commit-message: "chore(deps): update generated code"
+          title: "chore(deps): update generated code"
+          body: |
+            Update generated code from external sources
+          token: ${{ steps.generate-token.outputs.token }}
+
+      - uses: 8398a7/action-slack@28ba43ae48961b90635b50953d216767a6bea486 #v3.16.2
+        with:
+          status: ${{ job.status }}
+          fields: workflow,eventName,job
+          text: Grype-DB generated code update failed
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
+        if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}

Makefile

@@ -256,6 +256,13 @@ download-all-provider-cache:
 	@bash -c "oras pull $(GRYPE_DB_DATA_IMAGE_NAME):$(date) && $(GRYPE_DB) cache restore --path $(DB_ARCHIVE) || (echo 'no data cache found for today' && exit 1)"
 
 
+## Code and data generation targets #################################
+
+.PHONY: generate-processor-code
+generate-processor-code:
+	go generate ./pkg/process
+	make format
+
 ## Build-related targets #################################
 
 .PHONY: build

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a
 	github.com/anchore/grype v0.84.0
 	github.com/anchore/syft v1.16.0
+	github.com/dave/jennifer v1.7.1
 	github.com/dustin/go-humanize v1.0.1
 	github.com/glebarez/sqlite v1.11.0
 	github.com/go-test/deep v1.1.1

go.sum

@@ -367,6 +367,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo=
 github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo=
+github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=

pkg/process/generate.go

@@ -0,0 +1,3 @@
+package process
+
+//go:generate go run ./internal/codename/generate/main.go

pkg/process/internal/codename/codename.go

@@ -0,0 +1,24 @@
+package codename
+
+import "strings"
+
+func LookupOS(osName, majorVersion, minorVersion string) string {
+	majorVersion = strings.TrimLeft(majorVersion, "0")
+	if minorVersion != "0" {
+		minorVersion = strings.TrimLeft(minorVersion, "0")
+	}
+
+	// try to find the most specific match (major and minor version)
+	if versions, ok := normalizedOSCodenames[osName]; ok {
+		if minorMap, ok := versions[majorVersion]; ok {
+			if codename, ok := minorMap[minorVersion]; ok {
+				return codename
+			}
+			// fall back to the least specific match (only major version, allowing for any minor version explicitly)
+			if codename, ok := minorMap["*"]; ok {
+				return codename
+			}
+		}
+	}
+	return ""
+}

pkg/process/internal/codename/codename_test.go

@@ -0,0 +1,40 @@
+package codename
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLookupOSCodename(t *testing.T) {
+	tests := []struct {
+		Name             string
+		OSName           string
+		MajorVersion     string
+		MinorVersion     string
+		ExpectedCodename string
+	}{
+		{Name: "Ubuntu 20.04 exact", OSName: "ubuntu", MajorVersion: "20", MinorVersion: "04", ExpectedCodename: "focal"},
+		{Name: "Ubuntu 20.4 exact", OSName: "ubuntu", MajorVersion: "20", MinorVersion: "4", ExpectedCodename: "focal"},
+		{Name: "Ubuntu 0 (non existent) minor", OSName: "ubuntu", MajorVersion: "20", MinorVersion: "0", ExpectedCodename: ""},
+		{Name: "Ubuntu empty minor", OSName: "ubuntu", MajorVersion: "10", MinorVersion: "", ExpectedCodename: ""},
+		{Name: "Debian empty minor", OSName: "debian", MajorVersion: "10", MinorVersion: "", ExpectedCodename: "buster"},
+		{Name: "Ubuntu leading zeros in major", OSName: "ubuntu", MajorVersion: "020", MinorVersion: "04", ExpectedCodename: "focal"},
+		{Name: "Debian leading zeros in major", OSName: "debian", MajorVersion: "010", MinorVersion: "", ExpectedCodename: "buster"},
+		{Name: "Debian bad minor", OSName: "debian", MajorVersion: "11", MinorVersion: "99", ExpectedCodename: "bullseye"},
+		{Name: "Ubuntu bad minor", OSName: "ubuntu", MajorVersion: "22", MinorVersion: "99", ExpectedCodename: ""},
+		{Name: "Ubuntu 6.10 exact (legacy)", OSName: "ubuntu", MajorVersion: "6", MinorVersion: "10", ExpectedCodename: "edgy"},
+		{Name: "Ubuntu 6.6 exact (legacy)", OSName: "ubuntu", MajorVersion: "6", MinorVersion: "6", ExpectedCodename: "dapper"},
+		{Name: "Debian 2.1 exact", OSName: "debian", MajorVersion: "2", MinorVersion: "1", ExpectedCodename: "slink"},
+		{Name: "Debian 2 fallback to *", OSName: "debian", MajorVersion: "2", MinorVersion: "0", ExpectedCodename: "hamm"},
+		{Name: "Invalid OS name", OSName: "nonexistentOS", MajorVersion: "10", MinorVersion: "04", ExpectedCodename: ""},
+		{Name: "Invalid major version", OSName: "ubuntu", MajorVersion: "99", MinorVersion: "04", ExpectedCodename: ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			actualCodename := LookupOS(tt.OSName, tt.MajorVersion, tt.MinorVersion)
+			assert.Equal(t, tt.ExpectedCodename, actualCodename)
+		})
+	}
+}

pkg/process/internal/codename/codenames_generated.go

@@ -0,0 +1,114 @@
+// DO NOT EDIT: generated by pkg/process/v6/transformers/internal/codename/main.go
+
+package codename
+
+var normalizedOSCodenames = map[string]map[string]map[string]string{
+	"debian": {
+		"1": {
+			"1": "buzz",
+			"2": "rex",
+			"3": "bo",
+		},
+		"10": {"*": "buster"},
+		"11": {"*": "bullseye"},
+		"12": {"*": "bookworm"},
+		"2": {
+			"0": "hamm",
+			"1": "slink",
+			"2": "potato",
+		},
+		"3": {
+			"0": "woody",
+			"1": "sarge",
+		},
+		"4": {"*": "etch"},
+		"5": {"*": "lenny"},
+		"6": {"*": "squeeze"},
+		"7": {"*": "wheezy"},
+		"8": {"*": "jessie"},
+		"9": {"*": "stretch"},
+	},
+	"ubuntu": {
+		"10": {
+			"10": "maverick",
+			"4":  "lucid",
+		},
+		"11": {
+			"10": "oneiric",
+			"4":  "natty",
+		},
+		"12": {
+			"10": "quantal",
+			"4":  "precise",
+		},
+		"13": {
+			"10": "saucy",
+			"4":  "raring",
+		},
+		"14": {
+			"10": "utopic",
+			"4":  "trusty",
+		},
+		"15": {
+			"10": "wily",
+			"4":  "vivid",
+		},
+		"16": {
+			"10": "yakkety",
+			"4":  "xenial",
+		},
+		"17": {
+			"10": "artful",
+			"4":  "zesty",
+		},
+		"18": {
+			"10": "cosmic",
+			"4":  "bionic",
+		},
+		"19": {
+			"10": "eoan",
+			"4":  "disco",
+		},
+		"20": {
+			"10": "groovy",
+			"4":  "focal",
+		},
+		"21": {
+			"10": "impish",
+			"4":  "hirsute",
+		},
+		"22": {
+			"10": "kinetic",
+			"4":  "jammy",
+		},
+		"23": {
+			"10": "mantic",
+			"4":  "lunar",
+		},
+		"24": {
+			"10": "oracular",
+			"4":  "noble",
+		},
+		"4": {"10": "warty"},
+		"5": {
+			"10": "breezy",
+			"4":  "hoary",
+		},
+		"6": {
+			"10": "edgy",
+			"6":  "dapper",
+		},
+		"7": {
+			"10": "gutsy",
+			"4":  "feisty",
+		},
+		"8": {
+			"10": "intrepid",
+			"4":  "hardy",
+		},
+		"9": {
+			"10": "karmic",
+			"4":  "jaunty",
+		},
+	},
+}

pkg/process/internal/codename/generate/main.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/dave/jennifer/jen"
+)
+
+const (
+	outputPackage = "pkg/process/internal/codename"
+	outputPath    = "internal/codename/codenames_generated.go" // relative to where go generate is called
+)
+
+type Version struct {
+	Cycle    string `json:"cycle"`
+	Codename string `json:"codename"`
+}
+
+func main() {
+	osCodenames := make(map[string]map[string]map[string]string)
+
+	fmt.Println("Fetching and parsing data for operating system codenames")
+
+	fmt.Println("ubuntu:")
+	osCodenames["ubuntu"] = fetchAndParse("https://endoflife.date/api/ubuntu.json", ubuntuHandler)
+
+	fmt.Println("debian:")
+	osCodenames["debian"] = fetchAndParse("https://endoflife.date/api/debian.json", lowercaseHandler)
+
+	fmt.Printf("Generating code for %d operating system codenames\n", len(osCodenames))
+
+	f := jen.NewFile("codename")
+	f.HeaderComment("DO NOT EDIT: generated by pkg/process/v6/transformers/internal/codename/main.go")
+	f.ImportName(outputPackage, "pkg")
+	f.Var().Id("normalizedOSCodenames").Op("=").Map(jen.String()).Map(jen.String()).Map(jen.String()).String().Values(jen.DictFunc(func(d jen.Dict) {
+		for osName, versions := range osCodenames {
+			majorMap := jen.Dict{}
+			for major, minors := range versions {
+				minorMap := jen.Dict{}
+				for minor, codename := range minors {
+					minorMap[jen.Lit(minor)] = jen.Lit(codename)
+				}
+				majorMap[jen.Lit(major)] = jen.Values(minorMap)
+			}
+			d[jen.Lit(osName)] = jen.Values(majorMap)
+		}
+	}))
+
+	rendered := fmt.Sprintf("%#v", f)
+
+	file, err := os.OpenFile(outputPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		panic(fmt.Errorf("unable to open file: %w", err))
+	}
+	defer file.Close()
+
+	if _, err := file.WriteString(rendered); err != nil {
+		panic(fmt.Errorf("unable to write file: %w", err))
+	}
+
+	fmt.Printf("Code generation completed and written to %s\n", outputPath)
+}
+
+// fetchAndParse fetches the JSON data from a URL, parses it, and organizes it into a map.
+func fetchAndParse(url string, handler func(string) string) map[string]map[string]string {
+	resp, err := http.Get(url) //nolint:gosec
+	if err != nil {
+		panic(fmt.Errorf("error fetching data from %s: %w", url, err))
+	}
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		panic(fmt.Errorf("error reading response: %w", err))
+	}
+
+	var versions []Version
+	if err := json.Unmarshal(data, &versions); err != nil {
+		panic(fmt.Errorf("error parsing JSON: %w", err))
+	}
+
+	parsedData := make(map[string]map[string]string)
+	for _, version := range versions {
+		major, minor := parseVersion(version.Cycle)
+		if parsedData[major] == nil {
+			parsedData[major] = make(map[string]string)
+		}
+		codename := handler(version.Codename)
+		fmt.Printf("  adding %s.%s --> %s\n", major, minor, codename)
+		parsedData[major][minor] = codename
+	}
+
+	return parsedData
+}
+
+func lowercaseHandler(codename string) string {
+	return strings.ToLower(codename)
+}
+
+func ubuntuHandler(codename string) string {
+	return strings.ToLower(strings.Split(codename, " ")[0])
+}
+
+// parseVersion splits a version string like "20.04" into major "20" and minor "04".
+func parseVersion(version string) (string, string) {
+	parts := strings.Split(version, ".")
+	major := strings.TrimLeft(parts[0], "0")
+	minor := "*"
+	if len(parts) > 1 {
+		if parts[1] == "0" {
+			minor = parts[1]
+		} else {
+			minor = strings.TrimLeft(parts[1], "0")
+		}
+	}
+	return major, minor
+}