From f9d24403126a2adef5385b7f28a739ea17e55f7f Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 21 Nov 2024 13:37:07 -0500 Subject: [PATCH] add os codename fetcher/helper method (#433) Signed-off-by: Alex Goodman --- .github/workflows/update-generated-code.yaml | 56 ++++++++ Makefile | 7 + go.mod | 1 + go.sum | 2 + pkg/process/generate.go | 3 + pkg/process/internal/codename/codename.go | 24 ++++ .../internal/codename/codename_test.go | 40 ++++++ .../internal/codename/codenames_generated.go | 114 ++++++++++++++++ .../internal/codename/generate/main.go | 122 ++++++++++++++++++ 9 files changed, 369 insertions(+) create mode 100644 .github/workflows/update-generated-code.yaml create mode 100644 pkg/process/generate.go create mode 100644 pkg/process/internal/codename/codename.go create mode 100644 pkg/process/internal/codename/codename_test.go create mode 100644 pkg/process/internal/codename/codenames_generated.go create mode 100644 pkg/process/internal/codename/generate/main.go diff --git a/.github/workflows/update-generated-code.yaml b/.github/workflows/update-generated-code.yaml new file mode 100644 index 00000000..42c9f005 --- /dev/null +++ b/.github/workflows/update-generated-code.yaml @@ -0,0 +1,56 @@ +name: PR to update generated code +on: + schedule: + - cron: "0 1 * * 1" # every monday at 1 AM + + workflow_dispatch: + + # TODO: delete me + push: + +permissions: + contents: read + +env: + SLACK_NOTIFICATIONS: true + +jobs: + run-code-gen: + name: "Run code generation" + runs-on: ubuntu-latest + if: github.repository == 'anchore/grype-db' # only run for main repo + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 + + - name: Bootstrap environment + uses: ./.github/actions/bootstrap + + - run: | + make generate-processor-code + + - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a #v2.1.0 + id: generate-token + with: + app_id: ${{ secrets.TOKEN_APP_ID }} + private_key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }} + + - uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5 + with: + signoff: true + delete-branch: true + branch: auto/latest-code-gen + labels: dependencies + commit-message: "chore(deps): update generated code" + title: "chore(deps): update generated code" + body: | + Update generated code from external sources + token: ${{ steps.generate-token.outputs.token }} + + - uses: 8398a7/action-slack@28ba43ae48961b90635b50953d216767a6bea486 #v3.16.2 + with: + status: ${{ job.status }} + fields: workflow,eventName,job + text: Grype-DB generated code update failed + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} + if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }} \ No newline at end of file diff --git a/Makefile b/Makefile index 29bb1e4d..2efa24b9 100644 --- a/Makefile +++ b/Makefile @@ -256,6 +256,13 @@ download-all-provider-cache: @bash -c "oras pull $(GRYPE_DB_DATA_IMAGE_NAME):$(date) && $(GRYPE_DB) cache restore --path $(DB_ARCHIVE) || (echo 'no data cache found for today' && exit 1)" +## Code and data generation targets ################################# + +.PHONY: generate-processor-code +generate-processor-code: + go generate ./pkg/process + make format + ## Build-related targets ################################# .PHONY: build diff --git a/go.mod b/go.mod index f2a9190d..f2040722 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a github.com/anchore/grype v0.84.0 github.com/anchore/syft v1.16.0 + github.com/dave/jennifer v1.7.1 github.com/dustin/go-humanize v1.0.1 github.com/glebarez/sqlite v1.11.0 github.com/go-test/deep v1.1.1 diff --git a/go.sum b/go.sum index 40d65181..75fcddf7 100644 --- a/go.sum +++ b/go.sum @@ -367,6 +367,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= +github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= diff --git a/pkg/process/generate.go b/pkg/process/generate.go new file mode 100644 index 00000000..923c6fc3 --- /dev/null +++ b/pkg/process/generate.go @@ -0,0 +1,3 @@ +package process + +//go:generate go run ./internal/codename/generate/main.go diff --git a/pkg/process/internal/codename/codename.go b/pkg/process/internal/codename/codename.go new file mode 100644 index 00000000..e874cd64 --- /dev/null +++ b/pkg/process/internal/codename/codename.go @@ -0,0 +1,24 @@ +package codename + +import "strings" + +func LookupOS(osName, majorVersion, minorVersion string) string { + majorVersion = strings.TrimLeft(majorVersion, "0") + if minorVersion != "0" { + minorVersion = strings.TrimLeft(minorVersion, "0") + } + + // try to find the most specific match (major and minor version) + if versions, ok := normalizedOSCodenames[osName]; ok { + if minorMap, ok := versions[majorVersion]; ok { + if codename, ok := minorMap[minorVersion]; ok { + return codename + } + // fall back to the least specific match (only major version, allowing for any minor version explicitly) + if codename, ok := minorMap["*"]; ok { + return codename + } + } + } + return "" +} diff --git a/pkg/process/internal/codename/codename_test.go b/pkg/process/internal/codename/codename_test.go new file mode 100644 index 00000000..f43d3e34 --- /dev/null +++ b/pkg/process/internal/codename/codename_test.go @@ -0,0 +1,40 @@ +package codename + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLookupOSCodename(t *testing.T) { + tests := []struct { + Name string + OSName string + MajorVersion string + MinorVersion string + ExpectedCodename string + }{ + {Name: "Ubuntu 20.04 exact", OSName: "ubuntu", MajorVersion: "20", MinorVersion: "04", ExpectedCodename: "focal"}, + {Name: "Ubuntu 20.4 exact", OSName: "ubuntu", MajorVersion: "20", MinorVersion: "4", ExpectedCodename: "focal"}, + {Name: "Ubuntu 0 (non existent) minor", OSName: "ubuntu", MajorVersion: "20", MinorVersion: "0", ExpectedCodename: ""}, + {Name: "Ubuntu empty minor", OSName: "ubuntu", MajorVersion: "10", MinorVersion: "", ExpectedCodename: ""}, + {Name: "Debian empty minor", OSName: "debian", MajorVersion: "10", MinorVersion: "", ExpectedCodename: "buster"}, + {Name: "Ubuntu leading zeros in major", OSName: "ubuntu", MajorVersion: "020", MinorVersion: "04", ExpectedCodename: "focal"}, + {Name: "Debian leading zeros in major", OSName: "debian", MajorVersion: "010", MinorVersion: "", ExpectedCodename: "buster"}, + {Name: "Debian bad minor", OSName: "debian", MajorVersion: "11", MinorVersion: "99", ExpectedCodename: "bullseye"}, + {Name: "Ubuntu bad minor", OSName: "ubuntu", MajorVersion: "22", MinorVersion: "99", ExpectedCodename: ""}, + {Name: "Ubuntu 6.10 exact (legacy)", OSName: "ubuntu", MajorVersion: "6", MinorVersion: "10", ExpectedCodename: "edgy"}, + {Name: "Ubuntu 6.6 exact (legacy)", OSName: "ubuntu", MajorVersion: "6", MinorVersion: "6", ExpectedCodename: "dapper"}, + {Name: "Debian 2.1 exact", OSName: "debian", MajorVersion: "2", MinorVersion: "1", ExpectedCodename: "slink"}, + {Name: "Debian 2 fallback to *", OSName: "debian", MajorVersion: "2", MinorVersion: "0", ExpectedCodename: "hamm"}, + {Name: "Invalid OS name", OSName: "nonexistentOS", MajorVersion: "10", MinorVersion: "04", ExpectedCodename: ""}, + {Name: "Invalid major version", OSName: "ubuntu", MajorVersion: "99", MinorVersion: "04", ExpectedCodename: ""}, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + actualCodename := LookupOS(tt.OSName, tt.MajorVersion, tt.MinorVersion) + assert.Equal(t, tt.ExpectedCodename, actualCodename) + }) + } +} diff --git a/pkg/process/internal/codename/codenames_generated.go b/pkg/process/internal/codename/codenames_generated.go new file mode 100644 index 00000000..6dfc99c9 --- /dev/null +++ b/pkg/process/internal/codename/codenames_generated.go @@ -0,0 +1,114 @@ +// DO NOT EDIT: generated by pkg/process/v6/transformers/internal/codename/main.go + +package codename + +var normalizedOSCodenames = map[string]map[string]map[string]string{ + "debian": { + "1": { + "1": "buzz", + "2": "rex", + "3": "bo", + }, + "10": {"*": "buster"}, + "11": {"*": "bullseye"}, + "12": {"*": "bookworm"}, + "2": { + "0": "hamm", + "1": "slink", + "2": "potato", + }, + "3": { + "0": "woody", + "1": "sarge", + }, + "4": {"*": "etch"}, + "5": {"*": "lenny"}, + "6": {"*": "squeeze"}, + "7": {"*": "wheezy"}, + "8": {"*": "jessie"}, + "9": {"*": "stretch"}, + }, + "ubuntu": { + "10": { + "10": "maverick", + "4": "lucid", + }, + "11": { + "10": "oneiric", + "4": "natty", + }, + "12": { + "10": "quantal", + "4": "precise", + }, + "13": { + "10": "saucy", + "4": "raring", + }, + "14": { + "10": "utopic", + "4": "trusty", + }, + "15": { + "10": "wily", + "4": "vivid", + }, + "16": { + "10": "yakkety", + "4": "xenial", + }, + "17": { + "10": "artful", + "4": "zesty", + }, + "18": { + "10": "cosmic", + "4": "bionic", + }, + "19": { + "10": "eoan", + "4": "disco", + }, + "20": { + "10": "groovy", + "4": "focal", + }, + "21": { + "10": "impish", + "4": "hirsute", + }, + "22": { + "10": "kinetic", + "4": "jammy", + }, + "23": { + "10": "mantic", + "4": "lunar", + }, + "24": { + "10": "oracular", + "4": "noble", + }, + "4": {"10": "warty"}, + "5": { + "10": "breezy", + "4": "hoary", + }, + "6": { + "10": "edgy", + "6": "dapper", + }, + "7": { + "10": "gutsy", + "4": "feisty", + }, + "8": { + "10": "intrepid", + "4": "hardy", + }, + "9": { + "10": "karmic", + "4": "jaunty", + }, + }, +} diff --git a/pkg/process/internal/codename/generate/main.go b/pkg/process/internal/codename/generate/main.go new file mode 100644 index 00000000..6a29d3ac --- /dev/null +++ b/pkg/process/internal/codename/generate/main.go @@ -0,0 +1,122 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/dave/jennifer/jen" +) + +const ( + outputPackage = "pkg/process/internal/codename" + outputPath = "internal/codename/codenames_generated.go" // relative to where go generate is called +) + +type Version struct { + Cycle string `json:"cycle"` + Codename string `json:"codename"` +} + +func main() { + osCodenames := make(map[string]map[string]map[string]string) + + fmt.Println("Fetching and parsing data for operating system codenames") + + fmt.Println("ubuntu:") + osCodenames["ubuntu"] = fetchAndParse("https://endoflife.date/api/ubuntu.json", ubuntuHandler) + + fmt.Println("debian:") + osCodenames["debian"] = fetchAndParse("https://endoflife.date/api/debian.json", lowercaseHandler) + + fmt.Printf("Generating code for %d operating system codenames\n", len(osCodenames)) + + f := jen.NewFile("codename") + f.HeaderComment("DO NOT EDIT: generated by pkg/process/v6/transformers/internal/codename/main.go") + f.ImportName(outputPackage, "pkg") + f.Var().Id("normalizedOSCodenames").Op("=").Map(jen.String()).Map(jen.String()).Map(jen.String()).String().Values(jen.DictFunc(func(d jen.Dict) { + for osName, versions := range osCodenames { + majorMap := jen.Dict{} + for major, minors := range versions { + minorMap := jen.Dict{} + for minor, codename := range minors { + minorMap[jen.Lit(minor)] = jen.Lit(codename) + } + majorMap[jen.Lit(major)] = jen.Values(minorMap) + } + d[jen.Lit(osName)] = jen.Values(majorMap) + } + })) + + rendered := fmt.Sprintf("%#v", f) + + file, err := os.OpenFile(outputPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + panic(fmt.Errorf("unable to open file: %w", err)) + } + defer file.Close() + + if _, err := file.WriteString(rendered); err != nil { + panic(fmt.Errorf("unable to write file: %w", err)) + } + + fmt.Printf("Code generation completed and written to %s\n", outputPath) +} + +// fetchAndParse fetches the JSON data from a URL, parses it, and organizes it into a map. +func fetchAndParse(url string, handler func(string) string) map[string]map[string]string { + resp, err := http.Get(url) //nolint:gosec + if err != nil { + panic(fmt.Errorf("error fetching data from %s: %w", url, err)) + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + panic(fmt.Errorf("error reading response: %w", err)) + } + + var versions []Version + if err := json.Unmarshal(data, &versions); err != nil { + panic(fmt.Errorf("error parsing JSON: %w", err)) + } + + parsedData := make(map[string]map[string]string) + for _, version := range versions { + major, minor := parseVersion(version.Cycle) + if parsedData[major] == nil { + parsedData[major] = make(map[string]string) + } + codename := handler(version.Codename) + fmt.Printf(" adding %s.%s --> %s\n", major, minor, codename) + parsedData[major][minor] = codename + } + + return parsedData +} + +func lowercaseHandler(codename string) string { + return strings.ToLower(codename) +} + +func ubuntuHandler(codename string) string { + return strings.ToLower(strings.Split(codename, " ")[0]) +} + +// parseVersion splits a version string like "20.04" into major "20" and minor "04". +func parseVersion(version string) (string, string) { + parts := strings.Split(version, ".") + major := strings.TrimLeft(parts[0], "0") + minor := "*" + if len(parts) > 1 { + if parts[1] == "0" { + minor = parts[1] + } else { + minor = strings.TrimLeft(parts[1], "0") + } + } + return major, minor +}