diff --git a/go.mod b/go.mod index 99322e2ce..30ed9f028 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/prometheus/client_golang v1.19.1 github.com/samber/lo v1.39.0 go.uber.org/zap v1.26.0 + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f google.golang.org/protobuf v1.33.0 gotest.tools v2.2.0+incompatible istio.io/api v1.20.0 @@ -133,6 +134,7 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/tidwall/gjson v1.14.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect @@ -147,7 +149,6 @@ require ( go.starlark.net v0.0.0-20231121155337-90ade8b19d09 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.22.0 // indirect - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/oauth2 v0.19.0 // indirect golang.org/x/sync v0.7.0 // indirect diff --git a/go.sum b/go.sum index 13e5f5c7c..6c6383f8d 100644 --- a/go.sum +++ b/go.sum @@ -240,8 +240,6 @@ github.com/kuadrant/authorino v0.17.2 h1:UgWH4NY/n36IhoaU+ELUkoujaly1/9sx5mHY5vU github.com/kuadrant/authorino v0.17.2/go.mod h1:al71fN0FX6c9Orrhk9GR4CtjtC+CD/lUHJCs7drlRNM= github.com/kuadrant/authorino-operator v0.11.1 h1:jndTZhiHMU+2Dk0NU+KP2+MUSfvclrn+YtTCQDJj+1s= github.com/kuadrant/authorino-operator v0.11.1/go.mod h1:TeFFdX477vUTMushCojaHpvwPLga4DpErGI2oQbqFIs= -github.com/kuadrant/dns-operator v0.0.0-20240731163454-777df870df90 h1:T08iFChpKyulZ/umDEuYBLvYgJBuv/9nli3W0wjr8OA= -github.com/kuadrant/dns-operator v0.0.0-20240731163454-777df870df90/go.mod h1:Aq4LYFwhBzQYUew71KjtWPKr+e0jzgraX10Ki8wIKCY= github.com/kuadrant/dns-operator v0.0.0-20240809151102-e79ebbca8f70 h1:Jiq7dZWaepPZAVrG3QsDfVAIyR3qdgTdqN5v2lTvO8k= github.com/kuadrant/dns-operator v0.0.0-20240809151102-e79ebbca8f70/go.mod h1:Aq4LYFwhBzQYUew71KjtWPKr+e0jzgraX10Ki8wIKCY= github.com/kuadrant/limitador-operator v0.9.0 h1:hTQ6CFPayf/sL7cIzwWjCoU8uTn6fzWdsJgKbDlnFts= @@ -400,8 +398,8 @@ github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AV github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= -github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= diff --git a/quay/quay_overflow.go b/quay/quay_overflow.go new file mode 100644 index 000000000..019cb0419 --- /dev/null +++ b/quay/quay_overflow.go @@ -0,0 +1,251 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "regexp" + "sync" + "time" + + "golang.org/x/exp/maps" +) + +const ( + // Max number of entries returned as specified in Quay API docs for listing tags + pageLimit = 100 + accessTokenEnvKey = "ACCESS_TOKEN" +) + +var ( + accessToken = os.Getenv(accessTokenEnvKey) + preserveSubstrings = []string{ + "latest", + // Preserve semver release branch or semver tag regex - release-vX.Y.Z(-rc1) or vX.Y.Z(-rc1) + // Based on https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + "^(v|release-v)(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)(?:-(?P(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?P[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + } + client = &http.Client{Timeout: 5 * time.Second} +) + +// Tag represents a tag in the repository. +type Tag struct { + Name string `json:"name"` + Expiration string `json:"expiration"` +} + +// TagsResponse represents the structure of the API response that contains tags. +type TagsResponse struct { + Tags []Tag `json:"tags"` + // HasAdditional denotes whether there is still additional tags to be listed in the paginated response + HasAdditional bool `json:"has_additional"` +} + +func main() { + repo := flag.String("repo", "kuadrant/kuadrant-operator", "Repository name") + baseURL := flag.String("base-url", "https://quay.io/api/v1/repository", "Base API URL") + dryRun := flag.Bool("dry-run", true, "Dry run") + batchSize := flag.Int("batch-size", 50, "Batch size for deletion. API calls might get rate limited at large values") + flag.Parse() + + logger := log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime) + + if accessToken == "" { + logger.Fatalln("no access token provided") + } + + beginningTime := time.Now() + + // Fetch tags from the API + logger.Println("Fetching tags from Quay") + tags, err := fetchTags(baseURL, repo, accessToken) + if err != nil { + logger.Fatalln("Error fetching tags:", err) + } + + // Use filterTags to get tags to delete and preserved tags + logger.Println("Filtering tags") + tagsToDelete, preservedTags, err := filterTags(tags, preserveSubstrings) + if err != nil { + logger.Fatalln("Error filtering tags:", err) + } + + logger.Println("Tags to delete:", maps.Keys(tagsToDelete), "num", len(tagsToDelete)) + + var wg sync.WaitGroup + + // Delete tags in batches + i := 0 + for tagName := range tagsToDelete { + if i%*batchSize == 0 && i != 0 { + // Wait for the current batch to complete before starting a new one + wg.Wait() + } + + wg.Add(1) + go func(tagName string) { + defer wg.Done() + + if *dryRun { + logger.Printf("DRY RUN - Successfully deleted tag: %s\n", tagName) + } else { + if err := deleteTag(baseURL, repo, accessToken, tagName); err != nil { + logger.Println(err) + } + + logger.Printf("Successfully deleted tag: %s\n", tagName) + } + }(tagName) + + delete(tagsToDelete, tagName) // Remove deleted tag from tagsToDelete + i++ + } + + // Wait for the final batch to complete + wg.Wait() + + logger.Println("Preserved tags:", maps.Keys(preservedTags), "num", len(preservedTags)) + logger.Println("Tags not deleted successfully:", maps.Keys(tagsToDelete), len(tagsToDelete)) + logger.Println("Execution time:", time.Since(beginningTime).String()) +} + +// fetchTags retrieves the tags from the repository using the Quay.io API. +// https://docs.quay.io/api/swagger/#!/tag/listRepoTags +func fetchTags(baseURL, repo *string, accessToken string) ([]Tag, error) { + if baseURL == nil || repo == nil { + return nil, fmt.Errorf("baseURL or repo required") + } + + allTags := make([]Tag, 0) + i := 1 + + for { + url := fmt.Sprintf("%s/%s/tag/?page=%d&limit=%d", *baseURL, *repo, i, pageLimit) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + // Required for private repos + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", accessToken)) + + // Execute the request + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("error making request: %w", err) + } + defer resp.Body.Close() + + // Handle possible non-200 status codes + if resp.StatusCode != http.StatusOK { + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %w", err) + } + return nil, fmt.Errorf("error: received status code %d\nBody: %s", resp.StatusCode, string(body)) + } + + // Read the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %w", err) + } + + // Parse the JSON response + var tagsResp TagsResponse + if err := json.Unmarshal(body, &tagsResp); err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + allTags = append(allTags, tagsResp.Tags...) + + if tagsResp.HasAdditional { + i++ + continue + } + + // Has no additional pages + break + } + + return allTags, nil +} + +// deleteTag sends a DELETE request to remove the specified tag from the repository +// Returns nil if successful, error otherwise +// https://docs.quay.io/api/swagger/#!/tag/deleteFullTag +func deleteTag(baseURL, repo *string, accessToken, tagName string) error { + if baseURL == nil || repo == nil { + return fmt.Errorf("baseURL or repo required") + } + + url := fmt.Sprintf("%s/%s/tag/%s", *baseURL, *repo, tagName) + + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return fmt.Errorf("error creating DELETE request: %w", err) + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", accessToken)) + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("error deleting tag: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNoContent { + return nil + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("error reading response body: %w", err) + } + return fmt.Errorf("failed to delete tag %s: Status code %d Body: %s", tagName, resp.StatusCode, string(body)) +} + +// filterTags takes a slice of tags and preserves string regex and returns two maps: one for tags to delete and one for preserved tags. +func filterTags(tags []Tag, preserveSubstrings []string) (map[string]struct{}, map[string]struct{}, error) { + tagsToDelete := make(map[string]struct{}) + preservedTags := make(map[string]struct{}) + + // Compile the regexes for each preserve substring + preserveRegexes := make([]*regexp.Regexp, 0, len(preserveSubstrings)) + for _, substr := range preserveSubstrings { + regex, err := regexp.Compile(substr) + if err != nil { + return nil, nil, err + } + preserveRegexes = append(preserveRegexes, regex) + } + + for _, tag := range tags { + // Tags that have an expiration set are ignored as they could be historical tags that have already expired + // i.e. when an existing tag is updated, the previous tag of the same name is expired and is still returned when listing + // the tags + if tag.Expiration != "" { + continue + } + + // Check if the tag name matches any of the preserve substrings + preserve := false + for _, regex := range preserveRegexes { + if regex.MatchString(tag.Name) { + preserve = true + break + } + } + + if !preserve { + tagsToDelete[tag.Name] = struct{}{} + } else { + preservedTags[tag.Name] = struct{}{} + } + } + + return tagsToDelete, preservedTags, nil +} diff --git a/quay/quay_overflow_test.go b/quay/quay_overflow_test.go new file mode 100644 index 000000000..6fa3402a8 --- /dev/null +++ b/quay/quay_overflow_test.go @@ -0,0 +1,263 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" +) + +var ( + testBaseURL = "https://quay.io/api/v1/" + testRepo = "testOrg/kuadrant-operator" + testAccessToken = "fake_access_token" +) + +func Test_fetchTags(t *testing.T) { + t.Run("test error for non-200 status codes", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + tags, err := fetchTags(&server.URL, &testRepo, testAccessToken) + + if err == nil { + t.Error("error expected") + } + + if !strings.Contains(err.Error(), "error: received status code 500") { + t.Errorf("error expected, got %s", err.Error()) + } + + if tags != nil { + t.Error("expected nil tags") + } + }) + + t.Run("test error parsing json", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("{notTags: error}")) + })) + defer server.Close() + tags, err := fetchTags(&server.URL, &testRepo, testAccessToken) + + if err == nil { + t.Error("error expected") + } + + if !strings.Contains(err.Error(), "error unmarshalling response:") { + t.Errorf("error expected, got %s", err.Error()) + } + + if tags != nil { + t.Error("expected nil tags") + } + }) + + t.Run("test bearer token is added to header", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", testAccessToken) { + t.Errorf("unexpected authorization header: %v", r.Header.Get("Authorization")) + } + })) + defer server.Close() + + _, _ = fetchTags(&server.URL, &testRepo, testAccessToken) + }) + + t.Run("test successful response with tags", func(t *testing.T) { + mockJSONResponse := `{ + "tags": [ + {"name": "v1.0.0"}, + {"name": "v1.1.0"}, + {"name": "latest"} + ] + }` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(mockJSONResponse)) + })) + defer server.Close() + + tags, err := fetchTags(&server.URL, &testRepo, testAccessToken) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Validate the returned tags + if len(tags) != 3 { + t.Fatalf("expected 3 tags, got %d", len(tags)) + } + + expectedTags := []string{ + "v1.0.0", + "v1.1.0", + "latest", + } + + for _, tag := range tags { + if !slices.Contains(expectedTags, tag.Name) { + t.Errorf("unexpected tag: %v, does not exist in expected tags %v", tag, expectedTags) + } + } + }) + + t.Run("test error nil baseUrl", func(t *testing.T) { + _, err := fetchTags(nil, &testRepo, testAccessToken) + if err == nil { + t.Fatal("error expected") + } + + if err.Error() != "baseURL or repo required" { + t.Errorf("error expected, got %s", err.Error()) + } + }) + + t.Run("test error nil repo", func(t *testing.T) { + _, err := fetchTags(&testBaseURL, nil, testAccessToken) + if err == nil { + t.Fatal("error expected") + } + + if err.Error() != "baseURL or repo required" { + t.Errorf("error expected, got %s", err.Error()) + } + }) +} + +func Test_deleteTag(t *testing.T) { + t.Run("test successful delete", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + err := deleteTag(&server.URL, &testRepo, testAccessToken, "v1.0.0") + + if err != nil { + t.Errorf("expected successful delete, got error: %s", err.Error()) + } + }) + + t.Run("test delete with error response", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + err := deleteTag(&server.URL, &testRepo, testAccessToken, "v1.0.0") + + if err == nil { + t.Error("expected failure, got success") + } + + if !strings.Contains(err.Error(), "failed to delete tag v1.0.0: Status code 500") { + t.Errorf("error expected, got %s", err.Error()) + } + }) + + t.Run("test bearer token is added to header", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", testAccessToken) { + t.Errorf("unexpected authorization header: %v", r.Header.Get("Authorization")) + } + })) + defer server.Close() + + _ = deleteTag(&server.URL, &testRepo, testAccessToken, "v1.0.0") + }) + + t.Run("test error nil baseUrl", func(t *testing.T) { + err := deleteTag(nil, &testRepo, testAccessToken, "v1.0.0") + + if err == nil { + t.Error("expected failure, got success") + } + + if err.Error() != "baseURL or repo required" { + t.Errorf("error expected, got %s", err.Error()) + } + }) + + t.Run("test error nil repo", func(t *testing.T) { + err := deleteTag(&testBaseURL, nil, testAccessToken, "v1.0.0") + + if err == nil { + t.Error("expected failure, got success") + } + + if err.Error() != "baseURL or repo required" { + t.Errorf("error expected, got %s", err.Error()) + } + }) +} + +func Test_filterTags(t *testing.T) { + t.Run("test filter tags correctly", func(t *testing.T) { + tags := []Tag{ + {Name: "nightly-build"}, // Not a preserved tag, should be deleted + {Name: "latest"}, // Preserved tag, name is latest + {Name: "release-v1.0.0"}, // Preserved tag, name contains preserveSubstring branch release semver, release-v* + {Name: "v1.0.0"}, // Preserved tag, but name contains preserveSubstring tag semver release + {Name: "v1.1.0-rc1"}, // Preserved tag, but name contains preserveSubstring tag semver release-candidate + {Name: "expiry_set", Expiration: time.Now().Format(time.RFC1123)}, // Skipped tag, already has an expiry set + {Name: "release-not-semver"}, // Not a preserved tag, should be deleted + } + + tagsToDelete, remainingTags, err := filterTags(tags, preserveSubstrings) + + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if len(tagsToDelete) != 2 || len(remainingTags) != 4 { + t.Fatalf("expected 2 tag to delete and 4 remaining, got %d to delete and %d remaining", len(tagsToDelete), len(remainingTags)) + } + + if _, ok := tagsToDelete["nightly-build"]; !ok { + t.Error("expected nightly-build to be deleted") + } + + if _, ok := tagsToDelete["release-not-semver"]; !ok { + t.Error("expected release-not-semver to be deleted") + } + + if _, ok := remainingTags["latest"]; !ok { + t.Error("expected latest to be kept") + } + + if _, ok := remainingTags["release-v1.0.0"]; !ok { + t.Error("expected release-v1.0.0 to be kept") + } + + if _, ok := remainingTags["v1.0.0"]; !ok { + t.Error("expected v1.0.0 to be kept") + } + + if _, ok := remainingTags["v1.1.0-rc1"]; !ok { + t.Error("expected v1.1.0-rc1 to be kept") + } + }) + + t.Run("test filter tags with no deletions", func(t *testing.T) { + tags := []Tag{ + {Name: "v1.1.0"}, // Preserved tag, should be kept + {Name: "latest"}, // Preserved tag, should be kept + } + + tagsToDelete, remainingTags, err := filterTags(tags, preserveSubstrings) + + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if len(tagsToDelete) != 0 || len(remainingTags) != 2 { + t.Fatalf("expected 0 tags to delete and 2 remaining, got %d to delete and %d remaining", len(tagsToDelete), len(remainingTags)) + } + }) +}