Skip to content

Commit

Permalink
feat: support comparator for the same object
Browse files Browse the repository at this point in the history
  • Loading branch information
@adityathebe
adityathebe committed Dec 10, 2024
1 parent 2d8ded2 commit 690b319
Show file tree
Hide file tree
Showing 2 changed files with 143 additions and 68 deletions.
10 changes: 10 additions & 0 deletions scrapers/kubernetes/informers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,16 @@ func pqComparator(a, b any) int {
qa := a.(*QueueItem)
qb := b.(*QueueItem)

if qa.Obj.GetUID() == qb.Obj.GetUID() {
resourceVersionA, ok, _ := unstructured.NestedString(qa.Obj.Object, "metadata", "resourceVersion")
if ok {
resourceVersionB, _, _ := unstructured.NestedString(qb.Obj.Object, "metadata", "resourceVersion")

// Because of the way we are deduping, we want the earlier version in front of the queue.
return strings.Compare(resourceVersionA, resourceVersionB)
}
}

if opResult := pqCompareOperation(qa.Operation, qb.Operation); opResult != 0 {
return opResult
}
Expand Down
201 changes: 133 additions & 68 deletions scrapers/kubernetes/informers_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
package kubernetes

import (
"fmt"
"reflect"
"testing"
"time"

"github.com/emirpasic/gods/queues/priorityqueue"
"github.com/google/uuid"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
Expand All @@ -15,125 +17,166 @@ func TestPqComparator(t *testing.T) {

tests := []struct {
name string
a QueueItem
b QueueItem
Items []QueueItem
expected []string
}{
{
name: "add should have higher priority than update",
a: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "a", now),
},
b: QueueItem{
Operation: QueueItemOperationUpdate,
Obj: getUnstructured("Pod", "b", now),
Items: []QueueItem{
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "a", now),
},
{
Operation: QueueItemOperationUpdate,
Obj: getUnstructured("Pod", "b", now),
},
},
expected: []string{"a", "b"},
},
{
name: "update should have higher priority than delete",
a: QueueItem{
Operation: QueueItemOperationUpdate,
Obj: getUnstructured("Pod", "a", now),
},
b: QueueItem{
Operation: QueueItemOperationDelete,
Obj: getUnstructured("Pod", "b", now),
Items: []QueueItem{
{
Operation: QueueItemOperationUpdate,
Obj: getUnstructured("Pod", "a", now),
},
{
Operation: QueueItemOperationDelete,
Obj: getUnstructured("Pod", "b", now),
},
},
expected: []string{"a", "b"},
},
{
name: "same operation should compare by kind - Namespace vs Pod",
a: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Namespace", "a", now),
},
b: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "b", now),
Items: []QueueItem{
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Namespace", "a", now),
},
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "b", now),
},
},
expected: []string{"a", "b"},
},
{
name: "same operation and kind should compare by timestamp - earlier first",
a: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "a", now.Add(-1*time.Hour)),
},
b: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "b", now),
Items: []QueueItem{
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "a", now.Add(-1*time.Hour)),
},
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "b", now),
},
},
expected: []string{"a", "b"},
},
{
name: "namespace comes first even before a pod created earlier",
a: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "a", now.Add(-1*time.Hour)),
},
b: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Namespace", "b", now),
Items: []QueueItem{
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "a", now.Add(-1*time.Hour)),
},
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Namespace", "b", now),
},
},
expected: []string{"b", "a"},
},
{
name: "operation priority should override kind priority",
a: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "a", now),
},
b: QueueItem{
Operation: QueueItemOperationDelete,
Obj: getUnstructured("Namespace", "b", now),
Items: []QueueItem{
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "a", now),
},
{
Operation: QueueItemOperationDelete,
Obj: getUnstructured("Namespace", "b", now),
},
},
expected: []string{"a", "b"},
},
{
name: "unknown kind should use default priority",
a: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Canary", "a", now),
},
b: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "b", now),
Items: []QueueItem{
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Canary", "a", now),
},
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Pod", "b", now),
},
},
expected: []string{"a", "b"},
},
{
name: "events with managed fields",
a: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredEvent("Event", "a", now.Add(-2*time.Hour), now.Add(time.Hour)), // created ealier but re-created later
},
b: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredEvent("Event", "b", now.Add(-time.Hour), now.Add(time.Minute)),
Items: []QueueItem{
{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredEvent("Event", "a", now.Add(-2*time.Hour), now.Add(time.Hour)), // created ealier but re-created later
},
{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredEvent("Event", "b", now.Add(-time.Hour), now.Add(time.Minute)),
},
},
expected: []string{"b", "a"},
},
{
name: "identical objects of unknown kind with owner reference",
a: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredWithOwnerRef("Custom", "a", now, metav1.OwnerReference{Name: "http-canary", Kind: "Canary"}),
},
b: QueueItem{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Custom", "b", now),
Items: []QueueItem{
{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredWithOwnerRef("Custom", "a", now, metav1.OwnerReference{Name: "http-canary", Kind: "Canary"}),
},
{
Operation: QueueItemOperationAdd,
Obj: getUnstructured("Custom", "b", now),
},
},
expected: []string{"a", "b"},
},
{
name: "same objects",
Items: []QueueItem{
{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredWithResourceVersion("Pod", "a", "2c6a2f24-0199-435d-83a6-bd3f6d18d06d", "3", now.Add(-1*time.Hour)),
},
{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredWithResourceVersion("Pod", "a", "2c6a2f24-0199-435d-83a6-bd3f6d18d06d", "1", now.Add(-1*time.Hour)),
},
{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredWithResourceVersion("Pod", "a", "2c6a2f24-0199-435d-83a6-bd3f6d18d06d", "2", now.Add(-1*time.Hour)),
},
{
Operation: QueueItemOperationAdd,
Obj: getUnstructuredWithResourceVersion("Pod", "a", "2c6a2f24-0199-435d-83a6-bd3f6d18d06d", "4", now.Add(-1*time.Hour)),
},
},
expected: []string{"a-1", "a-2", "a-3", "a-4"},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
q := priorityqueue.NewWith(pqComparator)
q.Enqueue(&tt.a)
q.Enqueue(&tt.b)

for _, item := range tt.Items {
q.Enqueue(&item)
}

var result []string
for {
Expand All @@ -143,7 +186,13 @@ func TestPqComparator(t *testing.T) {
}

item := v.(*QueueItem)
result = append(result, item.Obj.GetName())

resourceVersion, ok, _ := unstructured.NestedString(item.Obj.Object, "metadata", "resourceVersion")
if ok {
result = append(result, fmt.Sprintf("%s-%s", item.Obj.GetName(), resourceVersion))
} else {
result = append(result, item.Obj.GetName())
}
}

if !reflect.DeepEqual(result, tt.expected) {
Expand Down Expand Up @@ -171,11 +220,26 @@ func getUnstructuredEvent(kind, name string, creationTimestamp, recreationTimest
}
}

func getUnstructuredWithResourceVersion(kind, name, uid, version string, creationTimestamp time.Time) *unstructured.Unstructured {
return &unstructured.Unstructured{
Object: map[string]any{
"kind": kind,
"metadata": map[string]any{
"uid": uid,
"name": name,
"creationTimestamp": creationTimestamp.Format(time.RFC3339),
"resourceVersion": version,
},
},
}
}

func getUnstructured(kind, name string, creationTimestamp time.Time) *unstructured.Unstructured {
return &unstructured.Unstructured{
Object: map[string]any{
"kind": kind,
"metadata": map[string]any{
"uid": uuid.NewString(),
"name": name,
"creationTimestamp": creationTimestamp.Format(time.RFC3339),
},
Expand All @@ -188,6 +252,7 @@ func getUnstructuredWithOwnerRef(kind, name string, creationTimestamp time.Time,
Object: map[string]any{
"kind": kind,
"metadata": map[string]any{
"uid": uuid.NewString(),
"name": name,
"creationTimestamp": creationTimestamp.Format(time.RFC3339),
"ownerReferences": []any{
Expand Down

0 comments on commit 690b319

Please sign in to comment.