From 93e7d8066142e3225ec04e7d699ff2843db97a81 Mon Sep 17 00:00:00 2001 From: Doron KG <70585638+doronkg@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:58:13 +0300 Subject: [PATCH] feat: Deduplicate exceptions workflow (#332) * feat: Add exception dedup workflow * chore: Dedup exceptions * perf: Added JSON parallel processing --- .github/workflows/exceptions-json-sort.yml | 19 ----- .../workflows/validate-exception-jsons.yml | 22 +++++ Makefile | 27 ++++++- .../exceptions/clusterroles/clusterroles.json | 8 -- pkg/kor/exceptions/configmaps/configmaps.json | 4 - pkg/kor/exceptions/crds/crds.json | 80 ------------------- .../storageclasses/storageclasses.json | 4 - 7 files changed, 47 insertions(+), 117 deletions(-) delete mode 100644 .github/workflows/exceptions-json-sort.yml create mode 100644 .github/workflows/validate-exception-jsons.yml diff --git a/.github/workflows/exceptions-json-sort.yml b/.github/workflows/exceptions-json-sort.yml deleted file mode 100644 index 00cbbc2f..00000000 --- a/.github/workflows/exceptions-json-sort.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Validate Exceptions JSON Sorting - -on: - pull_request: - branches: - - main - paths: - - "pkg/kor/exceptions/**" - -jobs: - sort_json: - runs-on: ubuntu-latest - - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Validte Exceptions JSON Sorting - run: make validate-exception-sorting diff --git a/.github/workflows/validate-exception-jsons.yml b/.github/workflows/validate-exception-jsons.yml new file mode 100644 index 00000000..1a7a9650 --- /dev/null +++ b/.github/workflows/validate-exception-jsons.yml @@ -0,0 +1,22 @@ +name: Validate Exception JSONs + +on: + pull_request: + branches: + - main + paths: + - "pkg/kor/exceptions/**" + +jobs: + sort_json: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Validate Exception JSON Sorting + run: make validate-exception-sorting + + - name: Validate Exception Deduplication + run: make validate-exception-duplications diff --git a/Makefile b/Makefile index 08db44f2..835f8dda 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ test: sort-exception-files: @echo "Sorting exception files..." - @find $(EXCEPTIONS_DIR) -name '$(EXCEPTIONS_FILE_PATTERN)' -exec sh -c ' \ + @find $(EXCEPTIONS_DIR) -name '$(EXCEPTIONS_FILE_PATTERN)' | xargs -I{} -P 4 sh -c ' \ jq "with_entries(.value |= sort_by(.Namespace, .ResourceName))" {} > {}.tmp && mv {}.tmp {} \ ' \; @@ -36,4 +36,27 @@ validate-exception-sorting: done; \ if [ "$$PRINT_ERR" = 0 ]; then \ echo "Run the following command to sort all files recursively: make sort-exception-files"; \ - fi; \ \ No newline at end of file + fi; \ + +dedup-exception-files: + @echo "Deduplicating exception files..." + @find $(EXCEPTIONS_DIR) -type f -name '$(EXCEPTIONS_FILE_PATTERN)' | xargs -I{} -P 4 sh -c ' \ + jq '\''keys[0] as $$key | { ($$key): (.[$$key] | group_by(.Namespace, .ResourceName) | map(.[0])) }'\'' "$$1" > "$$1.tmp" && mv "$$1.tmp" "$$1" \ + ' sh {} \; + + +validate-exception-duplications: + @PRINT_ERR=1; \ + for file in $(wildcard $(EXCEPTIONS_DIR)/*/$(EXCEPTIONS_FILE_PATTERN)); do \ + DUPLICATES=$$(jq 'keys[0] as $$key | .[$$key] | group_by(.Namespace, .ResourceName) | map(select(length > 1))' "$$file"); \ + if [ "$$DUPLICATES" != "[]" ]; then \ + if [ "$$PRINT_ERR" = 1 ]; then \ + echo "The following JSON files contain duplications:"; \ + PRINT_ERR=0; \ + fi; \ + echo "\t$$file"; \ + fi; \ + done; \ + if [ "$$PRINT_ERR" = 0 ]; then \ + echo "Run the following command to deduplicate all files recursively: make dedup-exception-files"; \ + fi; \ diff --git a/pkg/kor/exceptions/clusterroles/clusterroles.json b/pkg/kor/exceptions/clusterroles/clusterroles.json index 22b17047..e555b257 100644 --- a/pkg/kor/exceptions/clusterroles/clusterroles.json +++ b/pkg/kor/exceptions/clusterroles/clusterroles.json @@ -132,10 +132,6 @@ "Namespace": "", "ResourceName": "system:aggregated-metrics-reader" }, - { - "Namespace": "", - "ResourceName": "system:aggregated-metrics-reader" - }, { "Namespace": "", "ResourceName": "system:auth-delegator" @@ -204,10 +200,6 @@ "Namespace": "", "ResourceName": "system:node-bootstrapper" }, - { - "Namespace": "", - "ResourceName": "system:node-bootstrapper" - }, { "Namespace": "", "ResourceName": "system:node-problem-detector" diff --git a/pkg/kor/exceptions/configmaps/configmaps.json b/pkg/kor/exceptions/configmaps/configmaps.json index e6557669..307618c8 100644 --- a/pkg/kor/exceptions/configmaps/configmaps.json +++ b/pkg/kor/exceptions/configmaps/configmaps.json @@ -46,10 +46,6 @@ "Namespace": "kube-system", "ResourceName": "cluster-dns" }, - { - "Namespace": "kube-system", - "ResourceName": "cluster-dns" - }, { "Namespace": "kube-system", "ResourceName": "cluster-kubestore" diff --git a/pkg/kor/exceptions/crds/crds.json b/pkg/kor/exceptions/crds/crds.json index 17d30036..7d84deb0 100644 --- a/pkg/kor/exceptions/crds/crds.json +++ b/pkg/kor/exceptions/crds/crds.json @@ -124,10 +124,6 @@ "Namespace": "", "ResourceName": "etcdsnapshotfiles.k3s.cattle.io" }, - { - "Namespace": "", - "ResourceName": "etcdsnapshotfiles.k3s.cattle.io" - }, { "Namespace": "", "ResourceName": "firmwareschemas.metal3.io" @@ -152,10 +148,6 @@ "Namespace": "", "ResourceName": "helmchartconfigs.helm.cattle.io" }, - { - "Namespace": "", - "ResourceName": "helmchartconfigs.helm.cattle.io" - }, { "Namespace": "", "ResourceName": "hostfirmwaresettings.metal3.io" @@ -180,14 +172,6 @@ "Namespace": "", "ResourceName": "ingressroutes.traefik.containo.us" }, - { - "Namespace": "", - "ResourceName": "ingressroutes.traefik.containo.us" - }, - { - "Namespace": "", - "ResourceName": "ingressroutetcps.traefik.containo.us" - }, { "Namespace": "", "ResourceName": "ingressroutetcps.traefik.containo.us" @@ -196,22 +180,10 @@ "Namespace": "", "ResourceName": "ingressroutetcps.traefik.io" }, - { - "Namespace": "", - "ResourceName": "ingressroutetcps.traefik.io" - }, { "Namespace": "", "ResourceName": "ingressrouteudps.traefik.containo.us" }, - { - "Namespace": "", - "ResourceName": "ingressrouteudps.traefik.containo.us" - }, - { - "Namespace": "", - "ResourceName": "ingressrouteudps.traefik.io" - }, { "Namespace": "", "ResourceName": "ingressrouteudps.traefik.io" @@ -260,14 +232,6 @@ "Namespace": "", "ResourceName": "middlewares.traefik.containo.us" }, - { - "Namespace": "", - "ResourceName": "middlewares.traefik.containo.us" - }, - { - "Namespace": "", - "ResourceName": "middlewares.traefik.io" - }, { "Namespace": "", "ResourceName": "middlewares.traefik.io" @@ -276,14 +240,6 @@ "Namespace": "", "ResourceName": "middlewaretcps.traefik.containo.us" }, - { - "Namespace": "", - "ResourceName": "middlewaretcps.traefik.containo.us" - }, - { - "Namespace": "", - "ResourceName": "middlewaretcps.traefik.io" - }, { "Namespace": "", "ResourceName": "middlewaretcps.traefik.io" @@ -356,14 +312,6 @@ "Namespace": "", "ResourceName": "serverstransports.traefik.containo.us" }, - { - "Namespace": "", - "ResourceName": "serverstransports.traefik.containo.us" - }, - { - "Namespace": "", - "ResourceName": "serverstransports.traefik.io" - }, { "Namespace": "", "ResourceName": "serverstransports.traefik.io" @@ -372,10 +320,6 @@ "Namespace": "", "ResourceName": "serverstransporttcps.traefik.io" }, - { - "Namespace": "", - "ResourceName": "serverstransporttcps.traefik.io" - }, { "Namespace": "", "ResourceName": "serviceattachments.networking.gke.io" @@ -400,22 +344,10 @@ "Namespace": "", "ResourceName": "tlsoptions.traefik.containo.us" }, - { - "Namespace": "", - "ResourceName": "tlsoptions.traefik.containo.us" - }, { "Namespace": "", "ResourceName": "tlsoptions.traefik.io" }, - { - "Namespace": "", - "ResourceName": "tlsoptions.traefik.io" - }, - { - "Namespace": "", - "ResourceName": "tlsstores.traefik.containo.us" - }, { "Namespace": "", "ResourceName": "tlsstores.traefik.containo.us" @@ -424,14 +356,6 @@ "Namespace": "", "ResourceName": "tlsstores.traefik.io" }, - { - "Namespace": "", - "ResourceName": "tlsstores.traefik.io" - }, - { - "Namespace": "", - "ResourceName": "traefikservices.traefik.containo.us" - }, { "Namespace": "", "ResourceName": "traefikservices.traefik.containo.us" @@ -440,10 +364,6 @@ "Namespace": "", "ResourceName": "traefikservices.traefik.io" }, - { - "Namespace": "", - "ResourceName": "traefikservices.traefik.io" - }, { "Namespace": "", "ResourceName": "updateinfos.nodemanagement.gke.io" diff --git a/pkg/kor/exceptions/storageclasses/storageclasses.json b/pkg/kor/exceptions/storageclasses/storageclasses.json index 43a3aabb..9ab733e8 100644 --- a/pkg/kor/exceptions/storageclasses/storageclasses.json +++ b/pkg/kor/exceptions/storageclasses/storageclasses.json @@ -24,10 +24,6 @@ "Namespace": "", "ResourceName": "local-path" }, - { - "Namespace": "", - "ResourceName": "local-path" - }, { "Namespace": "", "ResourceName": "managed"