From 32348e993b8473c57f0437fd8dd701aa41eceee2 Mon Sep 17 00:00:00 2001 From: Juan Correa Date: Tue, 5 Dec 2023 19:28:31 -0500 Subject: [PATCH 1/2] feature: ddd parallel processing in namespace. --- .idea/.gitignore | 8 ++++++++ .idea/imagepullsecret-patcher.iml | 9 +++++++++ .idea/modules.xml | 8 ++++++++ .idea/vcs.xml | 6 ++++++ go.mod | 1 + go.sum | 10 ++++++++++ main.go | 26 +++++++++++++++++++++++--- 7 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/imagepullsecret-patcher.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/imagepullsecret-patcher.iml b/.idea/imagepullsecret-patcher.iml new file mode 100644 index 0000000..5e764c4 --- /dev/null +++ b/.idea/imagepullsecret-patcher.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..3db2865 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/go.mod b/go.mod index 69d876d..802dc1b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/titansoft-pte-ltd/imagepullsecret-patcher go 1.13 require ( + github.com/panjf2000/ants/v2 v2.9.0 // indirect github.com/sirupsen/logrus v1.4.2 k8s.io/api v0.17.0 k8s.io/apimachinery v0.17.0 diff --git a/go.sum b/go.sum index 3f7c8c8..5a7753e 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,8 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/panjf2000/ants/v2 v2.9.0 h1:SztCLkVxBRigbg+vt0S5QvF5vxAbxbKt09/YfAJ0tEo= +github.com/panjf2000/ants/v2 v2.9.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -104,11 +106,16 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -136,6 +143,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -181,6 +189,8 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM= diff --git a/main.go b/main.go index 2675634..7c107d5 100644 --- a/main.go +++ b/main.go @@ -3,8 +3,10 @@ package main import ( "flag" "fmt" + "github.com/panjf2000/ants/v2" "os" "strings" + "sync" "time" log "github.com/sirupsen/logrus" @@ -107,11 +109,19 @@ func loop(k8s *k8sClient) { } log.Debugf("Got %d namespaces", len(namespaces.Items)) - for _, ns := range namespaces.Items { + // the pool sizes means how many goroutines should be used to process tasks in parallel. + poolSize := 10 + // Use the pool with a function, + var wg sync.WaitGroup + p, _ := ants.NewPoolWithFunc(poolSize, func(namespaceInstance interface{}) { + + defer wg.Done() + + ns := namespaceInstance.(corev1.Namespace) namespace := ns.Name if namespaceIsExcluded(ns) { log.Infof("[%s] Namespace skipped", namespace) - continue + return } log.Debugf("[%s] Start processing", namespace) // for each namespace, make sure the dockerconfig secret exists @@ -119,14 +129,24 @@ func loop(k8s *k8sClient) { if err != nil { // if has error in processing secret, should skip processing service account log.Error(err) - continue + return } // get default service account, and patch image pull secret if not exist err = processServiceAccount(k8s, namespace) if err != nil { log.Error(err) } + + }) + defer p.Release() + + // Submit tasks to process each namespace in parallel + for i := 0; i < len(namespaces.Items); i++ { + wg.Add(1) + _ = p.Invoke(namespaces.Items[i]) } + wg.Wait() + } func namespaceIsExcluded(ns corev1.Namespace) bool { From 7ba5265c9ebbfe91c0c21e16f9812bf710ab719d Mon Sep 17 00:00:00 2001 From: Juan Correa Date: Tue, 5 Dec 2023 19:29:07 -0500 Subject: [PATCH 2/2] feature: update gitignore --- .gitignore | 4 +++- .idea/.gitignore | 8 -------- .idea/imagepullsecret-patcher.iml | 9 --------- .idea/modules.xml | 8 -------- .idea/vcs.xml | 6 ------ 5 files changed, 3 insertions(+), 32 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/imagepullsecret-patcher.iml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore index 6c8c389..9ee2194 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ imagepullsecret-patcher # Dependency directories (remove the comment below to include it) # vendor/ .vscode/ -.DS_Store \ No newline at end of file +.DS_Store + +.idea \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/imagepullsecret-patcher.iml b/.idea/imagepullsecret-patcher.iml deleted file mode 100644 index 5e764c4..0000000 --- a/.idea/imagepullsecret-patcher.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 3db2865..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1dd..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file