Merge pull request #37 from openinfradev/fix_snr

feature. change apply rule logic to reload

.github/workflows/golangcic-lint.yml

@@ -1,35 +1,35 @@
 name: Lint
 on:
   push:
+    tags:
+      - v*
+    branches:
+      - main
+      - develop
+      - release
   pull_request:
-
+    branches:
+      - main
+      - develop
+      - release
 jobs:
   golangci:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
         with:
-          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: latest
-          args: --timeout=5m
-
-          # Optional: working directory, useful for monorepos
-          # working-directory: somedir
-
-          # Optional: golangci-lint command line arguments.
-          # args: --issues-exit-code=0
-
-          # Optional: show only new issues if it's a pull request. The default value is `false`.
-          # only-new-issues: true
-
-          # Optional: if set to true then the action will use pre-installed Go.
-          # skip-go-installation: true
-
-          # Optional: if set to true then the action don't cache or restore ~/go/pkg.
-          # skip-pkg-cache: true
-
-          # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
-          # skip-build-cache: true
+          go-version: "1.21"
+          cache: false
+      - name: Install golangci-lint
+        # Install golangci-lint from source instead of using
+        # golangci-lint-action to ensure the golangci-lint binary is built with
+        # the same Go version we're targeting.
+        # Avoids incompatibility issues such as:
+        # - https://github.com/golangci/golangci-lint/issues/2922
+        # - https://github.com/golangci/golangci-lint/issues/2673
+        # - https://github.com/golangci/golangci-lint-action/issues/442
+        run: go install github.com/golangci/golangci-lint/cmd/[email protected]
+      - name: Run golangci-lint
+        run: golangci-lint run --verbose --out-format=github-actions

cmd/server/appgroup_status.go

@@ -18,7 +18,7 @@ func processAppGroupStatus() error {
 	if len(appGroups) == 0 {
 		return nil
 	}
-	log.Info(context.TODO(), "appGroups : ", appGroups)
+	log.Info(context.TODO(), "[processAppGroupStatus] appGroups : ", appGroups)
 
 	for i := range appGroups {
 		appGroup := appGroups[i]

cmd/server/cloud_account_status.go

@@ -17,7 +17,7 @@ func processCloudAccountStatus() error {
 	if len(cloudAccounts) == 0 {
 		return nil
 	}
-	log.Info(context.TODO(), "cloudAccounts : ", cloudAccounts)
+	log.Info(context.TODO(), "[processCloudAccountStatus] cloudAccounts : ", cloudAccounts)
 
 	for i := range cloudAccounts {
 		cloudaccount := cloudAccounts[i]

cmd/server/cluster_byoh.go

@@ -22,7 +22,7 @@ func processClusterByoh() error {
 	if len(clusters) == 0 {
 		return nil
 	}
-	log.Info(context.TODO(), "byoh clusters : ", clusters)
+	log.Info(context.TODO(), "[processClusterByoh] byoh clusters : ", clusters)
 
 	token = getTksApiToken()
 	if token != "" {

cmd/server/cluster_status.go

@@ -17,7 +17,7 @@ func processClusterStatus() error {
 	if len(clusters) == 0 {
 		return nil
 	}
-	log.Info(context.TODO(), "clusters : ", clusters)
+	log.Info(context.TODO(), "[processClusterStatus] clusters : ", clusters)
 
 	for i := range clusters {
 		cluster := clusters[i]

cmd/server/main.go

@@ -9,15 +9,15 @@ import (
 	_apiClient "github.com/openinfradev/tks-api/pkg/api-client"
 	argo "github.com/openinfradev/tks-api/pkg/argo-client"
 	"github.com/openinfradev/tks-api/pkg/log"
-	"github.com/spf13/pflag"
-	"github.com/spf13/viper"
-
 	"github.com/openinfradev/tks-batch/internal/application"
 	cloudAccount "github.com/openinfradev/tks-batch/internal/cloud-account"
 	"github.com/openinfradev/tks-batch/internal/cluster"
 	"github.com/openinfradev/tks-batch/internal/database"
 	"github.com/openinfradev/tks-batch/internal/organization"
 	systemNotificationRule "github.com/openinfradev/tks-batch/internal/system-notification-rule"
+	gcache "github.com/patrickmn/go-cache"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
 )
 
 const INTERVAL_SEC = 5
@@ -30,6 +30,7 @@ var (
 	organizationAccessor           *organization.OrganizationAccessor
 	systemNotificationRuleAccessor *systemNotificationRule.SystemNotificationAccessor
 	apiClient                      _apiClient.ApiClient
+	cache                          *gcache.Cache
 )
 
 func init() {
@@ -84,6 +85,8 @@ func main() {
 		log.Fatal(context.TODO(), "failed to create tks-api client : ", err)
 	}
 
+	cache = gcache.New(5*time.Minute, 10*time.Minute)
+
 	for {
 		err = processClusterStatus()
 		if err != nil {
@@ -109,7 +112,10 @@ func main() {
 		if err != nil {
 			log.Error(context.TODO(), err)
 		}
-
+		err = processReloadThanosRules()
+		if err != nil {
+			log.Error(context.TODO(), err)
+		}
 		time.Sleep(time.Second * INTERVAL_SEC)
 	}
 

cmd/server/organization_status.go

@@ -17,7 +17,7 @@ func processOrganizationStatus() error {
 	if len(organizations) == 0 {
 		return nil
 	}
-	log.Info(context.TODO(), "organizations : ", organizations)
+	log.Info(context.TODO(), "[processOrganizationStatus] organizations : ", organizations)
 
 	for i := range organizations {
 		organization := organizations[i]

cmd/server/system_notification_rule.go

@@ -55,7 +55,7 @@ func processSystemNotificationRule() error {
 	if len(rules) == 0 {
 		return nil
 	}
-	log.Info(context.TODO(), "incompleted rules : ", len(rules))
+	log.Info(context.TODO(), "[processSystemNotificationRule] incompleted rules : ", len(rules))
 
 	incompletedOrganizations := []string{}
 
@@ -219,14 +219,17 @@ func applyRules(organizationId string, primaryClusterId string, rc RulerConfig)
 	}
 
 	// restart thanos-ruler
-	deletePolicy := metav1.DeletePropagationForeground
-	err = clientset.CoreV1().Pods("lma").Delete(context.TODO(), "thanos-ruler-0", metav1.DeleteOptions{
-		PropagationPolicy: &deletePolicy,
-	})
-	if err != nil {
-		log.Error(context.TODO(), err)
-		return err
-	}
+	// thanos-ruler reload 방식으로 변경했으나, 혹시 몰라 일단 코드는 주석처리해둠
+	/*
+		deletePolicy := metav1.DeletePropagationForeground
+		err = clientset.CoreV1().Pods("lma").Delete(context.TODO(), "thanos-ruler-0", metav1.DeleteOptions{
+			PropagationPolicy: &deletePolicy,
+		})
+		if err != nil {
+			log.Error(context.TODO(), err)
+			return err
+		}
+	*/
 
 	// update status
 	err = systemNotificationRuleAccessor.UpdateSystemNotificationRuleStatus(organizationId, domain.SystemNotificationRuleStatus_APPLIED)

cmd/server/thanos_ruler.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+
+	"github.com/openinfradev/tks-api/pkg/kubernetes"
+	"github.com/openinfradev/tks-api/pkg/log"
+	gcache "github.com/patrickmn/go-cache"
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const LAST_UPDATED_MIN = 2
+
+func processReloadThanosRules() error {
+	organizationIds, err := systemNotificationRuleAccessor.GetRecentlyUpdatedOrganizations(LAST_UPDATED_MIN)
+	if err != nil {
+		return err
+	}
+	if len(organizationIds) == 0 {
+		return nil
+	}
+	log.Info(context.TODO(), "[processReloadThanosRules] new updated organizationIds : ", organizationIds)
+
+	for _, organizationId := range organizationIds {
+		organization, err := organizationAccessor.Get(organizationId)
+		if err != nil {
+			log.Error(context.TODO(), err)
+			continue
+		}
+
+		url, err := GetThanosRulerUrl(organization.PrimaryClusterId)
+		if err != nil {
+			log.Error(context.TODO(), err)
+			continue
+		}
+
+		if err = Reload(url); err != nil {
+			log.Error(context.TODO(), err)
+			continue
+		}
+	}
+
+	return nil
+}
+
+func GetThanosRulerUrl(primaryClusterId string) (url string, err error) {
+	const prefix = "CACHE_KEY_THANOS_RULER_URL"
+	value, found := cache.Get(prefix + primaryClusterId)
+	if found {
+		log.Info(context.TODO(), "Cache HIT [CACHE_KEY_THANOS_RULER_URL] ", value)
+		return value.(string), nil
+	}
+
+	clientset_admin, err := kubernetes.GetClientAdminCluster(context.TODO())
+	if err != nil {
+		return url, errors.Wrap(err, "Failed to get client set for user cluster")
+	}
+
+	secrets, err := clientset_admin.CoreV1().Secrets(primaryClusterId).Get(context.TODO(), "tks-endpoint-secret", metav1.GetOptions{})
+	if err != nil {
+		log.Info(context.TODO(), "cannot found tks-endpoint-secret. so use LoadBalancer...")
+
+		clientset_user, err := kubernetes.GetClientFromClusterId(context.TODO(), primaryClusterId)
+		if err != nil {
+			return url, errors.Wrap(err, "Failed to get client set for user cluster")
+		}
+
+		service, err := clientset_user.CoreV1().Services("lma").Get(context.TODO(), "thanos-ruler", metav1.GetOptions{})
+		if err != nil {
+			return url, errors.Wrap(err, "Failed to get services.")
+		}
+
+		// LoadBalaner 일경우, aws address 형태의 경우만 가정한다.
+		if service.Spec.Type != "LoadBalancer" {
+			return url, fmt.Errorf("Service type is not LoadBalancer. [%s] ", service.Spec.Type)
+		}
+
+		lbs := service.Status.LoadBalancer.Ingress
+		ports := service.Spec.Ports
+		if len(lbs) > 0 && len(ports) > 0 {
+			url = ports[0].TargetPort.StrVal + "://" + lbs[0].Hostname + ":" + strconv.Itoa(int(ports[0].Port))
+		}
+	} else {
+		url = "http://" + string(secrets.Data["thanos-ruler"])
+	}
+
+	cache.Set(prefix+primaryClusterId, url, gcache.DefaultExpiration)
+	return url, nil
+}
+
+func Reload(thanosRulerUrl string) (err error) {
+	reqUrl := thanosRulerUrl + "/-/reload"
+
+	log.Info(context.TODO(), "url : ", reqUrl)
+	resp, err := http.Post(reqUrl, "text/plain", nil)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	_, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	return nil
+}

go.mod

@@ -6,6 +6,9 @@ toolchain go1.21.7
 
 require (
 	github.com/gofrs/uuid v4.0.0+incompatible
+	github.com/openinfradev/tks-api v0.0.0-20240411053710-5b8a434e8797
+	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.18.2
 	gopkg.in/yaml.v2 v2.4.0
@@ -47,9 +50,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/openinfradev/tks-api v0.0.0-20240411053710-5b8a434e8797 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect

go.sum

@@ -142,10 +142,10 @@ github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
 github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
 github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
 github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
-github.com/openinfradev/tks-api v0.0.0-20240409091158-eff7241c1731 h1:gmVBHSDzJGdf9p4wm28bDFcA3yFU6QjZl4prCd2fvIg=
-github.com/openinfradev/tks-api v0.0.0-20240409091158-eff7241c1731/go.mod h1:OGfXiL0YRby+OzOm+OI0d+wtPkOj3SMCiAv3lvpmaiU=
 github.com/openinfradev/tks-api v0.0.0-20240411053710-5b8a434e8797 h1:DQ5naso3RdA0XxQ2Fj70xZ4O3vBhtWYR9Kpdy7LQqRE=
 github.com/openinfradev/tks-api v0.0.0-20240411053710-5b8a434e8797/go.mod h1:Ph4lPgdWg06R1GUNCtmXfzHNlNCW/XjUAvei+m5DD2o=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
 github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

internal/organization/organization.go

@@ -12,10 +12,11 @@ import (
 
 // Organization represents a kubernetes organization information.
 type Organization struct {
-	ID         string `gorm:"primarykey"`
-	WorkflowId string
-	Status     domain.OrganizationStatus
-	StatusDesc string
+	ID               string `gorm:"primarykey"`
+	WorkflowId       string
+	Status           domain.OrganizationStatus
+	StatusDesc       string
+	PrimaryClusterId string
 }
 
 // Accessor accesses organization info in DB.
@@ -49,6 +50,15 @@ func (x *OrganizationAccessor) GetIncompleteOrganizations() ([]Organization, err
 	return organizations, nil
 }
 
+func (x *OrganizationAccessor) Get(id string) (organization Organization, err error) {
+	res := x.db.Where("id = ?", id).First(&organization)
+	if res.Error != nil {
+		return organization, res.Error
+	}
+
+	return
+}
+
 func (x *OrganizationAccessor) UpdateOrganizationStatus(organizationId string, status domain.OrganizationStatus, statusDesc string, workflowId string) error {
 	log.Info(context.TODO(), fmt.Sprintf("UpdateOrganizationStatus. organizationId[%s], status[%d], statusDesc[%s], workflowId[%s]", organizationId, status, statusDesc, workflowId))
 	res := x.db.Model(Organization{}).