[CI] add basic static check, compile, test

Signed-off-by: xiaoming <[email protected]> [fix]playbook: fix format related variables position Signed-off-by: sjf <[email protected]> [fix] add support for recognizing kernel versions like "3.15.0_.*" Signed-off-by: jyf111 <[email protected]> support enable etcd auth Signed-off-by: wanghai01 <[email protected]> Feature: podman can be an alternative container engine Signed-off-by: caoxianfei1 <[email protected]> bugfix: docker ps specify both --format and --quiet will warning since v24.0.0. WARNING: Ignoring custom format, because both --format and --quiet are set. Signed-off-by: caoxianfei1 <[email protected]> change replicas to instances Signed-off-by: tec-rubbish maker <[email protected]> change topology variable change service_replicas_sequence and format_replicas_sequence to service_instances_sequence and format_instances_sequence Signed-off-by: tec-rubbish maker <[email protected]> fix the worng instances Signed-off-by: tec-rubbish maker <[email protected]> mistake Signed-off-by: tec-rubbish maker <[email protected]> Update cli/command/status.go Signed-off-by: Wine93 <[email protected]> [fix] older versions of the ss command do not have the --no-header option solution: execute the ss command in a temporary container to precheck for ports in use instead Signed-off-by: jyf111 <[email protected]> Signed-off-by: xiaoming <[email protected]>
opencurve · Oct 7, 2023 · f3fd9f0 · f3fd9f0
1 parent 3b6b0a1
commit f3fd9f0
Show file tree

Hide file tree

Showing 93 changed files with 739 additions and 475 deletions.
diff --git a/.github/workflows/CodeQL.yml b/.github/workflows/CodeQL.yml
@@ -0,0 +1,54 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master","develop" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "master","develop" ]
+  schedule:
+    - cron: '0 17 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    # timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -7,7 +7,23 @@ on:
     branches: [ $default-branch, "develop" ]
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.19'
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Setup golangci-lint
+        uses: golangci/[email protected]
+        with:
+          version: v1.52.2
+          args: --verbose
+
   test:
+    needs: lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -21,15 +37,13 @@ jobs:
         run: make test
 
   build:
-    needs: test
+    needs: lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
           go-version: 1.19
-      - name: Set dependencies
-        run: sudo apt update && sudo apt install musl-tools
       - name: Build
-        run: make build
+        run: go build -v ./...
diff --git a/.golangci.yml b/.golangci.yml
@@ -0,0 +1,38 @@
+run:
+  timeout: 5m
+linters:
+  enable:
+    # Simple linter to check that your code does not contain non-ASCII identifiers.
+    - asciicheck
+    # Go linter that checks if package imports are in a list of acceptable packages.
+    - depguard
+    # Checks for pointers to enclosing loop variables.
+    - exportloopref
+    # Gofmt checks whether code was gofmt-ed
+    - gofmt
+    # Inspects source code for security problems.	
+    - gosec
+    # Finds naked returns in functions greater than a specified function length.
+    - nakedret
+    # Reports ill-formed or insufficient nolint directives.
+    - nolintlint
+    # Checks for dangerous unicode character sequences.
+    - bidichk
+  disable:
+    # ignore errcheck
+    - errcheck
+linters-settings:
+  gosec:
+    # Available rules: https://github.com/securego/gosec#available-rules
+    excludes:
+      - G101	# Look for hard coded credentials
+      - G204	# Audit use of command execution
+      - G401	# Detect the usage of DES, RC4, MD5 or SHA1
+      - G403	# Ensure minimum RSA key length of 2048 bits
+      - G404	# Insecure random number source (rand)
+      - G501	# Import blocklist: crypto/md5
+      - G502	# Import blocklist: crypto/des
+      - G503	# Import blocklist: crypto/rc4
+      - G504	# Import blocklist: net/http/cgi
+      - G505	# Import blocklist: crypto/sha1
+      - G601	# Implicit memory aliasing of items from a range statement
diff --git a/Makefile b/Makefile
@@ -56,14 +56,16 @@ SERVER_PACKAGES := $(PWD)/cmd/service/main.go
 # tar
 VERSION := "unknown"
 
-build:
+build: fmt vet
 	$(GOENV) $(GO) build -o $(OUTPUT) $(BUILD_FLAGS) $(PACKAGES)
 	$(GOENV) $(GO) build -o $(SERVER_OUTPUT) $(BUILD_FLAGS) $(SERVER_PACKAGES)
 
-debug:
+
+debug: fmt vet
 	$(GOENV) $(GO) build -o $(OUTPUT) $(DEBUG_FLAGS) $(PACKAGES)
 	$(GOENV) $(GO) build -o $(SERVER_OUTPUT) $(DEBUG_FLAGS) $(SERVER_PACKAGES)
 
+
 test:
 	$(GO_TEST) $(TEST_FLAGS) ./...
 
@@ -74,3 +76,8 @@ lint:
 	go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCILINT_VERSION)
 	$(GOBIN_GOLANGCILINT) run -v
 
+fmt:
+	go fmt ./...
+
+vet:
+	go vet ./...
diff --git a/cli/cli/cli.go b/cli/cli/cli.go
@@ -94,11 +94,11 @@ func NewCurveAdm() (*CurveAdm, error) {
 
 	rootDir := fmt.Sprintf("%s/.curveadm", home)
 	curveadm := &CurveAdm{
-		rootDir:   rootDir,
-		dataDir:   path.Join(rootDir, "data"),
-		pluginDir: path.Join(rootDir, "plugins"),
-		logDir:    path.Join(rootDir, "logs"),
-		tempDir:   path.Join(rootDir, "temp"),
+		rootDir:      rootDir,
+		dataDir:      path.Join(rootDir, "data"),
+		pluginDir:    path.Join(rootDir, "plugins"),
+		logDir:       path.Join(rootDir, "logs"),
+		tempDir:      path.Join(rootDir, "temp"),
 		httpConfPath: path.Join(rootDir, "http/conf"),
 		httpLogPath:  path.Join(rootDir, "http/logs"),
 	}
@@ -248,7 +248,7 @@ func (curveadm *CurveAdm) detectVersion() {
 }
 
 func (curveadm *CurveAdm) Upgrade() (bool, error) {
-	if curveadm.config.GetAutoUpgrade() == false {
+	if !curveadm.config.GetAutoUpgrade() {
 		return false, nil
 	}
 
@@ -294,6 +294,7 @@ func (curveadm *CurveAdm) LogPath() string                   { return curveadm.l
 func (curveadm *CurveAdm) Config() *configure.CurveAdmConfig { return curveadm.config }
 func (curveadm *CurveAdm) SudoAlias() string                 { return curveadm.config.GetSudoAlias() }
 func (curveadm *CurveAdm) SSHTimeout() int                   { return curveadm.config.GetSSHTimeout() }
+func (curveadm *CurveAdm) Engine() string                    { return curveadm.config.GetEngine() }
 func (curveadm *CurveAdm) In() io.Reader                     { return curveadm.in }
 func (curveadm *CurveAdm) Out() io.Writer                    { return curveadm.out }
 func (curveadm *CurveAdm) Err() io.Writer                    { return curveadm.err }
@@ -421,6 +422,7 @@ func (curveadm *CurveAdm) ExecOptions() module.ExecOptions {
 		ExecInLocal:    false,
 		ExecSudoAlias:  curveadm.config.GetSudoAlias(),
 		ExecTimeoutSec: curveadm.config.GetTimeout(),
+		ExecWithEngine: curveadm.config.GetEngine(),
 	}
 }
 
@@ -549,8 +551,7 @@ func (curveadm *CurveAdm) PostAudit(id int64, ec error) {
 		return
 	}
 
-	auditLog := auditLogs[0]
-	status := auditLog.Status
+	var status int
 	errorCode := 0
 	if ec == nil {
 		status = comm.AUDIT_STATUS_SUCCESS

diff --git a/cli/command/clean.go b/cli/command/clean.go
@@ -121,7 +121,7 @@ func genCleanPlaybook(curveadm *cli.CurveAdm,
 			Configs: dcs,
 			Options: map[string]interface{}{
 				comm.KEY_CLEAN_ITEMS:      options.only,
-				comm.KEY_CLEAN_BY_RECYCLE: options.withoutRecycle == false,
+				comm.KEY_CLEAN_BY_RECYCLE: !options.withoutRecycle,
 			},
 		})
 	}

diff --git a/cli/command/client/map.go b/cli/command/client/map.go
@@ -27,7 +27,6 @@ import (
 	"strings"
 
 	"github.com/dustin/go-humanize"
-
 	"github.com/fatih/color"
 	"github.com/opencurve/curveadm/cli/cli"
 	comm "github.com/opencurve/curveadm/internal/common"

diff --git a/cli/command/cluster/import.go b/cli/command/cluster/import.go
@@ -138,10 +138,8 @@ func readDatabase(filename string) (storage.Cluster, []storage.Service, error) {
 		switch id {
 		case CLUSTER_DESCRIPTION:
 			cluster.Description = value
-			break
 		case CLUSTER_TOPOLOGY:
 			cluster.Topology = value
-			break
 		case SERVICE:
 			items := strings.Split(value, " ")
 			if len(items) != 2 {
@@ -166,7 +164,8 @@ func importCluster(storage *storage.Storage, name, dbfile string) error {
 	}
 
 	// insert cluster
-	if storage.InsertCluster(name, cluster.Description, cluster.Topology); err != nil {
+	err = storage.InsertCluster(name, cluster.Description, cluster.Topology)
+	if err != nil {
 		return err
 	}
 

diff --git a/cli/command/completion.go b/cli/command/completion.go
@@ -77,7 +77,7 @@ PowerShell:
 `, "curveadm"),
 		DisableFlagsInUseLine: true,
 		ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
-		Args:                  cobra.ExactValidArgs(1),
+		Args:                  cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			switch args[0] {
 			case "bash":

diff --git a/cli/command/deploy.go b/cli/command/deploy.go
@@ -43,6 +43,7 @@ const (
 	CREATE_CONTAINER           = playbook.CREATE_CONTAINER
 	SYNC_CONFIG                = playbook.SYNC_CONFIG
 	START_ETCD                 = playbook.START_ETCD
+	ENABLE_ETCD_AUTH           = playbook.ENABLE_ETCD_AUTH
 	START_MDS                  = playbook.START_MDS
 	CREATE_PHYSICAL_POOL       = playbook.CREATE_PHYSICAL_POOL
 	START_CHUNKSERVER          = playbook.START_CHUNKSERVER
@@ -65,6 +66,7 @@ var (
 		CREATE_CONTAINER,
 		SYNC_CONFIG,
 		START_ETCD,
+		ENABLE_ETCD_AUTH,
 		START_MDS,
 		CREATE_PHYSICAL_POOL,
 		START_CHUNKSERVER,
@@ -79,13 +81,15 @@ var (
 		CREATE_CONTAINER,
 		SYNC_CONFIG,
 		START_ETCD,
+		ENABLE_ETCD_AUTH,
 		START_MDS,
 		CREATE_LOGICAL_POOL,
 		START_METASERVER,
 	}
 
 	DEPLOY_FILTER_ROLE = map[int]string{
 		START_ETCD:           ROLE_ETCD,
+		ENABLE_ETCD_AUTH:     ROLE_ETCD,
 		START_MDS:            ROLE_MDS,
 		START_CHUNKSERVER:    ROLE_CHUNKSERVER,
 		START_SNAPSHOTCLONE:  ROLE_SNAPSHOTCLONE,
@@ -99,6 +103,7 @@ var (
 		CREATE_PHYSICAL_POOL: 1,
 		CREATE_LOGICAL_POOL:  1,
 		BALANCE_LEADER:       1,
+		ENABLE_ETCD_AUTH:     1,
 	}
 
 	CAN_SKIP_ROLES = []string{
@@ -160,11 +165,12 @@ func skipServiceRole(deployConfigs []*topology.DeployConfig, options deployOptio
 	return dcs
 }
 
-func skipDeploySteps(deploySteps []int, options deployOptions) []int {
+func skipDeploySteps(dcs []*topology.DeployConfig, deploySteps []int, options deployOptions) []int {
 	steps := []int{}
 	skipped := utils.Slice2Map(options.skip)
 	for _, step := range deploySteps {
-		if step == START_SNAPSHOTCLONE && skipped[ROLE_SNAPSHOTCLONE] {
+		if (step == START_SNAPSHOTCLONE && skipped[ROLE_SNAPSHOTCLONE]) ||
+			(step == ENABLE_ETCD_AUTH && len(dcs) > 0 && !dcs[0].GetEtcdAuthEnable()) {
 			continue
 		}
 		steps = append(steps, step)
@@ -211,7 +217,7 @@ func genDeployPlaybook(curveadm *cli.CurveAdm,
 	if kind == topology.KIND_CURVEBS {
 		steps = CURVEBS_DEPLOY_STEPS
 	}
-	steps = skipDeploySteps(steps, options)
+	steps = skipDeploySteps(dcs, steps, options)
 	poolset := options.poolset
 	diskType := options.poolsetDiskType
 
@@ -223,9 +229,9 @@ func genDeployPlaybook(curveadm *cli.CurveAdm,
 			role := DEPLOY_FILTER_ROLE[step]
 			config = curveadm.FilterDeployConfigByRole(config, role)
 		}
-		n := len(config)
+
 		if DEPLOY_LIMIT_SERVICE[step] > 0 {
-			n = DEPLOY_LIMIT_SERVICE[step]
+			n := DEPLOY_LIMIT_SERVICE[step]
 			config = config[:n]
 		}
 

diff --git a/cli/command/exec.go b/cli/command/exec.go
@@ -48,7 +48,6 @@ func NewExecCommand(curveadm *cli.CurveAdm) *cobra.Command {
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			options.id = args[0]
 			options.cmd = strings.Join(args[1:], " ")
-			args = args[:1]
 			return curveadm.CheckId(options.id)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {

diff --git a/cli/command/migrate.go b/cli/command/migrate.go
@@ -152,7 +152,7 @@ func checkMigrateTopology(curveadm *cli.CurveAdm, data string) error {
 		dcs2add[0].GetRole() != dcs2del[0].GetRole() {
 		return errno.ERR_REQUIRE_SAME_ROLE_SERVICES_FOR_MIGRATING
 	}
-	if len(dcs2del) != dcs2del[0].GetReplicas() {
+	if len(dcs2del) != dcs2del[0].GetInstances() {
 		return errno.ERR_REQUIRE_WHOLE_HOST_SERVICES_FOR_MIGRATING
 	}
 

diff --git a/cli/command/status.go b/cli/command/status.go
@@ -48,11 +48,11 @@ var (
 )
 
 type statusOptions struct {
-	id           string
-	role         string
-	host         string
-	verbose      bool
-	showReplicas bool
+	id            string
+	role          string
+	host          string
+	verbose       bool
+	showInstances bool
 }
 
 func NewStatusCommand(curveadm *cli.CurveAdm) *cobra.Command {
@@ -73,7 +73,7 @@ func NewStatusCommand(curveadm *cli.CurveAdm) *cobra.Command {
 	flags.StringVar(&options.role, "role", "*", "Specify service role")
 	flags.StringVar(&options.host, "host", "*", "Specify service host")
 	flags.BoolVarP(&options.verbose, "verbose", "v", false, "Verbose output for status")
-	flags.BoolVarP(&options.showReplicas, "show-replicas", "s", false, "Display service replicas")
+	flags.BoolVarP(&options.showInstances, "show-instances", "s", false, "Display service instances")
 
 	return cmd
 }
@@ -113,7 +113,7 @@ func displayStatus(curveadm *cli.CurveAdm, dcs []*topology.DeployConfig, options
 		}
 	}
 
-	output := tui.FormatStatus(statuses, options.verbose, options.showReplicas)
+	output := tui.FormatStatus(statuses, options.verbose, options.showInstances)
 	curveadm.WriteOutln("")
 	curveadm.WriteOutln("cluster name      : %s", curveadm.ClusterName())
 	curveadm.WriteOutln("cluster kind      : %s", dcs[0].GetKind())