Merge branch 'tencent-master' into issue-9093

src/agent/go.mod

@@ -20,7 +20,7 @@ require (
 )
 
 require (
-	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/alecthomas/participle v0.4.1 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
@@ -65,20 +65,34 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
+	github.com/fsouza/go-dockerclient v1.9.7
+	github.com/gorilla/websocket v1.5.0
+	golang.org/x/sync v0.1.0
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/blues/jsonata-go v1.5.4 // indirect
+	github.com/containerd/containerd v1.6.18 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/moby/patternmatcher v0.5.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/runc v1.1.5 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.5.0 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/tools v0.1.12 // indirect
 )
 
 require (

src/agent/src/cmd/agent/main.go

@@ -79,9 +79,7 @@ func main() {
 		systemutil.ExitProcess(1)
 	}
 
-	logs.Info("GOOS=%s, GOARCH=%s", runtime.GOOS, runtime.GOARCH)
-
-	runtime.GOMAXPROCS(4)
+	logs.Infof("GOOS=%s, GOARCH=%s", runtime.GOOS, runtime.GOARCH)
 
 	// 以agent安装目录为工作目录
 	workDir := systemutil.GetExecutableDir()

src/agent/src/cmd/daemon/main.go

@@ -40,6 +40,7 @@ import (
 
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/config"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/logs"
+	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/upgrade"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/util/fileutil"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/util/systemutil"
 
@@ -76,9 +77,7 @@ func main() {
 		systemutil.ExitProcess(1)
 	}
 
-	logs.Info("GOOS=%s, GOARCH=%s", runtime.GOOS, runtime.GOARCH)
-
-	runtime.GOMAXPROCS(4)
+	logs.Infof("GOOS=%s, GOARCH=%s", runtime.GOOS, runtime.GOARCH)
 
 	workDir := systemutil.GetExecutableDir()
 	err = os.Chdir(workDir)
@@ -119,7 +118,7 @@ func watch(isDebug bool) {
 		select {
 		case <-checkTimeTicker.C:
 			if err := totalLock.Lock(); err != nil {
-				logs.Error("failed to get agent lock: %v", err)
+				logs.Errorf("failed to get agent lock: %v", err)
 				continue
 			}
 
@@ -143,7 +142,7 @@ func doCheckAndLaunchAgent(isDebug bool) {
 		}()
 	}
 	if err != nil {
-		logs.Error("try to get agent.lock failed: %v", err)
+		logs.Errorf("try to get agent.lock failed: %v", err)
 		return
 	}
 	if !locked {
@@ -154,14 +153,14 @@ func doCheckAndLaunchAgent(isDebug bool) {
 
 	process, err := launch(workDir+"/"+config.AgentFileClientLinux, isDebug)
 	if err != nil {
-		logs.Error("launch agent failed: %v", err)
+		logs.Errorf("launch agent failed: %v", err)
 		return
 	}
 	if process == nil {
 		logs.Error("launch agent failed: got a nil process")
 		return
 	}
-	logs.Info("success to launch agent, pid: %d", process.Pid)
+	logs.Infof("success to launch agent, pid: %d", process.Pid)
 }
 
 func launch(agentPath string, isDebug bool) (*os.Process, error) {
@@ -174,7 +173,7 @@ func launch(agentPath string, isDebug bool) (*os.Process, error) {
 
 	cmd.Dir = systemutil.GetWorkDir()
 
-	logs.Info("start devops agent: %s", cmd.String())
+	logs.Infof("start devops agent: %s", cmd.String())
 	if !fileutil.Exists(agentPath) {
 		return nil, fmt.Errorf("agent file %s not exists", agentPath)
 	}
@@ -189,7 +188,14 @@ func launch(agentPath string, isDebug bool) (*os.Process, error) {
 	}
 
 	go func() {
-		_ = cmd.Wait()
+		if err := cmd.Wait(); err != nil {
+			if exiterr, ok := err.(*exec.ExitError); ok {
+				if exiterr.ExitCode() == upgrade.DAEMON_EXIT_CODE {
+					logs.Warnf("exit code %d daemon exit", upgrade.DAEMON_EXIT_CODE)
+					systemutil.ExitProcess(upgrade.DAEMON_EXIT_CODE)
+				}
+			}
+		}
 	}()
 
 	return cmd.Process, nil

src/agent/src/cmd/daemon/main_win.go

@@ -72,9 +72,7 @@ func main() {
 		systemutil.ExitProcess(1)
 	}
 
-	logs.Info("GOOS=%s, GOARCH=%s", runtime.GOOS, runtime.GOARCH)
-
-	runtime.GOMAXPROCS(4)
+	logs.Infof("GOOS=%s, GOARCH=%s", runtime.GOOS, runtime.GOARCH)
 
 	workDir := systemutil.GetExecutableDir()
 	err = os.Chdir(workDir)

src/agent/src/cmd/installer/main.go

@@ -32,7 +32,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/config"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/installer"
@@ -55,8 +54,6 @@ func main() {
 		systemutil.ExitProcess(1)
 	}
 
-	runtime.GOMAXPROCS(4)
-
 	defer func() {
 		if err := recover(); err != nil {
 			logs.Error("panic: ", err)

src/agent/src/cmd/upgrader/main.go

@@ -32,7 +32,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/config"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/logs"
@@ -53,8 +52,6 @@ func main() {
 		systemutil.ExitProcess(1)
 	}
 
-	runtime.GOMAXPROCS(4)
-
 	defer func() {
 		if err := recover(); err != nil {
 			logs.Error("panic: ", err)

src/agent/src/pkg/agent/agent.go

@@ -33,6 +33,7 @@ import (
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/cron"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/heartbeat"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/i18n"
+	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/imagedebug"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/job"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/logs"
 	"github.com/TencentBlueKing/bk-ci/src/agent/src/pkg/pipeline"
@@ -64,6 +65,10 @@ func Run(isDebug bool) {
 
 	// 定期清理
 	go cron.CleanJob()
+	go cron.CleanDebugContainer()
+
+	// 登录调试任务
+	go imagedebug.DoPullAndDebug()
 
 	job.DoPollAndBuild()
 }

src/agent/src/pkg/api/ErrorEnum.go

@@ -24,6 +24,7 @@ const (
 	DockerChmodInitshError
 	DockerCredGetError
 	DockerDockerOptions
+	DockerImageDebugError
 )
 
 type ErrorTypes string
@@ -120,6 +121,11 @@ var (
 		Code:    DockerChmodInitshError,
 		Message: "docker校验并修改启动脚本权限失败",
 	}
+	DockerImageDebugErrorEnum = &ErrorEnum{
+		Type:    User,
+		Code:    DockerImageDebugError,
+		Message: "启动docker登录调试失败",
+	}
 )
 
 func (t *ThirdPartyBuildInfo) ToFinish(

src/agent/src/pkg/api/api.go

@@ -152,6 +152,29 @@ func DownloadAgentInstallBatchZip(saveFile string) error {
 	return httputil.DownloadAgentInstallScript(url, config.GAgentConfig.GetAuthHeaderMap(), saveFile)
 }
 
+func PullDockerDebugTask() (*httputil.AgentResult, error) {
+	url := buildUrl("/ms/dispatch/api/buildAgent/agent/thirdPartyAgent/docker/startupDebug")
+	return httputil.NewHttpClient().Get(url).SetHeaders(config.GAgentConfig.GetAuthHeaderMap()).Execute().IntoAgentResult()
+}
+
+func FinishDockerDebug(imageDebug *ImageDebug, success bool, debugUrl string, error *Error) (*httputil.DevopsResult, error) {
+	url := buildUrl("/ms/dispatch/api/buildAgent/agent/thirdPartyAgent/docker/startupDebug")
+	body := &ImageDebugFinish{
+		ProjectId:  imageDebug.ProjectId,
+		DebugId:    imageDebug.DebugId,
+		PipelineId: imageDebug.PipelineId,
+		DebugUrl:   debugUrl,
+		Success:    success,
+		Error:      error,
+	}
+	return httputil.NewHttpClient().Post(url).Body(body).SetHeaders(config.GAgentConfig.GetAuthHeaderMap()).Execute().IntoDevopsResult()
+}
+
+func FetchDockerDebugStatus(debugId int64) (*httputil.DevopsResult, error) {
+	url := buildUrl(fmt.Sprintf("/ms/dispatch/api/buildAgent/agent/thirdPartyAgent/docker/debug/status?debugId=%d", debugId))
+	return httputil.NewHttpClient().Get(url).SetHeaders(config.GAgentConfig.GetAuthHeaderMap()).Execute().IntoDevopsResult()
+}
+
 // AuthHeaderDevopsBuildId log需要的buildId的header
 const (
 	AuthHeaderDevopsBuildId = "X-DEVOPS-BUILD-ID"

src/agent/src/pkg/api/type.go

@@ -56,12 +56,12 @@ const (
 )
 
 type ThirdPartyDockerBuildInfo struct {
-	AgentId         string         `json:"agentId"`
-	SecretKey       string         `json:"secretKey"`
-	Image           string         `json:"image"`
-	Credential      *Credential    `json:"credential"`
-	Options         *DockerOptions `json:"options"`
-	ImagePullPolicy string         `json:"imagePullPolicy"`
+	AgentId         string        `json:"agentId"`
+	SecretKey       string        `json:"secretKey"`
+	Image           string        `json:"image"`
+	Credential      Credential    `json:"credential"`
+	Options         DockerOptions `json:"options"`
+	ImagePullPolicy string        `json:"imagePullPolicy"`
 }
 
 type ImagePullPolicyEnum string
@@ -202,3 +202,25 @@ type LogMessage struct {
 	ExecuteCount *int    `json:"executeCount"`
 	SubTag       *string `json:"subTag"`
 }
+
+type ImageDebug struct {
+	ProjectId   string        `json:"projectId"`
+	BuildId     string        `json:"buildId"`
+	VmSeqId     string        `json:"vmSeqId"`
+	Workspace   string        `json:"workspace"`
+	PipelineId  string        `json:"pipelineId"`
+	DebugUserId string        `json:"debugUserId"`
+	DebugId     int64         `json:"debugId"`
+	Image       string        `json:"image"`
+	Credential  Credential    `json:"credential"`
+	Options     DockerOptions `json:"options"`
+}
+
+type ImageDebugFinish struct {
+	ProjectId  string `json:"projectId"`
+	DebugId    int64  `json:"debugId"`
+	PipelineId string `json:"pipelineId"`
+	DebugUrl   string `json:"debugUrl"`
+	Success    bool   `json:"success"`
+	Error      *Error `json:"error"`
+}

src/agent/src/pkg/collector/collector.go

@@ -239,14 +239,14 @@ func DoAgentCollect() {
 		logFile,
 	)
 	if err != nil {
-		logs.Error("init telegraf agent failed: %v", err)
+		logs.Errorf("init telegraf agent failed: %v", err)
 		return
 	}
 
 	for {
 		logs.Info("launch telegraf agent")
 		if err = tAgent.Run(context.Background()); err != nil {
-			logs.Error("telegraf agent exit: %v", err)
+			logs.Errorf("telegraf agent exit: %v", err)
 		}
 		time.Sleep(telegrafRelaunchTime)
 	}