支持拉取容器认证镜像和相关问题修复 (#21394)

* feat(keystone,region,host): pull authed container image

* fix(region,host): record starting container failed message

cmd/climc/shell/identity/credentials.go

@@ -21,14 +21,15 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/util/printutils"
 
+	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/identity"
 )
 
 func init() {
 	type CredentialListOptions struct {
 		Scope      string `help:"scope" choices:"project|domain|system"`
-		Type       string `help:"credential type" choices:"totp|recovery_secret|aksk|enc_key"`
+		Type       string `help:"credential type" choices:"totp|recovery_secret|aksk|enc_key|container_image"`
 		User       string `help:"filter by user"`
 		UserDomain string `help:"the domain of user"`
 	}
@@ -362,4 +363,31 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	type CredentialContainerImageOptions struct {
+		NAME          string `help:"container image credential name"`
+		USER          string `help:"container image credential user"`
+		PASSWORD      string `help:"container image credential password"`
+		Project       string `help:"Project"`
+		ProjectDomain string `help:"domain of user"`
+	}
+	R(&CredentialContainerImageOptions{}, "credential-create-container-image", "Create container image crendential", func(s *mcclient.ClientSession, args *CredentialContainerImageOptions) error {
+		var pid string
+		var err error
+		if len(args.Project) > 0 {
+			pid, err = modules.Projects.FetchId(s, args.Project, args.ProjectDomain)
+			if err != nil {
+				return err
+			}
+		}
+		obj, err := modules.Credentials.CreateContainerImageSecret(s, pid, args.NAME, &api.CredentialContainerImageBlob{
+			Username: args.USER,
+			Password: args.PASSWORD,
+		})
+		if err != nil {
+			return err
+		}
+		printObject(obj)
+		return nil
+	})
 }

pkg/apis/container.go

@@ -64,6 +64,8 @@ type ContainerSpec struct {
 	Image string `json:"image"`
 	// Image pull policy
 	ImagePullPolicy ImagePullPolicy `json:"image_pull_policy"`
+	// Image credential id
+	ImageCredentialId string `json:"image_credential_id"`
 	// Command to execute (i.e., entrypoint for docker)
 	Command []string `json:"command"`
 	// Args for the Command (i.e. command for docker)

pkg/apis/host/container.go

@@ -58,8 +58,9 @@ type ContainerVolumeMount struct {
 
 type ContainerSpec struct {
 	apis.ContainerSpec
-	VolumeMounts []*ContainerVolumeMount `json:"volume_mounts"`
-	Devices      []*ContainerDevice      `json:"devices"`
+	ImageCredentialToken string                  `json:"image_credential_token"`
+	VolumeMounts         []*ContainerVolumeMount `json:"volume_mounts"`
+	Devices              []*ContainerDevice      `json:"devices"`
 }
 
 type ContainerDevice struct {

pkg/apis/identity/aksk.go

@@ -26,6 +26,7 @@ const (
 	RECOVERY_SECRETS_TYPE = "recovery_secret"
 	OIDC_CREDENTIAL_TYPE  = "oidc"
 	ENCRYPT_KEY_TYPE      = "enc_key"
+	CONTAINER_IMAGE_TYPE  = "container_image"
 )
 
 type SAccessKeySecretBlob struct {

pkg/apis/identity/credential.go

@@ -50,3 +50,15 @@ type CredentialCreateInput struct {
 	// Ignore
 	KeyHash string `json:"key_hash"`
 }
+
+type CredentialContainerImageBlob struct {
+	Username      string `json:"username"`
+	Password      string `json:"password"`
+	Auth          string `json:"auth"`
+	ServerAddress string `json:"server_address,omitempty"`
+	// IdentityToken is used to authenticate the user and get
+	// an access token for the registry.
+	IdentityToken string `json:"identity_token,omitempty"`
+	// RegistryToken is a bearer token to be sent to a registry
+	RegistryToken string `json:"registry_token,omitempty"`
+}

pkg/cloudcommon/db/statusbase.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -105,7 +106,13 @@ func statusBaseSetStatus(ctx context.Context, model IStatusBaseModel, userCred m
 		}
 		OpsLog.LogEvent(model, ACT_UPDATE_STATUS, notes, userCred)
 		success := true
-		if strings.Contains(status, "fail") || status == apis.STATUS_UNKNOWN || status == api.CLOUD_PROVIDER_DISCONNECTED {
+		isFail := false
+		for _, sub := range []string{"fail", "crash"} {
+			if strings.Contains(status, sub) {
+				isFail = true
+			}
+		}
+		if isFail || sets.NewString(apis.STATUS_UNKNOWN, api.CLOUD_PROVIDER_DISCONNECTED, api.CONTAINER_STATUS_CRASH_LOOP_BACK_OFF).Has(status) {
 			success = false
 		}
 		logclient.AddSimpleActionLog(model, logclient.ACT_UPDATE_STATUS, notes, userCred, success)

pkg/compute/models/containers.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -31,13 +32,16 @@ import (
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	identityapi "yunion.io/x/onecloud/pkg/apis/identity"
 	imageapi "yunion.io/x/onecloud/pkg/apis/image"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	identitymod "yunion.io/x/onecloud/pkg/mcclient/modules/identity"
 	kubemod "yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
 )
 
@@ -74,7 +78,7 @@ type SContainer struct {
 	Spec *api.ContainerSpec `length:"long" create:"required" list:"user" update:"user"`
 
 	// 启动时间
-	StartedAt time.Time `nullable:"false" created_at:"false" index:"true" get:"user" list:"user" json:"started_at"`
+	StartedAt time.Time `nullable:"true" created_at:"false" index:"true" get:"user" list:"user" json:"started_at"`
 	// 上次退出时间
 	LastFinishedAt time.Time `nullable:"true" created_at:"false" index:"true" get:"user" list:"user" json:"last_finished_at"`
 
@@ -158,6 +162,11 @@ func (m *SContainerManager) ValidateSpec(ctx context.Context, userCred mcclient.
 	if !sets.NewString(apis.ImagePullPolicyAlways, apis.ImagePullPolicyIfNotPresent).Has(string(spec.ImagePullPolicy)) {
 		return httperrors.NewInputParameterError("invalid image_pull_policy %s", spec.ImagePullPolicy)
 	}
+	if spec.ImageCredentialId != "" {
+		if _, err := m.GetImageCredential(ctx, userCred, spec.ImageCredentialId); err != nil {
+			return errors.Wrapf(err, "get image credential by id: %s", spec.ImageCredentialId)
+		}
+	}
 
 	if pod != nil {
 		if err := m.ValidateSpecVolumeMounts(ctx, userCred, pod, spec); err != nil {
@@ -541,6 +550,67 @@ func (c *SContainer) StartDeleteTask(ctx context.Context, userCred mcclient.Toke
 	return task.ScheduleRun(nil)
 }
 
+func (m *SContainerManager) GetImageCredential(ctx context.Context, userCred mcclient.TokenCredential, id string) (*apis.ContainerPullImageAuthConfig, error) {
+	s := auth.GetSession(ctx, userCred, options.Options.Region)
+	ret, err := identitymod.Credentials.GetById(s, id, nil)
+	if err != nil {
+		if errors.Cause(err) == errors.ErrNotFound || strings.Contains(err.Error(), "NotFound") {
+			ret, err = identitymod.Credentials.GetByName(s, id, nil)
+			if err != nil {
+				return nil, errors.Wrapf(err, "get credential by id or name of %s", id)
+			}
+		}
+		return nil, errors.Wrap(err, "get credentials by id")
+	}
+	credType, _ := ret.GetString("type")
+	if credType != identityapi.CONTAINER_IMAGE_TYPE {
+		return nil, httperrors.NewNotSupportedError("unsupported credential type %s", credType)
+	}
+	blobStr, err := ret.GetString("blob")
+	if err != nil {
+		return nil, errors.Wrap(err, "get blob")
+	}
+	obj, err := jsonutils.ParseString(blobStr)
+	if err != nil {
+		return nil, errors.Wrapf(err, "json parse string: %s", blobStr)
+	}
+	blob := new(identityapi.CredentialContainerImageBlob)
+	if err := obj.Unmarshal(blob); err != nil {
+		return nil, errors.Wrap(err, "unmarshal blob")
+	}
+	out := &apis.ContainerPullImageAuthConfig{
+		Username:      blob.Username,
+		Password:      blob.Password,
+		Auth:          blob.Auth,
+		ServerAddress: blob.ServerAddress,
+		IdentityToken: blob.IdentityToken,
+		RegistryToken: blob.RegistryToken,
+	}
+	return out, nil
+}
+
+func (c *SContainer) GetImageCredential(ctx context.Context, userCred mcclient.TokenCredential) (*apis.ContainerPullImageAuthConfig, error) {
+	if c.Spec.ImageCredentialId == "" {
+		return nil, errors.Wrap(errors.ErrEmpty, "image_credential_id is empty")
+	}
+	return GetContainerManager().GetImageCredential(ctx, userCred, c.Spec.ImageCredentialId)
+}
+
+func (c *SContainer) GetHostPullImageInput(ctx context.Context, userCred mcclient.TokenCredential) (*hostapi.ContainerPullImageInput, error) {
+	input := &hostapi.ContainerPullImageInput{
+		Image:      c.Spec.Image,
+		PullPolicy: c.Spec.ImagePullPolicy,
+	}
+	if c.Spec.ImageCredentialId != "" {
+		cred, err := c.GetImageCredential(ctx, userCred)
+		if err != nil {
+			return nil, errors.Wrapf(err, "GetImageCredential %s", c.Spec.ImageCredentialId)
+		}
+		input.Auth = cred
+	}
+	return input, nil
+}
+
 func (c *SContainer) StartPullImageTask(ctx context.Context, userCred mcclient.TokenCredential, input *hostapi.ContainerPullImageInput, parentTaskId string) error {
 	c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_PULLING_IMAGE, "")
 	task, err := taskman.TaskManager.NewTask(ctx, "ContainerPullImageTask", c, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), parentTaskId, "", nil)
@@ -592,6 +662,11 @@ func (c *SContainer) ToHostContainerSpec(ctx context.Context, userCred mcclient.
 		VolumeMounts:  mounts,
 		Devices:       ctrDevs,
 	}
+	pullInput, err := c.GetHostPullImageInput(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetHostPullImageInput")
+	}
+	hSpec.ImageCredentialToken = base64.StdEncoding.EncodeToString([]byte(jsonutils.Marshal(pullInput.Auth).String()))
 	return hSpec, nil
 }
 
@@ -776,6 +851,10 @@ func (c *SContainer) PerformStatus(ctx context.Context, userCred mcclient.TokenC
 		if input.RestartCount > 0 {
 			c.RestartCount = input.RestartCount
 		}
+		if api.ContainerRunningStatus.Has(input.Status) {
+			// 当容器状态是运行时 restart_count 重新计数
+			c.RestartCount = 0
+		}
 		if input.StartedAt != nil {
 			c.StartedAt = *input.StartedAt
 		}

pkg/compute/tasks/container_create_task.go

@@ -21,7 +21,6 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
-	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
@@ -61,9 +60,10 @@ func (t *ContainerCreateTask) OnInit(ctx context.Context, obj db.IStandaloneMode
 
 func (t *ContainerCreateTask) startPullImage(ctx context.Context, container *models.SContainer) {
 	t.SetStage("OnImagePulled", nil)
-	input := &hostapi.ContainerPullImageInput{
-		Image:      container.Spec.Image,
-		PullPolicy: container.Spec.ImagePullPolicy,
+	input, err := container.GetHostPullImageInput(ctx, t.GetUserCred())
+	if err != nil {
+		t.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
+		return
 	}
 	if err := container.StartPullImageTask(ctx, t.GetUserCred(), input, t.GetTaskId()); err != nil {
 		t.SetStageFailed(ctx, jsonutils.NewString(err.Error()))

pkg/compute/tasks/pod_start_task.go

@@ -41,7 +41,6 @@ func (t *PodStartTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body
 }
 
 func (t *PodStartTask) OnPodStarted(ctx context.Context, pod *models.SGuest, _ jsonutils.JSONObject) {
-	t.SetStage("OnContainerStarted", nil)
 	pod.SetStatus(ctx, t.GetUserCred(), api.POD_STATUS_STARTING_CONTAINER, "")
 	ctrs, err := models.GetContainerManager().GetContainersByPod(pod.GetId())
 	if err != nil {
@@ -50,7 +49,7 @@ func (t *PodStartTask) OnPodStarted(ctx context.Context, pod *models.SGuest, _ j
 	}
 	isAllStarted := true
 	for i := range ctrs {
-		if ctrs[i].GetStatus() != api.CONTAINER_STATUS_RUNNING {
+		if !api.ContainerRunningStatus.Has(ctrs[i].GetStatus()) {
 			isAllStarted = false
 			ctrs[i].StartStartTask(ctx, t.GetUserCred(), t.GetTaskId())
 		}