feat(region,host): post overlay for container volume mount

cmd/climc/shell/compute/containers.go

@@ -46,6 +46,8 @@ func init() {
 	cmd.Perform("exec-sync", new(options.ContainerExecSyncOptions))
 	cmd.BatchPerform("set-resources-limit", new(options.ContainerSetResourcesLimitOptions))
 	cmd.Perform("commit", new(options.ContainerCommitOptions))
+	cmd.Perform("add-volume-mount-post-overlay", new(options.ContainerAddVolumeMountPostOverlayOptions))
+	cmd.Perform("remove-volume-mount-post-overlay", new(options.ContainerRemoveVolumeMountPostOverlayOptions))
 
 	type UpdateSpecOptions struct {
 		ID string `help:"ID or name of server" json:"-"`

pkg/apis/compute/container.go

@@ -82,6 +82,11 @@ const (
 	CONTAINER_STATUS_PROBING      = "probing"
 	CONTAINER_STATUS_PROBE_FAILED = "probe_failed"
 	CONTAINER_STATUS_NET_FAILED   = "net_failed"
+	// post overlay
+	CONTAINER_STATUS_ADD_POST_OVERLY           = "adding_post_overly"
+	CONTAINER_STATUS_ADD_POST_OVERLY_FAILED    = "add_post_overly_failed"
+	CONTAINER_STATUS_REMOVE_POST_OVERLY        = "removing_post_overly"
+	CONTAINER_STATUS_REMOVE_POST_OVERLY_FAILED = "remove_post_overly_failed"
 )
 
 var (
@@ -251,3 +256,14 @@ type ContainerResourcesSetInput struct {
 	apis.ContainerResources
 	DisableLimitCheck bool `json:"disable_limit_check"`
 }
+
+type ContainerVolumeMountAddPostOverlayInput struct {
+	Index       int                                         `json:"index"`
+	PostOverlay []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
+}
+
+type ContainerVolumeMountRemovePostOverlayInput struct {
+	Index       int                                         `json:"index"`
+	PostOverlay []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
+	ClearLayers bool                                        `json:"clear_layers"`
+}

pkg/apis/container.go

@@ -226,14 +226,24 @@ func (o ContainerVolumeMountDiskOverlay) IsValid() error {
 	return nil
 }
 
+type ContainerVolumeMountDiskPostOverlay struct {
+	// 宿主机底层目录
+	HostLowerDir []string `json:"host_lower_dir"`
+	// 合并后要挂载到容器的目录
+	ContainerTargetDir string `json:"container_target_dir"`
+}
+
 type ContainerVolumeMountDisk struct {
-	Index           *int                             `json:"index,omitempty"`
-	Id              string                           `json:"id"`
-	SubDirectory    string                           `json:"sub_directory"`
-	StorageSizeFile string                           `json:"storage_size_file"`
-	Overlay         *ContainerVolumeMountDiskOverlay `json:"overlay"`
+	Index           *int   `json:"index,omitempty"`
+	Id              string `json:"id"`
+	SubDirectory    string `json:"sub_directory"`
+	StorageSizeFile string `json:"storage_size_file"`
+	// lower overlay 设置，disk 的 volume 会作为 upper，最终 merged 的目录会传给容器
+	Overlay *ContainerVolumeMountDiskOverlay `json:"overlay"`
 	// case insensitive feature is incompatible with overlayfs
 	CaseInsensitivePaths []string `json:"case_insensitive_paths"`
+	// 当 disk volume 挂载完后，需要 overlay 的目录设置
+	PostOverlay []*ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
 }
 
 type ContainerVolumeMountHostPathType string

pkg/apis/host/container.go

@@ -21,13 +21,14 @@ import (
 )
 
 type ContainerVolumeMountDisk struct {
-	Index                *int                                  `json:"index,omitempty"`
-	Id                   string                                `json:"id"`
-	TemplateId           string                                `json:"template_id"`
-	SubDirectory         string                                `json:"sub_directory"`
-	StorageSizeFile      string                                `json:"storage_size_file"`
-	Overlay              *apis.ContainerVolumeMountDiskOverlay `json:"overlay"`
-	CaseInsensitivePaths []string                              `json:"case_insensitive_paths"`
+	Index                *int                                        `json:"index,omitempty"`
+	Id                   string                                      `json:"id"`
+	TemplateId           string                                      `json:"template_id"`
+	SubDirectory         string                                      `json:"sub_directory"`
+	StorageSizeFile      string                                      `json:"storage_size_file"`
+	Overlay              *apis.ContainerVolumeMountDiskOverlay       `json:"overlay"`
+	CaseInsensitivePaths []string                                    `json:"case_insensitive_paths"`
+	PostOverlay          []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
 }
 
 type ContainerVolumeMountCephFS struct {

pkg/compute/container_drivers/volume_mount/disk.go

@@ -135,6 +135,9 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre
 	if err := d.validateOverlay(ctx, userCred, vm, &diskObj); err != nil {
 		return nil, errors.Wrapf(err, "validate overlay")
 	}
+	if err := d.ValidatePostOverlay(vm); err != nil {
+		return nil, errors.Wrap(err, "validate post overlay")
+	}
 	return vm, nil
 }
 
@@ -184,6 +187,36 @@ func (d disk) validateOverlay(ctx context.Context, userCred mcclient.TokenCreden
 	return nil
 }
 
+func (d disk) ValidatePostOverlay(vm *apis.ContainerVolumeMount) error {
+	if len(vm.Disk.PostOverlay) == 0 {
+		return nil
+	}
+	ovs := vm.Disk.PostOverlay
+	var duplicateCtrDir string
+	for _, ov := range ovs {
+		if len(ov.HostLowerDir) == 0 {
+			return httperrors.NewNotEmptyError("host_lower_dir is required")
+		}
+		for i, hld := range ov.HostLowerDir {
+			if len(hld) == 0 {
+				return httperrors.NewNotEmptyError("host_lower_dir %d is empty", i)
+			}
+		}
+		if len(ov.ContainerTargetDir) == 0 {
+			return httperrors.NewNotEmptyError("container_target_dir is required")
+		}
+		if ov.ContainerTargetDir == duplicateCtrDir {
+			return httperrors.NewDuplicateNameError("container_target_dir", ov.ContainerTargetDir)
+		}
+		duplicateCtrDir = ov.ContainerTargetDir
+	}
+	if vm.Propagation == "" {
+		// 设置默认 propagation 为 rslave
+		vm.Propagation = apis.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER
+	}
+	return nil
+}
+
 type diskOverlayDir struct{}
 
 func newDiskOverlayDir() iDiskOverlay {

pkg/compute/guestdrivers/pod.go

@@ -472,6 +472,14 @@ func (p *SPodDriver) RequestPullContainerImage(ctx context.Context, userCred mcc
 	return p.performContainerAction(ctx, userCred, task, "pull-image", task.GetParams())
 }
 
+func (p *SPodDriver) RequestAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error {
+	return p.performContainerAction(ctx, userCred, task, "add-volume-mount-post-overlay", task.GetParams())
+}
+
+func (p *SPodDriver) RequestRemoveVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error {
+	return p.performContainerAction(ctx, userCred, task, "remove-volume-mount-post-overlay", task.GetParams())
+}
+
 type responder struct {
 	errorMessage string
 }

pkg/compute/models/containers.go

@@ -443,6 +443,7 @@ func (vm *ContainerVolumeMountRelation) toHostDiskMount(disk *apis.ContainerVolu
 		StorageSizeFile:      disk.StorageSizeFile,
 		Overlay:              disk.Overlay,
 		CaseInsensitivePaths: disk.CaseInsensitivePaths,
+		PostOverlay:          disk.PostOverlay,
 	}
 	return ret, nil
 }
@@ -987,3 +988,133 @@ func (c *SContainer) StartCommit(ctx context.Context, userCred mcclient.TokenCre
 	}
 	return task.ScheduleRun(nil)
 }
+
+func (c *SContainer) isPostOverlayExist(vm *apis.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) bool {
+	for _, cov := range vm.Disk.PostOverlay {
+		if ov.ContainerTargetDir == cov.ContainerTargetDir {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *SContainer) validateVolumeMountPostOverlayAction(action string, index int, ovs []*apis.ContainerVolumeMountDiskPostOverlay) (*apis.ContainerVolumeMount, error) {
+	if !api.ContainerExitedStatus.Has(c.Status) && !api.ContainerRunningStatus.Has(c.Status) {
+		return nil, httperrors.NewInvalidStatusError("can't %s post overlay on status %s", action, c.Status)
+	}
+	if index >= len(c.Spec.VolumeMounts) {
+		return nil, httperrors.NewInputParameterError("index %d out of volume_mount size %d", index, len(c.Spec.VolumeMounts))
+	}
+	vm := new(apis.ContainerVolumeMount)
+	curVm := c.Spec.VolumeMounts[index]
+	if err := jsonutils.Marshal(curVm).Unmarshal(vm); err != nil {
+		return nil, errors.Wrap(err, "use json unmarshal to new volume mount")
+	}
+	if vm.Type != apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK {
+		return nil, httperrors.NewInputParameterError("invalid volume mount type %s", vm.Type)
+	}
+	return vm, nil
+}
+
+func (c *SContainer) GetVolumeMountCopy(index int) (*apis.ContainerVolumeMount, error) {
+	if index >= len(c.Spec.VolumeMounts) {
+		return nil, httperrors.NewInputParameterError("index %d out of volume_mount size %d", index, len(c.Spec.VolumeMounts))
+	}
+	vm := new(apis.ContainerVolumeMount)
+	curVm := c.Spec.VolumeMounts[index]
+	if err := jsonutils.Marshal(curVm).Unmarshal(vm); err != nil {
+		return nil, errors.Wrap(err, "use json unmarshal to new volume mount")
+	}
+	return vm, nil
+}
+
+func (c *SContainer) getPostOverlayVolumeMount(
+	index int,
+	updateF func(mount *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error),
+) (*apis.ContainerVolumeMount, error) {
+	vm, err := c.GetVolumeMountCopy(index)
+	if err != nil {
+		return nil, err
+	}
+	return updateF(vm)
+}
+
+func (c *SContainer) GetAddPostOverlayVolumeMount(index int, ovs []*apis.ContainerVolumeMountDiskPostOverlay) (*apis.ContainerVolumeMount, error) {
+	return c.getPostOverlayVolumeMount(index, func(vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) {
+		if vm.Disk.PostOverlay == nil {
+			vm.Disk.PostOverlay = []*apis.ContainerVolumeMountDiskPostOverlay{}
+		}
+		vm.Disk.PostOverlay = append(vm.Disk.PostOverlay, ovs...)
+		return vm, nil
+	})
+}
+
+func (c *SContainer) GetRemovePostOverlayVolumeMount(index int, ovs []*apis.ContainerVolumeMountDiskPostOverlay) (*apis.ContainerVolumeMount, error) {
+	return c.getPostOverlayVolumeMount(index, func(vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) {
+		// remove post overlay
+		for _, ov := range ovs {
+			vm.Disk = c.removePostOverlay(vm.Disk, ov)
+		}
+		return vm, nil
+	})
+}
+
+func (c *SContainer) PerformAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, input *api.ContainerVolumeMountAddPostOverlayInput) (jsonutils.JSONObject, error) {
+	vm, err := c.validateVolumeMountPostOverlayAction("add", input.Index, input.PostOverlay)
+	if err != nil {
+		return nil, err
+	}
+	for _, ov := range input.PostOverlay {
+		isExist := c.isPostOverlayExist(vm, ov)
+		if isExist {
+			return nil, httperrors.NewInputParameterError("post overlay %s already exists", ov.ContainerTargetDir)
+		}
+	}
+	return nil, c.StartAddVolumeMountPostOverlayTask(ctx, userCred, input, "")
+}
+
+func (c *SContainer) StartAddVolumeMountPostOverlayTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ContainerVolumeMountAddPostOverlayInput, parentTaskId string) error {
+	c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_ADD_POST_OVERLY, "")
+	task, err := taskman.TaskManager.NewTask(ctx, "ContainerAddVolumeMountPostOverlayTask", c, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), parentTaskId, "", nil)
+	if err != nil {
+		return errors.Wrap(err, "New ContainerAddVolumeMountPostOverlayTask")
+	}
+	return task.ScheduleRun(nil)
+}
+
+func (c *SContainer) StartRemoveVolumeMountPostOverlayTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ContainerVolumeMountRemovePostOverlayInput, parentTaskId string) error {
+	c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_REMOVE_POST_OVERLY, "")
+	task, err := taskman.TaskManager.NewTask(ctx, "ContainerRemoveVolumeMountPostOverlayTask", c, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), parentTaskId, "", nil)
+	if err != nil {
+		return errors.Wrap(err, "New ContainerRemoveVolumeMountPostOverlayTask")
+	}
+	return task.ScheduleRun(nil)
+}
+
+func (c *SContainer) PerformRemoveVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, input *api.ContainerVolumeMountRemovePostOverlayInput) (jsonutils.JSONObject, error) {
+	vm, err := c.validateVolumeMountPostOverlayAction("remove", input.Index, input.PostOverlay)
+	if err != nil {
+		return nil, err
+	}
+	if len(vm.Disk.PostOverlay) == 0 {
+		return nil, httperrors.NewInputParameterError("no post overlay")
+	}
+	for _, ov := range input.PostOverlay {
+		isExist := c.isPostOverlayExist(vm, ov)
+		if !isExist {
+			return nil, httperrors.NewInputParameterError("post overlay %s not exists", ov.ContainerTargetDir)
+		}
+	}
+	return nil, c.StartRemoveVolumeMountPostOverlayTask(ctx, userCred, input, "")
+}
+
+func (c *SContainer) removePostOverlay(vmd *apis.ContainerVolumeMountDisk, ov *apis.ContainerVolumeMountDiskPostOverlay) *apis.ContainerVolumeMountDisk {
+	curOvs := vmd.PostOverlay
+	for i, cov := range curOvs {
+		if cov.ContainerTargetDir == ov.ContainerTargetDir {
+			curOvs = append(curOvs[:i], curOvs[i+1:]...)
+		}
+	}
+	vmd.PostOverlay = curOvs
+	return vmd
+}

pkg/compute/models/pod_driver.go

@@ -44,5 +44,8 @@ type IPodDriver interface {
 	RequestCommitContainer(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
 	RequestSaveVolumeMountImage(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
 	RequestExecSyncContainer(ctx context.Context, userCred mcclient.TokenCredential, ctr *SContainer, input *compute.ContainerExecSyncInput) (jsonutils.JSONObject, error)
-	RequestSetContainerResourcesLimit(ctx context.Context, cred mcclient.TokenCredential, c *SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error)
+	RequestSetContainerResourcesLimit(ctx context.Context, userCred mcclient.TokenCredential, c *SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error)
+
+	RequestAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
+	RequestRemoveVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
 }