feat(region,host): post overlay for container volume mount

cmd/climc/shell/compute/containers.go

@@ -46,6 +46,7 @@ func init() {
 	cmd.Perform("exec-sync", new(options.ContainerExecSyncOptions))
 	cmd.BatchPerform("set-resources-limit", new(options.ContainerSetResourcesLimitOptions))
 	cmd.Perform("commit", new(options.ContainerCommitOptions))
+	cmd.Perform("add-volume-mount-post-overlay", new(options.ContainerAddVolumeMountPostOverlayOptions))
 
 	type UpdateSpecOptions struct {
 		ID string `help:"ID or name of server" json:"-"`

pkg/apis/compute/container.go

@@ -251,3 +251,8 @@ type ContainerResourcesSetInput struct {
 	apis.ContainerResources
 	DisableLimitCheck bool `json:"disable_limit_check"`
 }
+
+type ContainerMountVolumeAddPostOverlayInput struct {
+	Index       int                                         `json:"index"`
+	PostOverlay []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
+}

pkg/apis/container.go

@@ -226,14 +226,24 @@ func (o ContainerVolumeMountDiskOverlay) IsValid() error {
 	return nil
 }
 
+type ContainerVolumeMountDiskPostOverlay struct {
+	// 宿主机底层目录
+	HostLowerDir []string `json:"host_lower_dir"`
+	// 合并后要挂载到容器的目录
+	ContainerTargetDir string `json:"container_target_dir"`
+}
+
 type ContainerVolumeMountDisk struct {
-	Index           *int                             `json:"index,omitempty"`
-	Id              string                           `json:"id"`
-	SubDirectory    string                           `json:"sub_directory"`
-	StorageSizeFile string                           `json:"storage_size_file"`
-	Overlay         *ContainerVolumeMountDiskOverlay `json:"overlay"`
+	Index           *int   `json:"index,omitempty"`
+	Id              string `json:"id"`
+	SubDirectory    string `json:"sub_directory"`
+	StorageSizeFile string `json:"storage_size_file"`
+	// lower overlay 设置，disk 的 volume 会作为 upper，最终 merged 的目录会传给容器
+	Overlay *ContainerVolumeMountDiskOverlay `json:"overlay"`
 	// case insensitive feature is incompatible with overlayfs
 	CaseInsensitivePaths []string `json:"case_insensitive_paths"`
+	// 当 disk volume 挂载完后，需要 overlay 的目录设置
+	PostOverlay []*ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
 }
 
 type ContainerVolumeMountHostPathType string

pkg/apis/host/container.go

@@ -21,13 +21,14 @@ import (
 )
 
 type ContainerVolumeMountDisk struct {
-	Index                *int                                  `json:"index,omitempty"`
-	Id                   string                                `json:"id"`
-	TemplateId           string                                `json:"template_id"`
-	SubDirectory         string                                `json:"sub_directory"`
-	StorageSizeFile      string                                `json:"storage_size_file"`
-	Overlay              *apis.ContainerVolumeMountDiskOverlay `json:"overlay"`
-	CaseInsensitivePaths []string                              `json:"case_insensitive_paths"`
+	Index                *int                                        `json:"index,omitempty"`
+	Id                   string                                      `json:"id"`
+	TemplateId           string                                      `json:"template_id"`
+	SubDirectory         string                                      `json:"sub_directory"`
+	StorageSizeFile      string                                      `json:"storage_size_file"`
+	Overlay              *apis.ContainerVolumeMountDiskOverlay       `json:"overlay"`
+	CaseInsensitivePaths []string                                    `json:"case_insensitive_paths"`
+	PostOverlay          []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"`
 }
 
 type ContainerVolumeMountCephFS struct {

pkg/compute/container_drivers/volume_mount/disk.go

@@ -135,6 +135,9 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre
 	if err := d.validateOverlay(ctx, userCred, vm, &diskObj); err != nil {
 		return nil, errors.Wrapf(err, "validate overlay")
 	}
+	if err := d.ValidatePostOverlay(vm); err != nil {
+		return nil, errors.Wrap(err, "validate post overlay")
+	}
 	return vm, nil
 }
 
@@ -184,6 +187,36 @@ func (d disk) validateOverlay(ctx context.Context, userCred mcclient.TokenCreden
 	return nil
 }
 
+func (d disk) ValidatePostOverlay(vm *apis.ContainerVolumeMount) error {
+	if len(vm.Disk.PostOverlay) == 0 {
+		return nil
+	}
+	ovs := vm.Disk.PostOverlay
+	var duplicateCtrDir string
+	for _, ov := range ovs {
+		if len(ov.HostLowerDir) == 0 {
+			return httperrors.NewNotEmptyError("host_lower_dir is required")
+		}
+		for i, hld := range ov.HostLowerDir {
+			if len(hld) == 0 {
+				return httperrors.NewNotEmptyError("host_lower_dir %d is empty", i)
+			}
+		}
+		if len(ov.ContainerTargetDir) == 0 {
+			return httperrors.NewNotEmptyError("container_target_dir is required")
+		}
+		if ov.ContainerTargetDir == duplicateCtrDir {
+			return httperrors.NewDuplicateNameError("container_target_dir", ov.ContainerTargetDir)
+		}
+		duplicateCtrDir = ov.ContainerTargetDir
+	}
+	if vm.Propagation == "" {
+		// 设置默认 propagation 为 rslave
+		vm.Propagation = apis.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER
+	}
+	return nil
+}
+
 type diskOverlayDir struct{}
 
 func newDiskOverlayDir() iDiskOverlay {

pkg/compute/guestdrivers/pod.go

@@ -518,6 +518,11 @@ func (p *SPodDriver) RequestSetContainerResourcesLimit(ctx context.Context, user
 	return p.requestContainerSyncAction(ctx, userCred, container, "set-resources-limit", jsonutils.Marshal(limit))
 }
 
+func (p *SPodDriver) RequestAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, c *models.SContainer, input *api.ContainerMountVolumeAddPostOverlayInput) error {
+	_, err := p.requestContainerSyncAction(ctx, userCred, c, "add-volume-mount-post-overlay", jsonutils.Marshal(input))
+	return err
+}
+
 func (p *SPodDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error {
 	// clean disk records in DB
 	return guest.DeleteAllDisksInDB(ctx, userCred)

pkg/compute/models/containers.go

@@ -443,6 +443,7 @@ func (vm *ContainerVolumeMountRelation) toHostDiskMount(disk *apis.ContainerVolu
 		StorageSizeFile:      disk.StorageSizeFile,
 		Overlay:              disk.Overlay,
 		CaseInsensitivePaths: disk.CaseInsensitivePaths,
+		PostOverlay:          disk.PostOverlay,
 	}
 	return ret, nil
 }
@@ -987,3 +988,55 @@ func (c *SContainer) StartCommit(ctx context.Context, userCred mcclient.TokenCre
 	}
 	return task.ScheduleRun(nil)
 }
+
+func (c *SContainer) PerformAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, input *api.ContainerMountVolumeAddPostOverlayInput) (jsonutils.JSONObject, error) {
+	if !api.ContainerExitedStatus.Has(c.Status) && !api.ContainerRunningStatus.Has(c.Status) {
+		return nil, httperrors.NewInvalidStatusError("can't add post overlay on status %s", c.Status)
+	}
+	if input.Index >= len(c.Spec.VolumeMounts) {
+		return nil, httperrors.NewInputParameterError("index %d out of volume_mount size %d", input.Index, len(c.Spec.VolumeMounts))
+	}
+	vm := new(apis.ContainerVolumeMount)
+	curVm := c.Spec.VolumeMounts[input.Index]
+	if err := jsonutils.Marshal(curVm).Unmarshal(vm); err != nil {
+		return nil, errors.Wrap(err, "use json unmarshal to new volume mount")
+	}
+	if vm.Type != apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK {
+		return nil, httperrors.NewInputParameterError("invalid volume mount type %s", vm.Type)
+	}
+
+	for _, ov := range input.PostOverlay {
+		isExist := c.isPostOverlayExist(vm, ov)
+		if isExist {
+			return nil, httperrors.NewInputParameterError("post overlay %s already exists", ov.ContainerTargetDir)
+		}
+	}
+
+	if api.ContainerRunningStatus.Has(c.Status) {
+		// invoke host to mount post overlay
+		if err := c.GetPodDriver().RequestAddVolumeMountPostOverlay(ctx, userCred, c, input); err != nil {
+			return nil, errors.Wrap(err, "request add post overlay")
+		}
+	}
+
+	if vm.Disk.PostOverlay == nil {
+		vm.Disk.PostOverlay = []*apis.ContainerVolumeMountDiskPostOverlay{}
+	}
+	if _, err := db.Update(c, func() error {
+		vm.Disk.PostOverlay = append(vm.Disk.PostOverlay, input.PostOverlay...)
+		c.Spec.VolumeMounts[input.Index] = vm
+		return nil
+	}); err != nil {
+		return nil, errors.Wrapf(err, "update spec.volume_mounts[%d]", input.Index)
+	}
+	return nil, nil
+}
+
+func (c *SContainer) isPostOverlayExist(vm *apis.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) bool {
+	for _, cov := range vm.Disk.PostOverlay {
+		if ov.ContainerTargetDir == cov.ContainerTargetDir {
+			return true
+		}
+	}
+	return false
+}

pkg/compute/models/pod_driver.go

@@ -44,5 +44,6 @@ type IPodDriver interface {
 	RequestCommitContainer(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
 	RequestSaveVolumeMountImage(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
 	RequestExecSyncContainer(ctx context.Context, userCred mcclient.TokenCredential, ctr *SContainer, input *compute.ContainerExecSyncInput) (jsonutils.JSONObject, error)
-	RequestSetContainerResourcesLimit(ctx context.Context, cred mcclient.TokenCredential, c *SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error)
+	RequestSetContainerResourcesLimit(ctx context.Context, userCred mcclient.TokenCredential, c *SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error)
+	RequestAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, c *SContainer, input *compute.ContainerMountVolumeAddPostOverlayInput) error
 }

pkg/hostman/container/volume_mount/cephfs.go

@@ -37,7 +37,9 @@ func (h cephFS) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMou
 	if err != nil {
 		return err
 	}
-	procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output()
+	if err := EnsureDir(dir); err != nil {
+		return errors.Wrap(err, "EnsureDir")
+	}
 	options := fmt.Sprintf("name=%s,secret=%s", vm.CephFS.Name, vm.CephFS.Secret)
 	if vm.ReadOnly {
 		options += ",ro"