From b6c1e23ac5b82e7570195159534c995dda2d4671 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 27 Nov 2024 16:45:52 +0800 Subject: [PATCH] feat(region,host): post overlay for container volume mount --- cmd/climc/shell/compute/containers.go | 1 + pkg/apis/compute/container.go | 5 + pkg/apis/container.go | 20 ++- pkg/apis/host/container.go | 15 +- .../container_drivers/volume_mount/disk.go | 33 ++++ pkg/compute/guestdrivers/pod.go | 5 + pkg/compute/models/containers.go | 53 ++++++ pkg/compute/models/pod_driver.go | 3 +- pkg/hostman/container/volume_mount/cephfs.go | 4 +- .../container/volume_mount/{ => disk}/disk.go | 142 ++++++---------- .../container/volume_mount/disk/doc.go | 1 + .../container/volume_mount/disk/overlay.go | 102 ++++++++++++ .../volume_mount/disk/post_overlay.go | 152 ++++++++++++++++++ pkg/hostman/container/volume_mount/helper.go | 8 + pkg/hostman/container/volume_mount/text.go | 5 +- pkg/hostman/guestman/pod.go | 16 +- .../guestman/podhandlers/podhandlers.go | 9 ++ pkg/mcclient/options/compute/containers.go | 26 +++ pkg/util/mountutils/mount.go | 32 +++- 19 files changed, 512 insertions(+), 120 deletions(-) rename pkg/hostman/container/volume_mount/{ => disk}/disk.go (64%) create mode 100644 pkg/hostman/container/volume_mount/disk/doc.go create mode 100644 pkg/hostman/container/volume_mount/disk/overlay.go create mode 100644 pkg/hostman/container/volume_mount/disk/post_overlay.go diff --git a/cmd/climc/shell/compute/containers.go b/cmd/climc/shell/compute/containers.go index 5dadad723c9..c4a3e06273c 100644 --- a/cmd/climc/shell/compute/containers.go +++ b/cmd/climc/shell/compute/containers.go @@ -46,6 +46,7 @@ func init() { cmd.Perform("exec-sync", new(options.ContainerExecSyncOptions)) cmd.BatchPerform("set-resources-limit", new(options.ContainerSetResourcesLimitOptions)) cmd.Perform("commit", new(options.ContainerCommitOptions)) + cmd.Perform("add-volume-mount-post-overlay", new(options.ContainerAddVolumeMountPostOverlayOptions)) type UpdateSpecOptions struct { ID string `help:"ID or name of server" json:"-"` diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index 32c0622d3e0..52574086749 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -251,3 +251,8 @@ type ContainerResourcesSetInput struct { apis.ContainerResources DisableLimitCheck bool `json:"disable_limit_check"` } + +type ContainerMountVolumeAddPostOverlayInput struct { + Index int `json:"index"` + PostOverlay []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"` +} diff --git a/pkg/apis/container.go b/pkg/apis/container.go index e32b683f2d0..e96c67f40a3 100644 --- a/pkg/apis/container.go +++ b/pkg/apis/container.go @@ -226,14 +226,24 @@ func (o ContainerVolumeMountDiskOverlay) IsValid() error { return nil } +type ContainerVolumeMountDiskPostOverlay struct { + // 宿主机底层目录 + HostLowerDir []string `json:"host_lower_dir"` + // 合并后要挂载到容器的目录 + ContainerTargetDir string `json:"container_target_dir"` +} + type ContainerVolumeMountDisk struct { - Index *int `json:"index,omitempty"` - Id string `json:"id"` - SubDirectory string `json:"sub_directory"` - StorageSizeFile string `json:"storage_size_file"` - Overlay *ContainerVolumeMountDiskOverlay `json:"overlay"` + Index *int `json:"index,omitempty"` + Id string `json:"id"` + SubDirectory string `json:"sub_directory"` + StorageSizeFile string `json:"storage_size_file"` + // lower overlay 设置，disk 的 volume 会作为 upper，最终 merged 的目录会传给容器 + Overlay *ContainerVolumeMountDiskOverlay `json:"overlay"` // case insensitive feature is incompatible with overlayfs CaseInsensitivePaths []string `json:"case_insensitive_paths"` + // 当 disk volume 挂载完后，需要 overlay 的目录设置 + PostOverlay []*ContainerVolumeMountDiskPostOverlay `json:"post_overlay"` } type ContainerVolumeMountHostPathType string diff --git a/pkg/apis/host/container.go b/pkg/apis/host/container.go index c26cfaf81e6..a8e013ab7c9 100644 --- a/pkg/apis/host/container.go +++ b/pkg/apis/host/container.go @@ -21,13 +21,14 @@ import ( ) type ContainerVolumeMountDisk struct { - Index *int `json:"index,omitempty"` - Id string `json:"id"` - TemplateId string `json:"template_id"` - SubDirectory string `json:"sub_directory"` - StorageSizeFile string `json:"storage_size_file"` - Overlay *apis.ContainerVolumeMountDiskOverlay `json:"overlay"` - CaseInsensitivePaths []string `json:"case_insensitive_paths"` + Index *int `json:"index,omitempty"` + Id string `json:"id"` + TemplateId string `json:"template_id"` + SubDirectory string `json:"sub_directory"` + StorageSizeFile string `json:"storage_size_file"` + Overlay *apis.ContainerVolumeMountDiskOverlay `json:"overlay"` + CaseInsensitivePaths []string `json:"case_insensitive_paths"` + PostOverlay []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"` } type ContainerVolumeMountCephFS struct { diff --git a/pkg/compute/container_drivers/volume_mount/disk.go b/pkg/compute/container_drivers/volume_mount/disk.go index f13a656deaa..fa745c453f4 100644 --- a/pkg/compute/container_drivers/volume_mount/disk.go +++ b/pkg/compute/container_drivers/volume_mount/disk.go @@ -135,6 +135,9 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre if err := d.validateOverlay(ctx, userCred, vm, &diskObj); err != nil { return nil, errors.Wrapf(err, "validate overlay") } + if err := d.ValidatePostOverlay(vm); err != nil { + return nil, errors.Wrap(err, "validate post overlay") + } return vm, nil } @@ -184,6 +187,36 @@ func (d disk) validateOverlay(ctx context.Context, userCred mcclient.TokenCreden return nil } +func (d disk) ValidatePostOverlay(vm *apis.ContainerVolumeMount) error { + if len(vm.Disk.PostOverlay) == 0 { + return nil + } + ovs := vm.Disk.PostOverlay + var duplicateCtrDir string + for _, ov := range ovs { + if len(ov.HostLowerDir) == 0 { + return httperrors.NewNotEmptyError("host_lower_dir is required") + } + for i, hld := range ov.HostLowerDir { + if len(hld) == 0 { + return httperrors.NewNotEmptyError("host_lower_dir %d is empty", i) + } + } + if len(ov.ContainerTargetDir) == 0 { + return httperrors.NewNotEmptyError("container_target_dir is required") + } + if ov.ContainerTargetDir == duplicateCtrDir { + return httperrors.NewDuplicateNameError("container_target_dir", ov.ContainerTargetDir) + } + duplicateCtrDir = ov.ContainerTargetDir + } + if vm.Propagation == "" { + // 设置默认 propagation 为 rslave + vm.Propagation = apis.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER + } + return nil +} + type diskOverlayDir struct{} func newDiskOverlayDir() iDiskOverlay { diff --git a/pkg/compute/guestdrivers/pod.go b/pkg/compute/guestdrivers/pod.go index 42908322d04..e7db8122838 100644 --- a/pkg/compute/guestdrivers/pod.go +++ b/pkg/compute/guestdrivers/pod.go @@ -518,6 +518,11 @@ func (p *SPodDriver) RequestSetContainerResourcesLimit(ctx context.Context, user return p.requestContainerSyncAction(ctx, userCred, container, "set-resources-limit", jsonutils.Marshal(limit)) } +func (p *SPodDriver) RequestAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, c *models.SContainer, input *api.ContainerMountVolumeAddPostOverlayInput) error { + _, err := p.requestContainerSyncAction(ctx, userCred, c, "add-volume-mount-post-overlay", jsonutils.Marshal(input)) + return err +} + func (p *SPodDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error { // clean disk records in DB return guest.DeleteAllDisksInDB(ctx, userCred) diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go index 11d58354fa7..c8457a6ccf7 100644 --- a/pkg/compute/models/containers.go +++ b/pkg/compute/models/containers.go @@ -443,6 +443,7 @@ func (vm *ContainerVolumeMountRelation) toHostDiskMount(disk *apis.ContainerVolu StorageSizeFile: disk.StorageSizeFile, Overlay: disk.Overlay, CaseInsensitivePaths: disk.CaseInsensitivePaths, + PostOverlay: disk.PostOverlay, } return ret, nil } @@ -987,3 +988,55 @@ func (c *SContainer) StartCommit(ctx context.Context, userCred mcclient.TokenCre } return task.ScheduleRun(nil) } + +func (c *SContainer) PerformAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, input *api.ContainerMountVolumeAddPostOverlayInput) (jsonutils.JSONObject, error) { + if !api.ContainerExitedStatus.Has(c.Status) && !api.ContainerRunningStatus.Has(c.Status) { + return nil, httperrors.NewInvalidStatusError("can't add post overlay on status %s", c.Status) + } + if input.Index >= len(c.Spec.VolumeMounts) { + return nil, httperrors.NewInputParameterError("index %d out of volume_mount size %d", input.Index, len(c.Spec.VolumeMounts)) + } + vm := new(apis.ContainerVolumeMount) + curVm := c.Spec.VolumeMounts[input.Index] + if err := jsonutils.Marshal(curVm).Unmarshal(vm); err != nil { + return nil, errors.Wrap(err, "use json unmarshal to new volume mount") + } + if vm.Type != apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK { + return nil, httperrors.NewInputParameterError("invalid volume mount type %s", vm.Type) + } + + for _, ov := range input.PostOverlay { + isExist := c.isPostOverlayExist(vm, ov) + if isExist { + return nil, httperrors.NewInputParameterError("post overlay %s already exists", ov.ContainerTargetDir) + } + } + + if api.ContainerRunningStatus.Has(c.Status) { + // invoke host to mount post overlay + if err := c.GetPodDriver().RequestAddVolumeMountPostOverlay(ctx, userCred, c, input); err != nil { + return nil, errors.Wrap(err, "request add post overlay") + } + } + + if vm.Disk.PostOverlay == nil { + vm.Disk.PostOverlay = []*apis.ContainerVolumeMountDiskPostOverlay{} + } + if _, err := db.Update(c, func() error { + vm.Disk.PostOverlay = append(vm.Disk.PostOverlay, input.PostOverlay...) + c.Spec.VolumeMounts[input.Index] = vm + return nil + }); err != nil { + return nil, errors.Wrapf(err, "update spec.volume_mounts[%d]", input.Index) + } + return nil, nil +} + +func (c *SContainer) isPostOverlayExist(vm *apis.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) bool { + for _, cov := range vm.Disk.PostOverlay { + if ov.ContainerTargetDir == cov.ContainerTargetDir { + return true + } + } + return false +} diff --git a/pkg/compute/models/pod_driver.go b/pkg/compute/models/pod_driver.go index 3fff890cc09..dd2ad1ea79d 100644 --- a/pkg/compute/models/pod_driver.go +++ b/pkg/compute/models/pod_driver.go @@ -44,5 +44,6 @@ type IPodDriver interface { RequestCommitContainer(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error RequestSaveVolumeMountImage(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error RequestExecSyncContainer(ctx context.Context, userCred mcclient.TokenCredential, ctr *SContainer, input *compute.ContainerExecSyncInput) (jsonutils.JSONObject, error) - RequestSetContainerResourcesLimit(ctx context.Context, cred mcclient.TokenCredential, c *SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error) + RequestSetContainerResourcesLimit(ctx context.Context, userCred mcclient.TokenCredential, c *SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error) + RequestAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, c *SContainer, input *compute.ContainerMountVolumeAddPostOverlayInput) error } diff --git a/pkg/hostman/container/volume_mount/cephfs.go b/pkg/hostman/container/volume_mount/cephfs.go index dd87c08cc14..41438b5875c 100644 --- a/pkg/hostman/container/volume_mount/cephfs.go +++ b/pkg/hostman/container/volume_mount/cephfs.go @@ -37,7 +37,9 @@ func (h cephFS) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMou if err != nil { return err } - procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + if err := EnsureDir(dir); err != nil { + return errors.Wrap(err, "EnsureDir") + } options := fmt.Sprintf("name=%s,secret=%s", vm.CephFS.Name, vm.CephFS.Secret) if vm.ReadOnly { options += ",ro" diff --git a/pkg/hostman/container/volume_mount/disk.go b/pkg/hostman/container/volume_mount/disk/disk.go similarity index 64% rename from pkg/hostman/container/volume_mount/disk.go rename to pkg/hostman/container/volume_mount/disk/disk.go index 3db5e3373b9..1dc5bb4eb03 100644 --- a/pkg/hostman/container/volume_mount/disk.go +++ b/pkg/hostman/container/volume_mount/disk/disk.go @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package volume_mount +package disk import ( "context" "fmt" "path/filepath" - "strings" "yunion.io/x/pkg/errors" @@ -27,6 +26,7 @@ import ( hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/container/storage" container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/httperrors" @@ -34,19 +34,21 @@ import ( ) func init() { - RegisterDriver(newDisk()) + volume_mount.RegisterDriver(newDisk()) } -type iDiskOverlay interface { - mount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error - unmount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error +type IVolumeMountDisk interface { + volume_mount.IUsageVolumeMount + + MountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error + UnmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error } type disk struct { overlayDrivers map[apis.ContainerDiskOverlayType]iDiskOverlay } -func newDisk() IVolumeMount { +func newDisk() IVolumeMountDisk { return &disk{ overlayDrivers: map[apis.ContainerDiskOverlayType]iDiskOverlay{ apis.CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY: newDiskOverlayDir(), @@ -59,12 +61,21 @@ func (d disk) GetType() apis.ContainerVolumeMountType { return apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK } -func (d disk) getRuntimeMountHostPath(pod IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) { +func (d disk) getHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) { diskInput := vm.Disk if diskInput == nil { return "", httperrors.NewNotEmptyError("disk is nil") } hostPath := filepath.Join(pod.GetVolumesDir(), diskInput.Id) + return hostPath, nil +} + +func (d disk) getRuntimeMountHostPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) { + hostPath, err := d.getHostDiskRootPath(pod, vm) + if err != nil { + return "", errors.Wrap(err, "get host disk root path") + } + diskInput := vm.Disk if diskInput.SubDirectory != "" { return filepath.Join(hostPath, diskInput.SubDirectory), nil } @@ -74,7 +85,7 @@ func (d disk) getRuntimeMountHostPath(pod IPodInfo, vm *hostapi.ContainerVolumeM return hostPath, nil } -func (d disk) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) (string, error) { +func (d disk) GetRuntimeMountHostPath(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) (string, error) { hostPath, err := d.getRuntimeMountHostPath(pod, vm) if err != nil { return "", errors.Wrap(err, "get runtime mount host_path") @@ -86,7 +97,7 @@ func (d disk) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *hostapi.Co return d.getOverlayMergedDir(pod, ctrId, vm, hostPath), nil } -func (d disk) getPodDisk(pod IPodInfo, vm *hostapi.ContainerVolumeMount) (storageman.IDisk, *desc.SGuestDisk, error) { +func (d disk) getPodDisk(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (storageman.IDisk, *desc.SGuestDisk, error) { var disk *desc.SGuestDisk = nil disks := pod.GetDisks() volDisk := vm.Disk @@ -111,7 +122,7 @@ func (d disk) getPodDisk(pod IPodInfo, vm *hostapi.ContainerVolumeMount) (storag return iDisk, disk, nil } -func (d disk) getDiskStorageDriver(pod IPodInfo, vm *hostapi.ContainerVolumeMount) (storage.IContainerStorage, error) { +func (d disk) getDiskStorageDriver(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (storage.IContainerStorage, error) { iDisk, _, err := d.getPodDisk(pod, vm) if err != nil { return nil, errors.Wrap(err, "get pod disk interface") @@ -123,18 +134,6 @@ func (d disk) getDiskStorageDriver(pod IPodInfo, vm *hostapi.ContainerVolumeMoun return drv, nil } -func (d disk) getOverlayDir(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, upperDir string, suffix string) string { - return filepath.Join(pod.GetVolumesOverlayDir(), vm.Disk.Id, ctrId, fmt.Sprintf("%s-%s", filepath.Base(upperDir), suffix)) -} - -func (d disk) getOverlayWorkDir(upperDir string) string { - return fmt.Sprintf("%s-work", upperDir) -} - -func (d disk) getOverlayMergedDir(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, upperDir string) string { - return d.getOverlayDir(pod, ctrId, vm, upperDir, "merged") -} - func (d disk) setDirCaseInsensitive(dir string) error { out, err := procutils.NewRemoteCommandAsFarAsPossible("chattr", "+F", dir).Output() if err != nil { @@ -143,7 +142,11 @@ func (d disk) setDirCaseInsensitive(dir string) error { return nil } -func (d disk) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { +func (d disk) newPostOverlay() iDiskPostOverlay { + return newDiskPostOverlay(d) +} + +func (d disk) Mount(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { iDisk, gd, err := d.getPodDisk(pod, vm) if err != nil { return errors.Wrap(err, "get pod disk interface") @@ -169,15 +172,13 @@ func (d disk) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount vmDisk := vm.Disk if vmDisk.SubDirectory != "" { subDir := filepath.Join(mntPoint, vmDisk.SubDirectory) - out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", subDir).Output() - if err != nil { - return errors.Wrapf(err, "make sub_directory %s inside %s: %s", vmDisk.SubDirectory, mntPoint, out) + if err := volume_mount.EnsureDir(subDir); err != nil { + return errors.Wrapf(err, "make sub_directory %s inside %s", vmDisk.SubDirectory, mntPoint) } for _, cd := range vmDisk.CaseInsensitivePaths { cdp := filepath.Join(subDir, cd) - out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", cdp).Output() - if err != nil { - return errors.Wrapf(err, "make %s inside %s: %s", cdp, vmDisk.SubDirectory, out) + if err := volume_mount.EnsureDir(cdp); err != nil { + return errors.Wrapf(err, "make %s inside %s", cdp, vmDisk.SubDirectory) } if err := d.setDirCaseInsensitive(cdp); err != nil { return errors.Wrapf(err, "enable case_insensitive %s", cdp) @@ -198,6 +199,11 @@ func (d disk) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount return errors.Wrapf(err, "mount container %s overlay dir: %#v", ctrId, vmDisk.Overlay) } } + if len(vmDisk.PostOverlay) != 0 { + if err := d.MountPostOverlays(pod, ctrId, vm, vmDisk.PostOverlay); err != nil { + return errors.Wrap(err, "mount post overlay dirs") + } + } return nil } @@ -216,7 +222,7 @@ func (d disk) createStorageSizeFile(iDisk storageman.IDisk, mntPoint string, inp return nil } -func (d disk) Unmount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { +func (d disk) Unmount(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { iDisk, _, err := d.getPodDisk(pod, vm) if err != nil { return errors.Wrap(err, "get pod disk interface") @@ -225,6 +231,11 @@ func (d disk) Unmount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMou if err != nil { return errors.Wrap(err, "get disk storage driver") } + if len(vm.Disk.PostOverlay) != 0 { + if err := d.UnmountPostOverlays(pod, ctrId, vm, vm.Disk.PostOverlay); err != nil { + return errors.Wrap(err, "mount post overlay dirs") + } + } if vm.Disk.Overlay != nil { if err := d.unmoutOverlay(pod, ctrId, vm); err != nil { return errors.Wrapf(err, "umount overlay") @@ -250,19 +261,19 @@ func (d disk) getOverlayDriver(ov *apis.ContainerVolumeMountDiskOverlay) iDiskOv return d.overlayDrivers[ov.GetType()] } -func (d disk) unmoutOverlay(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { +func (d disk) unmoutOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { return d.getOverlayDriver(vm.Disk.Overlay).unmount(d, pod, ctrId, vm) } -func (d disk) mountOverlay(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { +func (d disk) mountOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { return d.getOverlayDriver(vm.Disk.Overlay).mount(d, pod, ctrId, vm) } func (d disk) doTemplateOverlayAction( ctx context.Context, - pod IPodInfo, ctrId string, + pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, - ovAction func(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error) error { + ovAction func(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error) error { templateId := vm.Disk.TemplateId input := computeapi.CacheImageInput{ ImageId: templateId, @@ -287,65 +298,6 @@ func (d disk) doTemplateOverlayAction( return nil } -func (d disk) InjectUsageTags(usage *ContainerVolumeMountUsage, vol *hostapi.ContainerVolumeMount) { +func (d disk) InjectUsageTags(usage *volume_mount.ContainerVolumeMountUsage, vol *hostapi.ContainerVolumeMount) { usage.Tags["disk_id"] = vol.Disk.Id } - -type diskOverlayDir struct{} - -func newDiskOverlayDir() iDiskOverlay { - return &diskOverlayDir{} -} - -func (dod diskOverlayDir) mount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { - vmDisk := vm.Disk - lowerDir := vmDisk.Overlay.LowerDir - upperDir, err := d.getRuntimeMountHostPath(pod, vm) - if err != nil { - return errors.Wrap(err, "getRuntimeMountHostPath") - } - workDir := d.getOverlayWorkDir(upperDir) - mergedDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) - for _, dir := range []string{workDir, mergedDir} { - out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() - if err != nil { - return errors.Wrapf(err, "make directory %s: %s", dir, out) - } - } - - overlayArgs := []string{"-t", "overlay", "overlay", "-o", fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(lowerDir, ":"), upperDir, workDir), mergedDir} - if out, err := procutils.NewRemoteCommandAsFarAsPossible("mount", overlayArgs...).Output(); err != nil { - return errors.Wrapf(err, "mount %v: %s", overlayArgs, out) - } - - return nil -} - -func (dod diskOverlayDir) unmount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { - upperDir, err := d.getRuntimeMountHostPath(pod, vm) - if err != nil { - return errors.Wrap(err, "getRuntimeMountHostPath") - } - overlayDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) - return container_storage.Unmount(overlayDir) -} - -type diskOverlayImage struct{} - -func newDiskOverlayImage() iDiskOverlay { - return &diskOverlayImage{} -} - -func (di diskOverlayImage) mount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { - if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().mount); err != nil { - return errors.Wrapf(err, "mount template overlay") - } - return nil -} - -func (di diskOverlayImage) unmount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { - if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().unmount); err != nil { - return errors.Wrapf(err, "unmount template overlay") - } - return nil -} diff --git a/pkg/hostman/container/volume_mount/disk/doc.go b/pkg/hostman/container/volume_mount/disk/doc.go new file mode 100644 index 00000000000..c8cf275a58d --- /dev/null +++ b/pkg/hostman/container/volume_mount/disk/doc.go @@ -0,0 +1 @@ +package disk // import "yunion.io/x/onecloud/pkg/hostman/container/volume_mount/disk" diff --git a/pkg/hostman/container/volume_mount/disk/overlay.go b/pkg/hostman/container/volume_mount/disk/overlay.go new file mode 100644 index 00000000000..442d76c34cc --- /dev/null +++ b/pkg/hostman/container/volume_mount/disk/overlay.go @@ -0,0 +1,102 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package disk + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "yunion.io/x/pkg/errors" + + hostapi "yunion.io/x/onecloud/pkg/apis/host" + container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" + "yunion.io/x/onecloud/pkg/util/mountutils" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func (d disk) getOverlayDir(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, oType string, upperDir string) string { + baseDir := strings.TrimPrefix(upperDir, pod.GetVolumesDir()) + return filepath.Join(pod.GetVolumesOverlayDir(), vm.Disk.Id, ctrId, oType, baseDir) +} + +func (d disk) getOverlayWorkDir(upperDir string) string { + return fmt.Sprintf("%s-work", upperDir) +} + +func (d disk) getOverlayMergedDir(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, upperDir string) string { + return d.getOverlayDir(pod, ctrId, vm, "_merged_", upperDir) +} + +type iDiskOverlay interface { + mount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error + unmount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error +} + +type diskOverlayDir struct{} + +func newDiskOverlayDir() iDiskOverlay { + return &diskOverlayDir{} +} + +func (dod diskOverlayDir) mount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + vmDisk := vm.Disk + lowerDir := vmDisk.Overlay.LowerDir + upperDir, err := d.getRuntimeMountHostPath(pod, vm) + if err != nil { + return errors.Wrap(err, "getRuntimeMountHostPath") + } + workDir := d.getOverlayWorkDir(upperDir) + mergedDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) + for _, dir := range []string{workDir, mergedDir} { + out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + if err != nil { + return errors.Wrapf(err, "make directory %s: %s", dir, out) + } + } + + return mountutils.MountOverlay(lowerDir, upperDir, workDir, mergedDir) +} + +func (dod diskOverlayDir) unmount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + upperDir, err := d.getRuntimeMountHostPath(pod, vm) + if err != nil { + return errors.Wrap(err, "getRuntimeMountHostPath") + } + overlayDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) + return container_storage.Unmount(overlayDir) +} + +type diskOverlayImage struct{} + +func newDiskOverlayImage() iDiskOverlay { + return &diskOverlayImage{} +} + +func (di diskOverlayImage) mount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().mount); err != nil { + return errors.Wrapf(err, "mount template overlay") + } + return nil +} + +func (di diskOverlayImage) unmount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().unmount); err != nil { + return errors.Wrapf(err, "unmount template overlay") + } + return nil +} diff --git a/pkg/hostman/container/volume_mount/disk/post_overlay.go b/pkg/hostman/container/volume_mount/disk/post_overlay.go new file mode 100644 index 00000000000..8614fa5dc64 --- /dev/null +++ b/pkg/hostman/container/volume_mount/disk/post_overlay.go @@ -0,0 +1,152 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package disk + +import ( + "path/filepath" + "strings" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" + "yunion.io/x/onecloud/pkg/util/mountutils" +) + +const ( + POST_OVERLAY_PREFIX_WORK_DIR = "_post_overlay_work_" + POST_OVERLAY_PREFIX_UPPER_DIR = "_post_overlay_upper_" +) + +func (d disk) MountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error { + return d.newPostOverlay().mountPostOverlays(pod, ctrId, vm, ovs) +} + +func (d disk) UnmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error { + return d.newPostOverlay().unmountPostOverlays(pod, ctrId, vm, ovs) +} + +type iDiskPostOverlay interface { + mountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error + unmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error +} + +type diskPostOverlay struct { + disk disk +} + +func newDiskPostOverlay(d disk) iDiskPostOverlay { + return &diskPostOverlay{ + disk: d, + } +} + +func (d diskPostOverlay) mountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error { + for _, ov := range ovs { + if err := d.mountPostOverlay(pod, ctrId, vm, ov); err != nil { + return errors.Wrapf(err, "mount container %s post overlay dir: %#v", ctrId, ov) + } + } + return nil +} + +func (d diskPostOverlay) unmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error { + for _, ov := range ovs { + if err := d.unmountPostOverlay(pod, ctrId, vm, ov); err != nil { + return errors.Wrapf(err, "unmount container %s post overlay dir: %#v", ctrId, ov) + } + } + return nil +} + +func (d diskPostOverlay) getPostOverlayDirWithPrefix( + prefixDir string, + pod volume_mount.IPodInfo, ctrId string, + vm *hostapi.ContainerVolumeMount, + ov *apis.ContainerVolumeMountDiskPostOverlay, + ensure bool, +) (string, error) { + hostPath, err := d.disk.getHostDiskRootPath(pod, vm) + if err != nil { + return "", errors.Wrap(err, "get host disk root path") + } + workDir := filepath.Join(hostPath, prefixDir, ctrId, ov.ContainerTargetDir) + if ensure { + if err := volume_mount.EnsureDir(workDir); err != nil { + return "", errors.Wrapf(err, "make %s", workDir) + } + } + return workDir, nil +} + +func (d diskPostOverlay) getPostOverlayWorkDir( + pod volume_mount.IPodInfo, ctrId string, + vm *hostapi.ContainerVolumeMount, + ov *apis.ContainerVolumeMountDiskPostOverlay, + ensure bool, +) (string, error) { + return d.getPostOverlayDirWithPrefix(POST_OVERLAY_PREFIX_WORK_DIR, pod, ctrId, vm, ov, ensure) +} + +func (d diskPostOverlay) getPostOverlayUpperDir( + pod volume_mount.IPodInfo, ctrId string, + vm *hostapi.ContainerVolumeMount, + ov *apis.ContainerVolumeMountDiskPostOverlay, + ensure bool, +) (string, error) { + return d.getPostOverlayDirWithPrefix(POST_OVERLAY_PREFIX_UPPER_DIR, pod, ctrId, vm, ov, ensure) +} + +func (d diskPostOverlay) getPostOverlayMountpoint(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) (string, error) { + ctrMountHostPath, err := d.disk.GetRuntimeMountHostPath(pod, ctrId, vm) + if err != nil { + return "", errors.Wrap(err, "get runtime mount host_path") + } + // remove hostPath sub_directory path + ctrMountHostPath = strings.TrimSuffix(ctrMountHostPath, vm.Disk.SubDirectory) + mergedDir := filepath.Join(ctrMountHostPath, ov.ContainerTargetDir) + if err := volume_mount.EnsureDir(mergedDir); err != nil { + return "", errors.Wrap(err, "make merged mountpoint dir") + } + return mergedDir, nil +} + +func (d diskPostOverlay) mountPostOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { + upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) + } + + workDir, err := d.getPostOverlayWorkDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) + } + + mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) + if err != nil { + return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) + } + + return mountutils.MountOverlay(ov.HostLowerDir, upperDir, workDir, mergedDir) +} + +func (d diskPostOverlay) unmountPostOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { + mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) + if err != nil { + return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) + } + return mountutils.Unmount(mergedDir) +} diff --git a/pkg/hostman/container/volume_mount/helper.go b/pkg/hostman/container/volume_mount/helper.go index 6020f8a2887..a7070d3eb7d 100644 --- a/pkg/hostman/container/volume_mount/helper.go +++ b/pkg/hostman/container/volume_mount/helper.go @@ -9,6 +9,14 @@ import ( "yunion.io/x/onecloud/pkg/util/procutils" ) +func EnsureDir(dir string) error { + out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + if err != nil { + return errors.Wrapf(err, "mkdir -p %s: %s", dir, out) + } + return nil +} + func ChangeDirOwner(pod IPodInfo, drv IVolumeMount, ctrId string, vol *hostapi.ContainerVolumeMount) error { if vol.FsUser == nil && vol.FsGroup == nil { return errors.Errorf("specify fs_user or fs_group") diff --git a/pkg/hostman/container/volume_mount/text.go b/pkg/hostman/container/volume_mount/text.go index a94a34a5b67..f6935f520e2 100644 --- a/pkg/hostman/container/volume_mount/text.go +++ b/pkg/hostman/container/volume_mount/text.go @@ -48,9 +48,8 @@ func (t text) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *hostapi.Co if ti == nil { return "", httperrors.NewNotEmptyError("text is nil") } - out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", pod.GetVolumesDir()).Output() - if err != nil { - return "", errors.Wrapf(err, "mkdir %s: %s", pod.GetVolumesDir(), out) + if err := EnsureDir(pod.GetVolumesDir()); err != nil { + return "", errors.Wrapf(err, "mkdir %s", pod.GetVolumesDir()) } mntPath := filepath.Join(pod.GetVolumesDir(), fmt.Sprintf("%s-%s", ctrId, strings.ReplaceAll(vm.MountPath, "/", "_"))) if err := t.writeContent(ti, mntPath); err != nil { diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 6d8d8625351..2ad3dc64298 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -45,6 +45,8 @@ import ( proberesults "yunion.io/x/onecloud/pkg/hostman/container/prober/results" "yunion.io/x/onecloud/pkg/hostman/container/status" "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount/disk" + _ "yunion.io/x/onecloud/pkg/hostman/container/volume_mount/disk" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/hostman/guestman/pod/runtime" deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" @@ -126,6 +128,7 @@ type PodInstance interface { ContainerExecSync(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerExecSyncInput) (jsonutils.JSONObject, error) SetContainerResourceLimit(ctrId string, limit *apis.ContainerResources) (jsonutils.JSONObject, error) CommitContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerCommitInput) (jsonutils.JSONObject, error) + AddContainerVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerMountVolumeAddPostOverlayInput) error ReadLogs(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.PodLogOptions, stdout, stderr io.Writer) error @@ -650,7 +653,7 @@ func (s *sPodGuestInstance) GetVolumesDir() string { } func (s *sPodGuestInstance) GetVolumesOverlayDir() string { - return filepath.Join(s.GetVolumesDir(), "overlay") + return filepath.Join(s.GetVolumesDir(), "_overlay_") } func (s *sPodGuestInstance) GetDiskMountPoint(disk storageman.IDisk) string { @@ -2412,3 +2415,14 @@ func (s *sPodGuestInstance) CommitContainer(ctx context.Context, userCred mcclie "image_repository": imgRepo, }), nil } + +func (s *sPodGuestInstance) AddContainerVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerMountVolumeAddPostOverlayInput) error { + ctrSpec := s.GetContainerById(ctrId) + vol := ctrSpec.Spec.VolumeMounts[input.Index] + drv := volume_mount.GetDriver(vol.Type) + diskDrv, ok := drv.(disk.IVolumeMountDisk) + if !ok { + return errors.Errorf("invalid disk volume driver of %s", vol.Type) + } + return diskDrv.MountPostOverlays(s, ctrId, vol, input.PostOverlay) +} diff --git a/pkg/hostman/guestman/podhandlers/podhandlers.go b/pkg/hostman/guestman/podhandlers/podhandlers.go index 3402a8feecd..36ee4e3dd5a 100644 --- a/pkg/hostman/guestman/podhandlers/podhandlers.go +++ b/pkg/hostman/guestman/podhandlers/podhandlers.go @@ -152,6 +152,7 @@ func AddPodHandlers(prefix string, app *appsrv.Application) { syncWorker := appsrv.NewWorkerManager("container-sync-action-worker", 16, appsrv.DEFAULT_BACKLOG, false) app.AddHandler3(newContainerWorkerHandler("POST", fmt.Sprintf("%s/pods/%s/containers/%s/set-resources-limit", prefix, POD_ID, CONTAINER_ID), syncWorker, containerSyncActionHandler(containerSetResourcesLimit))) + app.AddHandler3(newContainerWorkerHandler("POST", fmt.Sprintf("%s/pods/%s/containers/%s/add-volume-mount-post-overlay", prefix, POD_ID, CONTAINER_ID), syncWorker, containerSyncActionHandler(containerAddVolumeMountPostOverlay))) } func pullImage(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { @@ -323,3 +324,11 @@ func commitContainer(ctx context.Context, userCred mcclient.TokenCredential, pod } return pod.CommitContainer(ctx, userCred, ctrId, input) } + +func containerAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, containerId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { + input := new(compute.ContainerMountVolumeAddPostOverlayInput) + if err := body.Unmarshal(input); err != nil { + return nil, errors.Wrap(err, "unmarshal to ContainerMountVolumeAddPostOverlayInput") + } + return nil, pod.AddContainerVolumeMountPostOverlay(ctx, userCred, containerId, input) +} diff --git a/pkg/mcclient/options/compute/containers.go b/pkg/mcclient/options/compute/containers.go index 301f250ab85..dec58d6472e 100644 --- a/pkg/mcclient/options/compute/containers.go +++ b/pkg/mcclient/options/compute/containers.go @@ -451,3 +451,29 @@ func (o *ContainerCommitOptions) Params() (jsonutils.JSONObject, error) { } return jsonutils.Marshal(input), nil } + +type ContainerAddVolumeMountPostOverlayOptions struct { + ServerIdOptions + INDEX int `help:"INDEX of volume mount"` + MountDesc []string `help:"Mount description, :" short-token:"m"` +} + +func (o *ContainerAddVolumeMountPostOverlayOptions) Params() (jsonutils.JSONObject, error) { + input := &computeapi.ContainerMountVolumeAddPostOverlayInput{ + Index: o.INDEX, + PostOverlay: make([]*apis.ContainerVolumeMountDiskPostOverlay, 0), + } + for _, md := range o.MountDesc { + segs := strings.Split(md, ":") + if len(segs) != 2 { + return nil, errors.Errorf("invalid mount description: %s", md) + } + lowerDir := segs[0] + containerTargetDir := segs[1] + input.PostOverlay = append(input.PostOverlay, &apis.ContainerVolumeMountDiskPostOverlay{ + HostLowerDir: []string{lowerDir}, + ContainerTargetDir: containerTargetDir, + }) + } + return jsonutils.Marshal(input), nil +} diff --git a/pkg/util/mountutils/mount.go b/pkg/util/mountutils/mount.go index cec8e10de5c..1d42655ab35 100644 --- a/pkg/util/mountutils/mount.go +++ b/pkg/util/mountutils/mount.go @@ -30,7 +30,7 @@ import ( "yunion.io/x/onecloud/pkg/util/procutils" ) -func Mount(devPath string, mountPoint string, fsType string) error { +func mountWrap(mountPoint string, action func() error) error { if !fileutils2.Exists(mountPoint) { output, err := procutils.NewCommand("mkdir", "-p", mountPoint).Output() if err != nil { @@ -41,12 +41,30 @@ func Mount(devPath string, mountPoint string, fsType string) error { log.Warningf("mountpoint %s is already mounted", mountPoint) return nil } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", "-t", fsType, devPath, mountPoint).Output(); err != nil { - return errors.Wrapf(err, "mount %s to %s with fs %s: %s", devPath, mountPoint, fsType, string(out)) - } - return nil + return action() +} + +func Mount(devPath string, mountPoint string, fsType string) error { + return mountWrap(mountPoint, func() error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", "-t", fsType, devPath, mountPoint).Output(); err != nil { + return errors.Wrapf(err, "mount %s to %s with fs %s: %s", devPath, mountPoint, fsType, string(out)) + } + return nil + }) +} + +func MountOverlay(lowerDir []string, upperDir string, workDir string, mergedDir string) error { + return mountWrap(mergedDir, func() error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + overlayArgs := []string{"-t", "overlay", "overlay", "-o", fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(lowerDir, ":"), upperDir, workDir), mergedDir} + if out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", overlayArgs...).Output(); err != nil { + return errors.Wrapf(err, "mount %v: %s", overlayArgs, out) + } + return nil + }) } func Unmount(mountPoint string) error {