diff --git a/pkg/apis/container.go b/pkg/apis/container.go index e96c67f40a3..1eb93ca6993 100644 --- a/pkg/apis/container.go +++ b/pkg/apis/container.go @@ -172,11 +172,13 @@ var ( ) type ContainerVolumeMount struct { - Type ContainerVolumeMountType `json:"type"` - Disk *ContainerVolumeMountDisk `json:"disk"` - HostPath *ContainerVolumeMountHostPath `json:"host_path"` - Text *ContainerVolumeMountText `json:"text"` - CephFS *ContainerVolumeMountCephFS `json:"ceph_fs"` + // 用于标识当前 pod volume mount 的唯一性 + UniqueName string `json:"unique_name"` + Type ContainerVolumeMountType `json:"type"` + Disk *ContainerVolumeMountDisk `json:"disk"` + HostPath *ContainerVolumeMountHostPath `json:"host_path"` + Text *ContainerVolumeMountText `json:"text"` + CephFS *ContainerVolumeMountCephFS `json:"ceph_fs"` // Mounted read-only if true, read-write otherwise (false or unspecified). ReadOnly bool `json:"read_only"` // Path within the container at which the volume should be mounted. Must diff --git a/pkg/apis/host/container.go b/pkg/apis/host/container.go index a8e013ab7c9..6c530336205 100644 --- a/pkg/apis/host/container.go +++ b/pkg/apis/host/container.go @@ -40,11 +40,13 @@ type ContainerVolumeMountCephFS struct { } type ContainerVolumeMount struct { - Type apis.ContainerVolumeMountType `json:"type"` - Disk *ContainerVolumeMountDisk `json:"disk"` - HostPath *apis.ContainerVolumeMountHostPath `json:"host_path"` - Text *apis.ContainerVolumeMountText `json:"text"` - CephFS *ContainerVolumeMountCephFS `json:"ceph_fs"` + // 用于标识当前 pod volume mount 的唯一性 + UniqueName string `json:"unique_name"` + Type apis.ContainerVolumeMountType `json:"type"` + Disk *ContainerVolumeMountDisk `json:"disk"` + HostPath *apis.ContainerVolumeMountHostPath `json:"host_path"` + Text *apis.ContainerVolumeMountText `json:"text"` + CephFS *ContainerVolumeMountCephFS `json:"ceph_fs"` // Mounted read-only if true, read-write otherwise (false or unspecified). ReadOnly bool `json:"read_only"` // Path within the container at which the volume should be mounted. Must diff --git a/pkg/compute/guestdrivers/pod.go b/pkg/compute/guestdrivers/pod.go index b6c820b7081..98ad32f742f 100644 --- a/pkg/compute/guestdrivers/pod.go +++ b/pkg/compute/guestdrivers/pod.go @@ -87,15 +87,26 @@ func (p *SPodDriver) ValidateCreateData(ctx context.Context, userCred mcclient.T return nil, errors.Wrap(err, "validate port mappings") }*/ - sameName := "" + ctrNames := sets.NewString() + volUniqNames := sets.NewString() for idx, ctr := range input.Pod.Containers { if err := p.validateContainerData(ctx, userCred, idx, input.Name, ctr, input); err != nil { return nil, errors.Wrapf(err, "data of %d container", idx) } - if ctr.Name == sameName { + if ctrNames.Has(ctr.Name) { return nil, httperrors.NewDuplicateNameError("same name %s of containers", ctr.Name) } - sameName = ctr.Name + ctrNames.Insert(ctr.Name) + for volIdx := range ctr.VolumeMounts { + vol := ctr.VolumeMounts[volIdx] + if vol.UniqueName != "" { + if volUniqNames.Has(vol.UniqueName) { + return nil, httperrors.NewDuplicateNameError("same volume unique name %s", fmt.Sprintf("container %s volume_mount %d %s", ctr.Name, volIdx, vol.UniqueName)) + } else { + volUniqNames.Insert(vol.UniqueName) + } + } + } } return input, nil diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go index d2dc9324d20..3cf0ca053d6 100644 --- a/pkg/compute/models/containers.go +++ b/pkg/compute/models/containers.go @@ -227,7 +227,24 @@ func (m *SContainerManager) ValidateSpecVolumeMounts(ctx context.Context, userCr if err != nil { return errors.Wrap(err, "GetVolumeMountRelations") } + + curCtrs, _ := m.GetContainersByPod(pod.GetId()) + volUniqNames := sets.NewString() + for idx := range curCtrs { + for _, vol := range curCtrs[idx].Spec.VolumeMounts { + if vol.UniqueName != "" { + volUniqNames.Insert(vol.UniqueName) + } + } + } for idx, vm := range spec.VolumeMounts { + if vm.UniqueName != "" { + if volUniqNames.Has(vm.UniqueName) { + return httperrors.NewDuplicateNameError("volume_mount unique_name %s", vm.UniqueName) + } else { + volUniqNames.Insert(vm.UniqueName) + } + } newVm, err := m.ValidateSpecVolumeMount(ctx, userCred, pod, vm) if err != nil { return errors.Wrapf(err, "validate volume mount %s", jsonutils.Marshal(vm)) @@ -478,6 +495,7 @@ func (vm *ContainerVolumeMountRelation) toCephFSMount(fs *apis.ContainerVolumeMo func (vm *ContainerVolumeMountRelation) ToHostMount(ctx context.Context, userCred mcclient.TokenCredential) (*hostapi.ContainerVolumeMount, error) { ret := &hostapi.ContainerVolumeMount{ + UniqueName: vm.VolumeMount.UniqueName, Type: vm.VolumeMount.Type, Disk: nil, CephFS: nil, diff --git a/pkg/hostman/container/volume_mount/disk/disk.go b/pkg/hostman/container/volume_mount/disk/disk.go index 8b05ba79cde..ddd14d85f2a 100644 --- a/pkg/hostman/container/volume_mount/disk/disk.go +++ b/pkg/hostman/container/volume_mount/disk/disk.go @@ -42,6 +42,11 @@ type IVolumeMountDisk interface { MountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error UnmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay, clearLayers bool) error + + GetHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) + + GetPostOverlayRootWorkDir(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error) + GetPostOverlayRootUpperDir(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error) } type disk struct { @@ -61,7 +66,7 @@ func (d disk) GetType() apis.ContainerVolumeMountType { return apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK } -func (d disk) getHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) { +func (d disk) GetHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) { diskInput := vm.Disk if diskInput == nil { return "", httperrors.NewNotEmptyError("disk is nil") @@ -71,7 +76,7 @@ func (d disk) getHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.Contain } func (d disk) getRuntimeMountHostPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) { - hostPath, err := d.getHostDiskRootPath(pod, vm) + hostPath, err := d.GetHostDiskRootPath(pod, vm) if err != nil { return "", errors.Wrap(err, "get host disk root path") } @@ -254,6 +259,9 @@ func (d disk) Unmount(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.Conta return errors.Wrapf(err, "DisconnectDisk %s %s", iDisk.GetPath(), mntPoint) } } + if err := volume_mount.RemoveDir(mntPoint); err != nil { + return errors.Wrapf(err, "remove dir %s", mntPoint) + } return nil } diff --git a/pkg/hostman/container/volume_mount/disk/post_overlay.go b/pkg/hostman/container/volume_mount/disk/post_overlay.go index 87f4a7a6562..ac0ebb7c3a7 100644 --- a/pkg/hostman/container/volume_mount/disk/post_overlay.go +++ b/pkg/hostman/container/volume_mount/disk/post_overlay.go @@ -39,6 +39,26 @@ func (d disk) UnmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *h return d.newPostOverlay().unmountPostOverlays(pod, ctrId, vm, ovs, clearLayers) } +func (d disk) getPostOverlayRootPrefixDir(prefixDir string, pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error) { + hostPath, err := d.GetHostDiskRootPath(pod, vm) + if err != nil { + return "", errors.Wrap(err, "get host disk root path") + } + uniqDir := ctrId + if vm.UniqueName != "" { + uniqDir = vm.UniqueName + } + return filepath.Join(hostPath, prefixDir, uniqDir), nil +} + +func (d disk) GetPostOverlayRootWorkDir(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error) { + return d.getPostOverlayRootPrefixDir(POST_OVERLAY_PREFIX_WORK_DIR, pod, vm, ctrId) +} + +func (d disk) GetPostOverlayRootUpperDir(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error) { + return d.getPostOverlayRootPrefixDir(POST_OVERLAY_PREFIX_UPPER_DIR, pod, vm, ctrId) +} + type iDiskPostOverlay interface { mountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error unmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay, clearLayers bool) error @@ -79,11 +99,12 @@ func (d diskPostOverlay) getPostOverlayDirWithPrefix( ov *apis.ContainerVolumeMountDiskPostOverlay, ensure bool, ) (string, error) { - hostPath, err := d.disk.getHostDiskRootPath(pod, vm) + rootPath, err := d.disk.getPostOverlayRootPrefixDir(prefixDir, pod, vm, ctrId) if err != nil { - return "", errors.Wrap(err, "get host disk root path") + return "", errors.Wrap(err, "get post overlay root path") } - workDir := filepath.Join(hostPath, prefixDir, ctrId, ov.ContainerTargetDir) + + workDir := filepath.Join(rootPath, ov.ContainerTargetDir) if ensure { if err := volume_mount.EnsureDir(workDir); err != nil { return "", errors.Wrapf(err, "make %s", workDir) diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index fed45eabb43..a23c9775ecd 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -62,6 +62,7 @@ import ( imagemod "yunion.io/x/onecloud/pkg/mcclient/modules/image" "yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset" "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/mountutils" "yunion.io/x/onecloud/pkg/util/pod" "yunion.io/x/onecloud/pkg/util/pod/logs" "yunion.io/x/onecloud/pkg/util/pod/nerdctl" @@ -2155,6 +2156,7 @@ func (s *sPodGuestInstance) DoSnapshot(ctx context.Context, params *SDiskSnapsho return nil, errors.Wrap(err, "EnsureBackupDir") } defer storageman.CleanupDirOrFile(tmpBackRootDir) + povTmpBackRootDir := []string{} for _, vol := range vols { drv := volume_mount.GetDriver(vol.Type) if err := drv.Mount(s, input.ContainerId, vol); err != nil { @@ -2179,27 +2181,51 @@ func (s *sPodGuestInstance) DoSnapshot(ctx context.Context, params *SDiskSnapsho return nil, errors.Wrapf(err, "touch %s: %s", targetBindMntPath, out) } } else { - if out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", targetBindMntPath).Output(); err != nil { - return nil, errors.Wrapf(err, "mkdir -p %s: %s", targetBindMntPath, out) + if err := volume_mount.EnsureDir(targetBindMntPath); err != nil { + return nil, errors.Wrap(err, "ensure dir") } } // do bind mount - out, err := procutils.NewRemoteCommandAsFarAsPossible("mount", "--bind", mntPath, targetBindMntPath).Output() - if err != nil { - return nil, errors.Wrapf(err, "bind mount %s to %s: %s", mntPath, targetBindMntPath, out) + if err := mountutils.MountBind(mntPath, targetBindMntPath); err != nil { + return nil, errors.Wrapf(err, "bind mount %s to %s", mntPath, targetBindMntPath) + } + // process post overlay + diskDrv := drv.(disk.IVolumeMountDisk) + for _, pov := range vol.Disk.PostOverlay { + // bind mount post overlay dirs to tmpBackRootDir + upperDir, err := diskDrv.GetPostOverlayRootUpperDir(s, vol, input.ContainerId) + if err != nil { + return nil, errors.Wrapf(err, "get post overlay root upper dir: %s", jsonutils.Marshal(pov)) + } + workDir, err := diskDrv.GetPostOverlayRootWorkDir(s, vol, input.ContainerId) + if err != nil { + return nil, errors.Wrapf(err, "get post overlay root upper dir: %s", jsonutils.Marshal(pov)) + } + hostDiskRootPath, _ := diskDrv.GetHostDiskRootPath(s, vol) + for _, srcDir := range []string{upperDir, workDir} { + targetSubDir := strings.TrimPrefix(srcDir, hostDiskRootPath) + targetPovBindMntPath := filepath.Join(tmpBackRootDir, targetSubDir) + if err := mountutils.MountBind(srcDir, targetPovBindMntPath); err != nil { + return nil, errors.Wrap(err, "bind mount post overlay dir") + } + povTmpBackRootDir = append(povTmpBackRootDir, targetPovBindMntPath) + } } } deferUmount := func() error { for _, vol := range vols { // unbind mount - // umount + for _, povPath := range povTmpBackRootDir { + if err := mountutils.Unmount(povPath); err != nil { + return errors.Wrapf(err, "umount bind point %s", povTmpBackRootDir) + } + } targetBindMntPath := filepath.Join(tmpBackRootDir, vol.Disk.SubDirectory) if vol.Disk.StorageSizeFile != "" { targetBindMntPath = filepath.Join(tmpBackRootDir, vol.Disk.StorageSizeFile) } - out, err := procutils.NewRemoteCommandAsFarAsPossible("umount", targetBindMntPath).Output() - if err != nil { - return errors.Wrapf(err, "umount bind point %s: %s", targetBindMntPath, out) + if err := mountutils.Unmount(targetBindMntPath); err != nil { + return errors.Wrapf(err, "umount bind point %s", targetBindMntPath) } } if !isCtrRunning && !s.IsRunning() { diff --git a/pkg/util/losetup/ioctl/remove.go b/pkg/util/losetup/ioctl/remove.go index e05bb96404f..08ce3a283a0 100644 --- a/pkg/util/losetup/ioctl/remove.go +++ b/pkg/util/losetup/ioctl/remove.go @@ -114,7 +114,7 @@ func DetachAndRemoveDevice(devPath string) error { } errs := []error{} - for i := 1; i <= 3; i++ { + for i := 1; i <= 5; i++ { if err := removeDev(); err != nil { err = errors.Wrapf(err, "remove loop device, %d times", i) log.Warningf("remove device %s: %v", devPath, err) diff --git a/pkg/util/mountutils/mount.go b/pkg/util/mountutils/mount.go index d93a92d7a81..1fed1d62157 100644 --- a/pkg/util/mountutils/mount.go +++ b/pkg/util/mountutils/mount.go @@ -67,6 +67,18 @@ func MountOverlay(lowerDir []string, upperDir string, workDir string, mergedDir }) } +func MountBind(src, target string) error { + return mountWrap(target, func() error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + bindArgs := []string{"--bind", src, target} + if out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", bindArgs...).Output(); err != nil { + return errors.Wrapf(err, "mount %v: %s", bindArgs, out) + } + return nil + }) +} + func Unmount(mountPoint string) error { err := unmount(mountPoint) errs := []error{}