diff --git a/cmd/climc/shell/compute/containers.go b/cmd/climc/shell/compute/containers.go index 5dadad723c9..de62fda694a 100644 --- a/cmd/climc/shell/compute/containers.go +++ b/cmd/climc/shell/compute/containers.go @@ -46,6 +46,8 @@ func init() { cmd.Perform("exec-sync", new(options.ContainerExecSyncOptions)) cmd.BatchPerform("set-resources-limit", new(options.ContainerSetResourcesLimitOptions)) cmd.Perform("commit", new(options.ContainerCommitOptions)) + cmd.Perform("add-volume-mount-post-overlay", new(options.ContainerAddVolumeMountPostOverlayOptions)) + cmd.Perform("remove-volume-mount-post-overlay", new(options.ContainerRemoveVolumeMountPostOverlayOptions)) type UpdateSpecOptions struct { ID string `help:"ID or name of server" json:"-"` diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index 32c0622d3e0..2fc94eeff10 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -82,6 +82,11 @@ const ( CONTAINER_STATUS_PROBING = "probing" CONTAINER_STATUS_PROBE_FAILED = "probe_failed" CONTAINER_STATUS_NET_FAILED = "net_failed" + // post overlay + CONTAINER_STATUS_ADD_POST_OVERLY = "adding_post_overly" + CONTAINER_STATUS_ADD_POST_OVERLY_FAILED = "add_post_overly_failed" + CONTAINER_STATUS_REMOVE_POST_OVERLY = "removing_post_overly" + CONTAINER_STATUS_REMOVE_POST_OVERLY_FAILED = "remove_post_overly_failed" ) var ( @@ -251,3 +256,14 @@ type ContainerResourcesSetInput struct { apis.ContainerResources DisableLimitCheck bool `json:"disable_limit_check"` } + +type ContainerVolumeMountAddPostOverlayInput struct { + Index int `json:"index"` + PostOverlay []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"` +} + +type ContainerVolumeMountRemovePostOverlayInput struct { + Index int `json:"index"` + PostOverlay []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"` + ClearLayers bool `json:"clear_layers"` +} diff --git a/pkg/apis/container.go b/pkg/apis/container.go index e32b683f2d0..e96c67f40a3 100644 --- a/pkg/apis/container.go +++ b/pkg/apis/container.go @@ -226,14 +226,24 @@ func (o ContainerVolumeMountDiskOverlay) IsValid() error { return nil } +type ContainerVolumeMountDiskPostOverlay struct { + // 宿主机底层目录 + HostLowerDir []string `json:"host_lower_dir"` + // 合并后要挂载到容器的目录 + ContainerTargetDir string `json:"container_target_dir"` +} + type ContainerVolumeMountDisk struct { - Index *int `json:"index,omitempty"` - Id string `json:"id"` - SubDirectory string `json:"sub_directory"` - StorageSizeFile string `json:"storage_size_file"` - Overlay *ContainerVolumeMountDiskOverlay `json:"overlay"` + Index *int `json:"index,omitempty"` + Id string `json:"id"` + SubDirectory string `json:"sub_directory"` + StorageSizeFile string `json:"storage_size_file"` + // lower overlay 设置，disk 的 volume 会作为 upper，最终 merged 的目录会传给容器 + Overlay *ContainerVolumeMountDiskOverlay `json:"overlay"` // case insensitive feature is incompatible with overlayfs CaseInsensitivePaths []string `json:"case_insensitive_paths"` + // 当 disk volume 挂载完后，需要 overlay 的目录设置 + PostOverlay []*ContainerVolumeMountDiskPostOverlay `json:"post_overlay"` } type ContainerVolumeMountHostPathType string diff --git a/pkg/apis/host/container.go b/pkg/apis/host/container.go index c26cfaf81e6..a8e013ab7c9 100644 --- a/pkg/apis/host/container.go +++ b/pkg/apis/host/container.go @@ -21,13 +21,14 @@ import ( ) type ContainerVolumeMountDisk struct { - Index *int `json:"index,omitempty"` - Id string `json:"id"` - TemplateId string `json:"template_id"` - SubDirectory string `json:"sub_directory"` - StorageSizeFile string `json:"storage_size_file"` - Overlay *apis.ContainerVolumeMountDiskOverlay `json:"overlay"` - CaseInsensitivePaths []string `json:"case_insensitive_paths"` + Index *int `json:"index,omitempty"` + Id string `json:"id"` + TemplateId string `json:"template_id"` + SubDirectory string `json:"sub_directory"` + StorageSizeFile string `json:"storage_size_file"` + Overlay *apis.ContainerVolumeMountDiskOverlay `json:"overlay"` + CaseInsensitivePaths []string `json:"case_insensitive_paths"` + PostOverlay []*apis.ContainerVolumeMountDiskPostOverlay `json:"post_overlay"` } type ContainerVolumeMountCephFS struct { diff --git a/pkg/compute/container_drivers/volume_mount/disk.go b/pkg/compute/container_drivers/volume_mount/disk.go index f13a656deaa..fa745c453f4 100644 --- a/pkg/compute/container_drivers/volume_mount/disk.go +++ b/pkg/compute/container_drivers/volume_mount/disk.go @@ -135,6 +135,9 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre if err := d.validateOverlay(ctx, userCred, vm, &diskObj); err != nil { return nil, errors.Wrapf(err, "validate overlay") } + if err := d.ValidatePostOverlay(vm); err != nil { + return nil, errors.Wrap(err, "validate post overlay") + } return vm, nil } @@ -184,6 +187,36 @@ func (d disk) validateOverlay(ctx context.Context, userCred mcclient.TokenCreden return nil } +func (d disk) ValidatePostOverlay(vm *apis.ContainerVolumeMount) error { + if len(vm.Disk.PostOverlay) == 0 { + return nil + } + ovs := vm.Disk.PostOverlay + var duplicateCtrDir string + for _, ov := range ovs { + if len(ov.HostLowerDir) == 0 { + return httperrors.NewNotEmptyError("host_lower_dir is required") + } + for i, hld := range ov.HostLowerDir { + if len(hld) == 0 { + return httperrors.NewNotEmptyError("host_lower_dir %d is empty", i) + } + } + if len(ov.ContainerTargetDir) == 0 { + return httperrors.NewNotEmptyError("container_target_dir is required") + } + if ov.ContainerTargetDir == duplicateCtrDir { + return httperrors.NewDuplicateNameError("container_target_dir", ov.ContainerTargetDir) + } + duplicateCtrDir = ov.ContainerTargetDir + } + if vm.Propagation == "" { + // 设置默认 propagation 为 rslave + vm.Propagation = apis.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER + } + return nil +} + type diskOverlayDir struct{} func newDiskOverlayDir() iDiskOverlay { diff --git a/pkg/compute/guestdrivers/pod.go b/pkg/compute/guestdrivers/pod.go index 42908322d04..b6c820b7081 100644 --- a/pkg/compute/guestdrivers/pod.go +++ b/pkg/compute/guestdrivers/pod.go @@ -472,6 +472,14 @@ func (p *SPodDriver) RequestPullContainerImage(ctx context.Context, userCred mcc return p.performContainerAction(ctx, userCred, task, "pull-image", task.GetParams()) } +func (p *SPodDriver) RequestAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error { + return p.performContainerAction(ctx, userCred, task, "add-volume-mount-post-overlay", task.GetParams()) +} + +func (p *SPodDriver) RequestRemoveVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error { + return p.performContainerAction(ctx, userCred, task, "remove-volume-mount-post-overlay", task.GetParams()) +} + type responder struct { errorMessage string } diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go index 11d58354fa7..6aebf5d08d5 100644 --- a/pkg/compute/models/containers.go +++ b/pkg/compute/models/containers.go @@ -443,6 +443,7 @@ func (vm *ContainerVolumeMountRelation) toHostDiskMount(disk *apis.ContainerVolu StorageSizeFile: disk.StorageSizeFile, Overlay: disk.Overlay, CaseInsensitivePaths: disk.CaseInsensitivePaths, + PostOverlay: disk.PostOverlay, } return ret, nil } @@ -987,3 +988,133 @@ func (c *SContainer) StartCommit(ctx context.Context, userCred mcclient.TokenCre } return task.ScheduleRun(nil) } + +func (c *SContainer) isPostOverlayExist(vm *apis.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) bool { + for _, cov := range vm.Disk.PostOverlay { + if ov.ContainerTargetDir == cov.ContainerTargetDir { + return true + } + } + return false +} + +func (c *SContainer) validateVolumeMountPostOverlayAction(action string, index int, ovs []*apis.ContainerVolumeMountDiskPostOverlay) (*apis.ContainerVolumeMount, error) { + if !api.ContainerExitedStatus.Has(c.Status) && !api.ContainerRunningStatus.Has(c.Status) { + return nil, httperrors.NewInvalidStatusError("can't %s post overlay on status %s", action, c.Status) + } + if index >= len(c.Spec.VolumeMounts) { + return nil, httperrors.NewInputParameterError("index %d out of volume_mount size %d", index, len(c.Spec.VolumeMounts)) + } + vm := new(apis.ContainerVolumeMount) + curVm := c.Spec.VolumeMounts[index] + if err := jsonutils.Marshal(curVm).Unmarshal(vm); err != nil { + return nil, errors.Wrap(err, "use json unmarshal to new volume mount") + } + if vm.Type != apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK { + return nil, httperrors.NewInputParameterError("invalid volume mount type %s", vm.Type) + } + return vm, nil +} + +func (c *SContainer) GetVolumeMountCopy(index int) (*apis.ContainerVolumeMount, error) { + if index >= len(c.Spec.VolumeMounts) { + return nil, httperrors.NewInputParameterError("index %d out of volume_mount size %d", index, len(c.Spec.VolumeMounts)) + } + vm := new(apis.ContainerVolumeMount) + curVm := c.Spec.VolumeMounts[index] + if err := jsonutils.Marshal(curVm).Unmarshal(vm); err != nil { + return nil, errors.Wrap(err, "use json unmarshal to new volume mount") + } + return vm, nil +} + +func (c *SContainer) getPostOverlayVolumeMount( + index int, + updateF func(mount *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error), +) (*apis.ContainerVolumeMount, error) { + vm, err := c.GetVolumeMountCopy(index) + if err != nil { + return nil, err + } + return updateF(vm) +} + +func (c *SContainer) GetAddPostOverlayVolumeMount(index int, ovs []*apis.ContainerVolumeMountDiskPostOverlay) (*apis.ContainerVolumeMount, error) { + return c.getPostOverlayVolumeMount(index, func(vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) { + if vm.Disk.PostOverlay == nil { + vm.Disk.PostOverlay = []*apis.ContainerVolumeMountDiskPostOverlay{} + } + vm.Disk.PostOverlay = append(vm.Disk.PostOverlay, ovs...) + return vm, nil + }) +} + +func (c *SContainer) GetRemovePostOverlayVolumeMount(index int, ovs []*apis.ContainerVolumeMountDiskPostOverlay) (*apis.ContainerVolumeMount, error) { + return c.getPostOverlayVolumeMount(index, func(vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) { + // remove post overlay + for _, ov := range ovs { + vm.Disk = c.removePostOverlay(vm.Disk, ov) + } + return vm, nil + }) +} + +func (c *SContainer) PerformAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, input *api.ContainerVolumeMountAddPostOverlayInput) (jsonutils.JSONObject, error) { + vm, err := c.validateVolumeMountPostOverlayAction("add", input.Index, input.PostOverlay) + if err != nil { + return nil, err + } + for _, ov := range input.PostOverlay { + isExist := c.isPostOverlayExist(vm, ov) + if isExist { + return nil, httperrors.NewInputParameterError("post overlay %s already exists", ov.ContainerTargetDir) + } + } + return nil, c.StartAddVolumeMountPostOverlayTask(ctx, userCred, input, "") +} + +func (c *SContainer) StartAddVolumeMountPostOverlayTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ContainerVolumeMountAddPostOverlayInput, parentTaskId string) error { + c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_ADD_POST_OVERLY, "") + task, err := taskman.TaskManager.NewTask(ctx, "ContainerAddVolumeMountPostOverlayTask", c, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), parentTaskId, "", nil) + if err != nil { + return errors.Wrap(err, "New ContainerAddVolumeMountPostOverlayTask") + } + return task.ScheduleRun(nil) +} + +func (c *SContainer) StartRemoveVolumeMountPostOverlayTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ContainerVolumeMountRemovePostOverlayInput, parentTaskId string) error { + c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_REMOVE_POST_OVERLY, "") + task, err := taskman.TaskManager.NewTask(ctx, "ContainerRemoveVolumeMountPostOverlayTask", c, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), parentTaskId, "", nil) + if err != nil { + return errors.Wrap(err, "New ContainerRemoveVolumeMountPostOverlayTask") + } + return task.ScheduleRun(nil) +} + +func (c *SContainer) PerformRemoveVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, input *api.ContainerVolumeMountRemovePostOverlayInput) (jsonutils.JSONObject, error) { + vm, err := c.validateVolumeMountPostOverlayAction("remove", input.Index, input.PostOverlay) + if err != nil { + return nil, err + } + if len(vm.Disk.PostOverlay) == 0 { + return nil, httperrors.NewInputParameterError("no post overlay") + } + for _, ov := range input.PostOverlay { + isExist := c.isPostOverlayExist(vm, ov) + if !isExist { + return nil, httperrors.NewInputParameterError("post overlay %s not exists", ov.ContainerTargetDir) + } + } + return nil, c.StartRemoveVolumeMountPostOverlayTask(ctx, userCred, input, "") +} + +func (c *SContainer) removePostOverlay(vmd *apis.ContainerVolumeMountDisk, ov *apis.ContainerVolumeMountDiskPostOverlay) *apis.ContainerVolumeMountDisk { + curOvs := vmd.PostOverlay + for i, cov := range curOvs { + if cov.ContainerTargetDir == ov.ContainerTargetDir { + curOvs = append(curOvs[:i], curOvs[i+1:]...) + } + } + vmd.PostOverlay = curOvs + return vmd +} diff --git a/pkg/compute/models/pod_driver.go b/pkg/compute/models/pod_driver.go index 3fff890cc09..70542294559 100644 --- a/pkg/compute/models/pod_driver.go +++ b/pkg/compute/models/pod_driver.go @@ -44,5 +44,8 @@ type IPodDriver interface { RequestCommitContainer(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error RequestSaveVolumeMountImage(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error RequestExecSyncContainer(ctx context.Context, userCred mcclient.TokenCredential, ctr *SContainer, input *compute.ContainerExecSyncInput) (jsonutils.JSONObject, error) - RequestSetContainerResourcesLimit(ctx context.Context, cred mcclient.TokenCredential, c *SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error) + RequestSetContainerResourcesLimit(ctx context.Context, userCred mcclient.TokenCredential, c *SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error) + + RequestAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error + RequestRemoveVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error } diff --git a/pkg/compute/tasks/container_volume_mount_task.go b/pkg/compute/tasks/container_volume_mount_task.go new file mode 100644 index 00000000000..3ce13852603 --- /dev/null +++ b/pkg/compute/tasks/container_volume_mount_task.go @@ -0,0 +1,162 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func init() { + taskman.RegisterTask(ContainerAddVolumeMountPostOverlayTask{}) + taskman.RegisterTask(ContainerRemoveVolumeMountPostOverlayTask{}) +} + +type ContainerVolumeMountTaskPostOverlay struct { + ContainerBaseTask +} + +func (t *ContainerVolumeMountTaskPostOverlay) UpdateContainerVolume(c *models.SContainer, index int, vm *apis.ContainerVolumeMount) error { + if _, err := db.Update(c, func() error { + c.Spec.VolumeMounts[index] = vm + return nil + }); err != nil { + return errors.Wrapf(err, "UpdateContainerVolume %d", index) + } + return nil +} + +type ContainerAddVolumeMountPostOverlayTask struct { + ContainerVolumeMountTaskPostOverlay +} + +func (t *ContainerAddVolumeMountPostOverlayTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + t.requestAdd(ctx, obj.(*models.SContainer)) +} + +func (t *ContainerAddVolumeMountPostOverlayTask) getInput() (*api.ContainerVolumeMountAddPostOverlayInput, error) { + input := new(api.ContainerVolumeMountAddPostOverlayInput) + if err := t.GetParams().Unmarshal(input); err != nil { + return nil, err + } + return input, nil +} + +func (t *ContainerAddVolumeMountPostOverlayTask) requestAdd(ctx context.Context, c *models.SContainer) { + t.SetStage("OnAdded", nil) + if err := t.GetPodDriver().RequestAddVolumeMountPostOverlay(ctx, t.GetUserCred(), t); err != nil { + t.OnAddedFailed(ctx, c, jsonutils.NewString(err.Error())) + return + } +} + +func (t *ContainerAddVolumeMountPostOverlayTask) OnAdded(ctx context.Context, c *models.SContainer, _ jsonutils.JSONObject) { + if err := t.updateVolume(ctx, c); err != nil { + t.OnAddedFailed(ctx, c, jsonutils.NewString(err.Error())) + return + } + t.SetStage("OnSynced", nil) + c.GetPod().StartSyncTask(ctx, t.GetUserCred(), false, t.GetTaskId()) +} + +func (t *ContainerAddVolumeMountPostOverlayTask) updateVolume(ctx context.Context, c *models.SContainer) error { + input, err := t.getInput() + if err != nil { + return errors.Wrap(err, "getInput") + } + vm, err := c.GetAddPostOverlayVolumeMount(input.Index, input.PostOverlay) + if err != nil { + return errors.Wrap(err, "GetAddPostOverlayVolumeMount") + } + if err := t.UpdateContainerVolume(c, input.Index, vm); err != nil { + return errors.Wrap(err, "UpdateContainerVolume") + } + return nil +} + +func (t *ContainerAddVolumeMountPostOverlayTask) OnAddedFailed(ctx context.Context, c *models.SContainer, reason jsonutils.JSONObject) { + c.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_ADD_POST_OVERLY_FAILED, reason.String()) + t.SetStageFailed(ctx, reason) +} + +func (t *ContainerAddVolumeMountPostOverlayTask) OnSynced(ctx context.Context, c *models.SContainer, _ jsonutils.JSONObject) { + t.SetStageComplete(ctx, nil) +} + +type ContainerRemoveVolumeMountPostOverlayTask struct { + ContainerVolumeMountTaskPostOverlay +} + +func (t *ContainerRemoveVolumeMountPostOverlayTask) getInput() (*api.ContainerVolumeMountRemovePostOverlayInput, error) { + input := new(api.ContainerVolumeMountRemovePostOverlayInput) + if err := t.GetParams().Unmarshal(input); err != nil { + return nil, err + } + return input, nil +} + +func (t *ContainerRemoveVolumeMountPostOverlayTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + t.requestRemove(ctx, obj.(*models.SContainer)) +} + +func (t *ContainerRemoveVolumeMountPostOverlayTask) requestRemove(ctx context.Context, c *models.SContainer) { + t.SetStage("OnRemoved", nil) + // 如果是关机情况，并且还要 clear layers ，需要 mount disk 起来，然后清理 + if err := t.GetPodDriver().RequestRemoveVolumeMountPostOverlay(ctx, t.GetUserCred(), t); err != nil { + t.OnRemovedFailed(ctx, c, jsonutils.NewString(err.Error())) + return + } +} + +func (t *ContainerRemoveVolumeMountPostOverlayTask) updateVolume(ctx context.Context, c *models.SContainer) error { + input, err := t.getInput() + if err != nil { + return errors.Wrap(err, "getInput") + } + vm, err := c.GetRemovePostOverlayVolumeMount(input.Index, input.PostOverlay) + if err != nil { + return errors.Wrap(err, "GetAddPostOverlayVolumeMount") + } + if err := t.UpdateContainerVolume(c, input.Index, vm); err != nil { + return errors.Wrap(err, "UpdateContainerVolume") + } + return nil +} + +func (t *ContainerRemoveVolumeMountPostOverlayTask) OnRemoved(ctx context.Context, c *models.SContainer, _ jsonutils.JSONObject) { + if err := t.updateVolume(ctx, c); err != nil { + t.OnRemovedFailed(ctx, c, jsonutils.NewString(err.Error())) + return + } + t.SetStage("OnSynced", nil) + c.GetPod().StartSyncTask(ctx, t.GetUserCred(), false, t.GetTaskId()) +} + +func (t *ContainerRemoveVolumeMountPostOverlayTask) OnRemovedFailed(ctx context.Context, c *models.SContainer, reason jsonutils.JSONObject) { + c.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_REMOVE_POST_OVERLY_FAILED, reason.String()) + t.SetStageFailed(ctx, reason) +} + +func (t *ContainerRemoveVolumeMountPostOverlayTask) OnSynced(ctx context.Context, c *models.SContainer, _ jsonutils.JSONObject) { + t.SetStageComplete(ctx, nil) +} diff --git a/pkg/hostman/container/prober/prober.go b/pkg/hostman/container/prober/prober.go index c36e77689d9..6f2e3691f57 100644 --- a/pkg/hostman/container/prober/prober.go +++ b/pkg/hostman/container/prober/prober.go @@ -42,7 +42,6 @@ import ( hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/container/prober/results" "yunion.io/x/onecloud/pkg/hostman/guestman/container" - "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/util/exec" "yunion.io/x/onecloud/pkg/util/probe" execprobe "yunion.io/x/onecloud/pkg/util/probe/exec" @@ -67,7 +66,7 @@ func newProber(runner container.CommandRunner) *prober { } // probe probes the container. -func (pb *prober) probe(probeType apis.ContainerProbeType, pod *desc.SGuestDesc, container *hostapi.ContainerDesc) (results.ProbeResult, error) { +func (pb *prober) probe(probeType apis.ContainerProbeType, pod IPod, container *hostapi.ContainerDesc) (results.ProbeResult, error) { var probeSpec *apis.ContainerProbe switch probeType { //case apis.ContainerProbeTypeLiveness: @@ -79,7 +78,7 @@ func (pb *prober) probe(probeType apis.ContainerProbeType, pod *desc.SGuestDesc, return results.NewFailure(err.Error()), err } - ctrName := fmt.Sprintf("%s:%s", pod.Name, container.Name) + ctrName := fmt.Sprintf("%s:%s", pod.GetDesc().Name, container.Name) if probeSpec == nil { log.Warningf("%s probe for %s is nil", probeType, ctrName) return results.NewSuccess("probe is not defined"), nil @@ -111,7 +110,7 @@ func (pb *prober) probe(probeType apis.ContainerProbeType, pod *desc.SGuestDesc, // runProbeWithRetries tries to probe the container in a finite loop, it returns the last result // if it never succeeds. -func (pb *prober) runProbeWithRetries(probeType apis.ContainerProbeType, p *apis.ContainerProbe, pod *desc.SGuestDesc, container *hostapi.ContainerDesc, retries int) (probe.Result, string, error) { +func (pb *prober) runProbeWithRetries(probeType apis.ContainerProbeType, p *apis.ContainerProbe, pod IPod, container *hostapi.ContainerDesc, retries int) (probe.Result, string, error) { var err error var result probe.Result var output string @@ -124,17 +123,17 @@ func (pb *prober) runProbeWithRetries(probeType apis.ContainerProbeType, p *apis return result, output, err } -func (pb *prober) runProbe(probeType apis.ContainerProbeType, p *apis.ContainerProbe, pod *desc.SGuestDesc, container *hostapi.ContainerDesc) (probe.Result, string, error) { +func (pb *prober) runProbe(probeType apis.ContainerProbeType, p *apis.ContainerProbe, pod IPod, container *hostapi.ContainerDesc) (probe.Result, string, error) { timeout := time.Duration(p.TimeoutSeconds) * time.Second if p.Exec != nil { - log.Debugf("Exec-Probe Pod: %v, Container: %v, Command: %v", pod.Name, container.Name, p.Exec.Command) + log.Debugf("Exec-Probe Pod: %v, Container: %v, Command: %v", pod.GetDesc().Name, container.Name, p.Exec.Command) return pb.exec.Probe(pb.newExecInContainer(pod, container, p.Exec.Command, timeout)) } if p.TCPSocket != nil { port := p.TCPSocket.Port host := p.TCPSocket.Host if host == "" { - for _, nic := range pod.Nics { + for _, nic := range pod.GetDesc().Nics { if nic.Ip != "" { host = nic.Ip break @@ -147,7 +146,7 @@ func (pb *prober) runProbe(probeType apis.ContainerProbeType, p *apis.ContainerP log.Debugf("TCP-Probe Host: %v, Port: %v, Timeout: %v", host, port, timeout) return pb.tcp.Probe(host, port, timeout) } - errMsg := fmt.Sprintf("Failed to find probe builder for pod %v, container: %v", pod.Name, container.Name) + errMsg := fmt.Sprintf("Failed to find probe builder for pod %v, container: %v", pod.GetName(), container.Name) log.Warningf(errMsg) return probe.Unknown, "", errors.Error(errMsg) } @@ -159,10 +158,10 @@ type execInContainer struct { writer io.Writer } -func (pb *prober) newExecInContainer(pod *desc.SGuestDesc, container *hostapi.ContainerDesc, cmd []string, timeout time.Duration) exec.Cmd { +func (pb *prober) newExecInContainer(pod IPod, container *hostapi.ContainerDesc, cmd []string, timeout time.Duration) exec.Cmd { return &execInContainer{ run: func() ([]byte, error) { - return pb.runner.RunInContainer(pod, container.Id, cmd, timeout) + return pb.runner.RunInContainer(pod.GetId(), container.Id, cmd, timeout) }, } } diff --git a/pkg/hostman/container/prober/prober_manager.go b/pkg/hostman/container/prober/prober_manager.go index 9e29adf27bf..344c4dce20f 100644 --- a/pkg/hostman/container/prober/prober_manager.go +++ b/pkg/hostman/container/prober/prober_manager.go @@ -38,6 +38,7 @@ import ( "yunion.io/x/pkg/util/wait" "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/container/prober/results" "yunion.io/x/onecloud/pkg/hostman/container/status" "yunion.io/x/onecloud/pkg/hostman/guestman/container" @@ -51,6 +52,14 @@ type probeKey struct { probeType apis.ContainerProbeType } +type IPod interface { + GetId() string + GetName() string + GetDesc() *desc.SGuestDesc + GetContainers() []*host.ContainerDesc + IsRunning() bool +} + // Manager manages pod probing. It creates a probe "worker" for every container that specifies a // probe (AddPod). The worker periodically probes its assigned container and caches the results. The // manager use the cached probe results to set the appropriate Ready state in the PodStatus when @@ -59,11 +68,11 @@ type probeKey struct { type Manager interface { // AddPod creates new probe workers for every container probe. This should be called for every // pod created. - AddPod(pod *desc.SGuestDesc) + AddPod(pod IPod) // RemovePod handles cleaning up the removed pod state, including terminating probe workers and // deleting cached results. - RemovePod(pod *desc.SGuestDesc) + RemovePod(pod IPod) // CleanupPods handles cleaning up pods which should no longer be running. // It takes a map of "desired pods" which should not be cleaned up. @@ -135,17 +144,17 @@ func (m *manager) Start() { go wait.Forever(m.updateStartup, 0) } -func (m *manager) AddPod(pod *desc.SGuestDesc) { +func (m *manager) AddPod(pod IPod) { m.workerLock.Lock() defer m.workerLock.Unlock() - key := probeKey{podUid: pod.Uuid} - for _, c := range pod.Containers { + key := probeKey{podUid: pod.GetId()} + for _, c := range pod.GetContainers() { key.containerName = c.Name if c.Spec.StartupProbe != nil { key.probeType = apis.ContainerProbeTypeStartup if _, ok := m.workers[key]; ok { - log.Errorf("Startup probe already exists: %s:%s", pod.Name, c.Name) + log.Errorf("Startup probe already exists: %s:%s", pod.GetName(), c.Name) return } w := newWorker(m, key.probeType, pod, c) @@ -166,12 +175,12 @@ func (m *manager) AddPod(pod *desc.SGuestDesc) { } } -func (m *manager) RemovePod(pod *desc.SGuestDesc) { +func (m *manager) RemovePod(pod IPod) { m.workerLock.RLock() defer m.workerLock.RUnlock() - key := probeKey{podUid: pod.Uuid} - for _, c := range pod.Containers { + key := probeKey{podUid: pod.GetId()} + for _, c := range pod.GetContainers() { key.containerName = c.Name for _, probeType := range []apis.ContainerProbeType{apis.ContainerProbeTypeLiveness, apis.ContainerProbeTypeReadiness, apis.ContainerProbeTypeStartup} { key.probeType = probeType @@ -226,5 +235,11 @@ func (m *manager) updateStartup() { update := <-m.startupManager.Updates() started := update.Result.Result == results.Success - m.statusManager.SetContainerStartup(update.PodUID, update.ContainerID, started, update.Result) + m.statusManager.SetContainerStartup( + update.PodUID, + update.ContainerID, + started, + update.Result, + update.Pod, + ) } diff --git a/pkg/hostman/container/prober/results/results_manager.go b/pkg/hostman/container/prober/results/results_manager.go index a27befca7ab..fb5d6d319d3 100644 --- a/pkg/hostman/container/prober/results/results_manager.go +++ b/pkg/hostman/container/prober/results/results_manager.go @@ -34,8 +34,6 @@ import ( "fmt" "strings" "sync" - - "yunion.io/x/onecloud/pkg/hostman/guestman/desc" ) func NewFailure(reason string) ProbeResult { @@ -107,11 +105,17 @@ func (r Result) String() string { } } +type IPod interface { + GetId() string + IsRunning() bool +} + // Update is an enum of the types of updates sent over the Updates channel. type Update struct { ContainerID string Result ProbeResult PodUID string + Pod IPod } // Manager provides a probe results cache and channel of updates @@ -120,7 +124,7 @@ type Manager interface { Get(containerId string) (ProbeResult, bool) // Set sets the cached result for the container with the given ID. // The pod is only included to be sent with the update. - Set(containerId string, result ProbeResult, pod *desc.SGuestDesc, force bool) + Set(containerId string, result ProbeResult, pod IPod, force bool) // Remove clears the cached result for the container with the given ID. Remove(containerId string) // Updates creates a channel that receives an Update whenever its result changes (but not @@ -154,9 +158,14 @@ func (m *manager) Get(id string) (ProbeResult, bool) { return result, found } -func (m *manager) Set(id string, result ProbeResult, pod *desc.SGuestDesc, force bool) { +func (m *manager) Set(id string, result ProbeResult, pod IPod, force bool) { if m.setInternal(id, result, force) { - m.updates <- Update{ContainerID: id, Result: result, PodUID: pod.Uuid} + m.updates <- Update{ + ContainerID: id, + Result: result, + PodUID: pod.GetId(), + Pod: pod, + } } } diff --git a/pkg/hostman/container/prober/worker.go b/pkg/hostman/container/prober/worker.go index d5564f91da2..53af67743b5 100644 --- a/pkg/hostman/container/prober/worker.go +++ b/pkg/hostman/container/prober/worker.go @@ -40,7 +40,6 @@ import ( "yunion.io/x/onecloud/pkg/apis" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/container/prober/results" - "yunion.io/x/onecloud/pkg/hostman/guestman/desc" ) // worker handles the periodic probing of its assigned container. Each worker has a go-routine @@ -52,7 +51,7 @@ type worker struct { stopCh chan struct{} // The pod containing this probe (read-only) - pod *desc.SGuestDesc + pod IPod // The container to probe (read-only) container *hostapi.ContainerDesc @@ -85,7 +84,7 @@ type worker struct { func newWorker( m *manager, probeType apis.ContainerProbeType, - pod *desc.SGuestDesc, + pod IPod, container *hostapi.ContainerDesc) *worker { w := &worker{ stopCh: make(chan struct{}, 1), // Buffer so stop() can be non-blocking. @@ -127,7 +126,7 @@ func (w *worker) run() { w.resultsManager.Remove(w.containerId) } - w.probeManager.removeWorker(w.pod.Uuid, w.container.Name, w.probeType) + w.probeManager.removeWorker(w.pod.GetId(), w.container.Name, w.probeType) }() probeLoop: @@ -162,7 +161,7 @@ func (w *worker) doProbe() (keepGoing bool) { result, err := w.probeManager.prober.probe(w.probeType, w.pod, w.container) if err != nil { - log.Errorf("probe: %s, pod: %s, container: %s, error: %v", w.probeType, w.pod.Uuid, w.container.Id, err) + log.Errorf("probe: %s, pod: %s, container: %s, error: %v", w.probeType, w.pod.GetId(), w.container.Id, err) // prober error, throw away the result. return true } diff --git a/pkg/hostman/container/status/status_manager.go b/pkg/hostman/container/status/status_manager.go index d991852877e..3c9e6b21499 100644 --- a/pkg/hostman/container/status/status_manager.go +++ b/pkg/hostman/container/status/status_manager.go @@ -29,7 +29,7 @@ import ( type Manager interface { // SetContainerStartup updates the container status with the given startup // and triggers a status update. - SetContainerStartup(podId string, containerId string, started bool, result results.ProbeResult) + SetContainerStartup(podId string, containerId string, started bool, result results.ProbeResult, pod results.IPod) } type manager struct{} @@ -38,12 +38,12 @@ func NewManager() Manager { return &manager{} } -func (m *manager) SetContainerStartup(podId string, containerId string, started bool, result results.ProbeResult) { +func (m *manager) SetContainerStartup(podId string, containerId string, started bool, result results.ProbeResult, pod results.IPod) { status := computeapi.CONTAINER_STATUS_PROBE_FAILED if started { status = computeapi.CONTAINER_STATUS_RUNNING } else { - if result.IsNetFailedError() { + if result.IsNetFailedError() && pod.IsRunning() { status = computeapi.CONTAINER_STATUS_NET_FAILED } } diff --git a/pkg/hostman/container/volume_mount/cephfs.go b/pkg/hostman/container/volume_mount/cephfs.go index dd87c08cc14..41438b5875c 100644 --- a/pkg/hostman/container/volume_mount/cephfs.go +++ b/pkg/hostman/container/volume_mount/cephfs.go @@ -37,7 +37,9 @@ func (h cephFS) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMou if err != nil { return err } - procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + if err := EnsureDir(dir); err != nil { + return errors.Wrap(err, "EnsureDir") + } options := fmt.Sprintf("name=%s,secret=%s", vm.CephFS.Name, vm.CephFS.Secret) if vm.ReadOnly { options += ",ro" diff --git a/pkg/hostman/container/volume_mount/disk.go b/pkg/hostman/container/volume_mount/disk/disk.go similarity index 64% rename from pkg/hostman/container/volume_mount/disk.go rename to pkg/hostman/container/volume_mount/disk/disk.go index 3db5e3373b9..8b05ba79cde 100644 --- a/pkg/hostman/container/volume_mount/disk.go +++ b/pkg/hostman/container/volume_mount/disk/disk.go @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package volume_mount +package disk import ( "context" "fmt" "path/filepath" - "strings" "yunion.io/x/pkg/errors" @@ -27,6 +26,7 @@ import ( hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/container/storage" container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/httperrors" @@ -34,19 +34,21 @@ import ( ) func init() { - RegisterDriver(newDisk()) + volume_mount.RegisterDriver(newDisk()) } -type iDiskOverlay interface { - mount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error - unmount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error +type IVolumeMountDisk interface { + volume_mount.IUsageVolumeMount + + MountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error + UnmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay, clearLayers bool) error } type disk struct { overlayDrivers map[apis.ContainerDiskOverlayType]iDiskOverlay } -func newDisk() IVolumeMount { +func newDisk() IVolumeMountDisk { return &disk{ overlayDrivers: map[apis.ContainerDiskOverlayType]iDiskOverlay{ apis.CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY: newDiskOverlayDir(), @@ -59,12 +61,21 @@ func (d disk) GetType() apis.ContainerVolumeMountType { return apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK } -func (d disk) getRuntimeMountHostPath(pod IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) { +func (d disk) getHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) { diskInput := vm.Disk if diskInput == nil { return "", httperrors.NewNotEmptyError("disk is nil") } hostPath := filepath.Join(pod.GetVolumesDir(), diskInput.Id) + return hostPath, nil +} + +func (d disk) getRuntimeMountHostPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) { + hostPath, err := d.getHostDiskRootPath(pod, vm) + if err != nil { + return "", errors.Wrap(err, "get host disk root path") + } + diskInput := vm.Disk if diskInput.SubDirectory != "" { return filepath.Join(hostPath, diskInput.SubDirectory), nil } @@ -74,7 +85,7 @@ func (d disk) getRuntimeMountHostPath(pod IPodInfo, vm *hostapi.ContainerVolumeM return hostPath, nil } -func (d disk) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) (string, error) { +func (d disk) GetRuntimeMountHostPath(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) (string, error) { hostPath, err := d.getRuntimeMountHostPath(pod, vm) if err != nil { return "", errors.Wrap(err, "get runtime mount host_path") @@ -86,7 +97,7 @@ func (d disk) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *hostapi.Co return d.getOverlayMergedDir(pod, ctrId, vm, hostPath), nil } -func (d disk) getPodDisk(pod IPodInfo, vm *hostapi.ContainerVolumeMount) (storageman.IDisk, *desc.SGuestDisk, error) { +func (d disk) getPodDisk(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (storageman.IDisk, *desc.SGuestDisk, error) { var disk *desc.SGuestDisk = nil disks := pod.GetDisks() volDisk := vm.Disk @@ -111,7 +122,7 @@ func (d disk) getPodDisk(pod IPodInfo, vm *hostapi.ContainerVolumeMount) (storag return iDisk, disk, nil } -func (d disk) getDiskStorageDriver(pod IPodInfo, vm *hostapi.ContainerVolumeMount) (storage.IContainerStorage, error) { +func (d disk) getDiskStorageDriver(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (storage.IContainerStorage, error) { iDisk, _, err := d.getPodDisk(pod, vm) if err != nil { return nil, errors.Wrap(err, "get pod disk interface") @@ -123,18 +134,6 @@ func (d disk) getDiskStorageDriver(pod IPodInfo, vm *hostapi.ContainerVolumeMoun return drv, nil } -func (d disk) getOverlayDir(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, upperDir string, suffix string) string { - return filepath.Join(pod.GetVolumesOverlayDir(), vm.Disk.Id, ctrId, fmt.Sprintf("%s-%s", filepath.Base(upperDir), suffix)) -} - -func (d disk) getOverlayWorkDir(upperDir string) string { - return fmt.Sprintf("%s-work", upperDir) -} - -func (d disk) getOverlayMergedDir(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, upperDir string) string { - return d.getOverlayDir(pod, ctrId, vm, upperDir, "merged") -} - func (d disk) setDirCaseInsensitive(dir string) error { out, err := procutils.NewRemoteCommandAsFarAsPossible("chattr", "+F", dir).Output() if err != nil { @@ -143,7 +142,11 @@ func (d disk) setDirCaseInsensitive(dir string) error { return nil } -func (d disk) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { +func (d disk) newPostOverlay() iDiskPostOverlay { + return newDiskPostOverlay(d) +} + +func (d disk) Mount(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { iDisk, gd, err := d.getPodDisk(pod, vm) if err != nil { return errors.Wrap(err, "get pod disk interface") @@ -169,15 +172,13 @@ func (d disk) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount vmDisk := vm.Disk if vmDisk.SubDirectory != "" { subDir := filepath.Join(mntPoint, vmDisk.SubDirectory) - out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", subDir).Output() - if err != nil { - return errors.Wrapf(err, "make sub_directory %s inside %s: %s", vmDisk.SubDirectory, mntPoint, out) + if err := volume_mount.EnsureDir(subDir); err != nil { + return errors.Wrapf(err, "make sub_directory %s inside %s", vmDisk.SubDirectory, mntPoint) } for _, cd := range vmDisk.CaseInsensitivePaths { cdp := filepath.Join(subDir, cd) - out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", cdp).Output() - if err != nil { - return errors.Wrapf(err, "make %s inside %s: %s", cdp, vmDisk.SubDirectory, out) + if err := volume_mount.EnsureDir(cdp); err != nil { + return errors.Wrapf(err, "make %s inside %s", cdp, vmDisk.SubDirectory) } if err := d.setDirCaseInsensitive(cdp); err != nil { return errors.Wrapf(err, "enable case_insensitive %s", cdp) @@ -198,6 +199,11 @@ func (d disk) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount return errors.Wrapf(err, "mount container %s overlay dir: %#v", ctrId, vmDisk.Overlay) } } + if len(vmDisk.PostOverlay) != 0 { + if err := d.MountPostOverlays(pod, ctrId, vm, vmDisk.PostOverlay); err != nil { + return errors.Wrap(err, "mount post overlay dirs") + } + } return nil } @@ -216,7 +222,7 @@ func (d disk) createStorageSizeFile(iDisk storageman.IDisk, mntPoint string, inp return nil } -func (d disk) Unmount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { +func (d disk) Unmount(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { iDisk, _, err := d.getPodDisk(pod, vm) if err != nil { return errors.Wrap(err, "get pod disk interface") @@ -225,6 +231,11 @@ func (d disk) Unmount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMou if err != nil { return errors.Wrap(err, "get disk storage driver") } + if len(vm.Disk.PostOverlay) != 0 { + if err := d.UnmountPostOverlays(pod, ctrId, vm, vm.Disk.PostOverlay, false); err != nil { + return errors.Wrap(err, "mount post overlay dirs") + } + } if vm.Disk.Overlay != nil { if err := d.unmoutOverlay(pod, ctrId, vm); err != nil { return errors.Wrapf(err, "umount overlay") @@ -250,19 +261,19 @@ func (d disk) getOverlayDriver(ov *apis.ContainerVolumeMountDiskOverlay) iDiskOv return d.overlayDrivers[ov.GetType()] } -func (d disk) unmoutOverlay(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { +func (d disk) unmoutOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { return d.getOverlayDriver(vm.Disk.Overlay).unmount(d, pod, ctrId, vm) } -func (d disk) mountOverlay(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { +func (d disk) mountOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { return d.getOverlayDriver(vm.Disk.Overlay).mount(d, pod, ctrId, vm) } func (d disk) doTemplateOverlayAction( ctx context.Context, - pod IPodInfo, ctrId string, + pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, - ovAction func(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error) error { + ovAction func(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error) error { templateId := vm.Disk.TemplateId input := computeapi.CacheImageInput{ ImageId: templateId, @@ -287,65 +298,6 @@ func (d disk) doTemplateOverlayAction( return nil } -func (d disk) InjectUsageTags(usage *ContainerVolumeMountUsage, vol *hostapi.ContainerVolumeMount) { +func (d disk) InjectUsageTags(usage *volume_mount.ContainerVolumeMountUsage, vol *hostapi.ContainerVolumeMount) { usage.Tags["disk_id"] = vol.Disk.Id } - -type diskOverlayDir struct{} - -func newDiskOverlayDir() iDiskOverlay { - return &diskOverlayDir{} -} - -func (dod diskOverlayDir) mount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { - vmDisk := vm.Disk - lowerDir := vmDisk.Overlay.LowerDir - upperDir, err := d.getRuntimeMountHostPath(pod, vm) - if err != nil { - return errors.Wrap(err, "getRuntimeMountHostPath") - } - workDir := d.getOverlayWorkDir(upperDir) - mergedDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) - for _, dir := range []string{workDir, mergedDir} { - out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() - if err != nil { - return errors.Wrapf(err, "make directory %s: %s", dir, out) - } - } - - overlayArgs := []string{"-t", "overlay", "overlay", "-o", fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(lowerDir, ":"), upperDir, workDir), mergedDir} - if out, err := procutils.NewRemoteCommandAsFarAsPossible("mount", overlayArgs...).Output(); err != nil { - return errors.Wrapf(err, "mount %v: %s", overlayArgs, out) - } - - return nil -} - -func (dod diskOverlayDir) unmount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { - upperDir, err := d.getRuntimeMountHostPath(pod, vm) - if err != nil { - return errors.Wrap(err, "getRuntimeMountHostPath") - } - overlayDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) - return container_storage.Unmount(overlayDir) -} - -type diskOverlayImage struct{} - -func newDiskOverlayImage() iDiskOverlay { - return &diskOverlayImage{} -} - -func (di diskOverlayImage) mount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { - if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().mount); err != nil { - return errors.Wrapf(err, "mount template overlay") - } - return nil -} - -func (di diskOverlayImage) unmount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { - if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().unmount); err != nil { - return errors.Wrapf(err, "unmount template overlay") - } - return nil -} diff --git a/pkg/hostman/container/volume_mount/disk/doc.go b/pkg/hostman/container/volume_mount/disk/doc.go new file mode 100644 index 00000000000..c8cf275a58d --- /dev/null +++ b/pkg/hostman/container/volume_mount/disk/doc.go @@ -0,0 +1 @@ +package disk // import "yunion.io/x/onecloud/pkg/hostman/container/volume_mount/disk" diff --git a/pkg/hostman/container/volume_mount/disk/overlay.go b/pkg/hostman/container/volume_mount/disk/overlay.go new file mode 100644 index 00000000000..442d76c34cc --- /dev/null +++ b/pkg/hostman/container/volume_mount/disk/overlay.go @@ -0,0 +1,102 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package disk + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "yunion.io/x/pkg/errors" + + hostapi "yunion.io/x/onecloud/pkg/apis/host" + container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" + "yunion.io/x/onecloud/pkg/util/mountutils" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func (d disk) getOverlayDir(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, oType string, upperDir string) string { + baseDir := strings.TrimPrefix(upperDir, pod.GetVolumesDir()) + return filepath.Join(pod.GetVolumesOverlayDir(), vm.Disk.Id, ctrId, oType, baseDir) +} + +func (d disk) getOverlayWorkDir(upperDir string) string { + return fmt.Sprintf("%s-work", upperDir) +} + +func (d disk) getOverlayMergedDir(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, upperDir string) string { + return d.getOverlayDir(pod, ctrId, vm, "_merged_", upperDir) +} + +type iDiskOverlay interface { + mount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error + unmount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error +} + +type diskOverlayDir struct{} + +func newDiskOverlayDir() iDiskOverlay { + return &diskOverlayDir{} +} + +func (dod diskOverlayDir) mount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + vmDisk := vm.Disk + lowerDir := vmDisk.Overlay.LowerDir + upperDir, err := d.getRuntimeMountHostPath(pod, vm) + if err != nil { + return errors.Wrap(err, "getRuntimeMountHostPath") + } + workDir := d.getOverlayWorkDir(upperDir) + mergedDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) + for _, dir := range []string{workDir, mergedDir} { + out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + if err != nil { + return errors.Wrapf(err, "make directory %s: %s", dir, out) + } + } + + return mountutils.MountOverlay(lowerDir, upperDir, workDir, mergedDir) +} + +func (dod diskOverlayDir) unmount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + upperDir, err := d.getRuntimeMountHostPath(pod, vm) + if err != nil { + return errors.Wrap(err, "getRuntimeMountHostPath") + } + overlayDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) + return container_storage.Unmount(overlayDir) +} + +type diskOverlayImage struct{} + +func newDiskOverlayImage() iDiskOverlay { + return &diskOverlayImage{} +} + +func (di diskOverlayImage) mount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().mount); err != nil { + return errors.Wrapf(err, "mount template overlay") + } + return nil +} + +func (di diskOverlayImage) unmount(d disk, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().unmount); err != nil { + return errors.Wrapf(err, "unmount template overlay") + } + return nil +} diff --git a/pkg/hostman/container/volume_mount/disk/post_overlay.go b/pkg/hostman/container/volume_mount/disk/post_overlay.go new file mode 100644 index 00000000000..87f4a7a6562 --- /dev/null +++ b/pkg/hostman/container/volume_mount/disk/post_overlay.go @@ -0,0 +1,172 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package disk + +import ( + "path/filepath" + "strings" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" + "yunion.io/x/onecloud/pkg/util/mountutils" +) + +const ( + POST_OVERLAY_PREFIX_WORK_DIR = "_post_overlay_work_" + POST_OVERLAY_PREFIX_UPPER_DIR = "_post_overlay_upper_" +) + +func (d disk) MountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error { + return d.newPostOverlay().mountPostOverlays(pod, ctrId, vm, ovs) +} + +func (d disk) UnmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay, clearLayers bool) error { + return d.newPostOverlay().unmountPostOverlays(pod, ctrId, vm, ovs, clearLayers) +} + +type iDiskPostOverlay interface { + mountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error + unmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay, clearLayers bool) error +} + +type diskPostOverlay struct { + disk disk +} + +func newDiskPostOverlay(d disk) iDiskPostOverlay { + return &diskPostOverlay{ + disk: d, + } +} + +func (d diskPostOverlay) mountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error { + for _, ov := range ovs { + if err := d.mountPostOverlay(pod, ctrId, vm, ov); err != nil { + return errors.Wrapf(err, "mount container %s post overlay dir: %#v", ctrId, ov) + } + } + return nil +} + +func (d diskPostOverlay) unmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay, clearLayers bool) error { + for _, ov := range ovs { + if err := d.unmountPostOverlay(pod, ctrId, vm, ov, clearLayers); err != nil { + return errors.Wrapf(err, "unmount container %s post overlay dir: %#v", ctrId, ov) + } + } + return nil +} + +func (d diskPostOverlay) getPostOverlayDirWithPrefix( + prefixDir string, + pod volume_mount.IPodInfo, ctrId string, + vm *hostapi.ContainerVolumeMount, + ov *apis.ContainerVolumeMountDiskPostOverlay, + ensure bool, +) (string, error) { + hostPath, err := d.disk.getHostDiskRootPath(pod, vm) + if err != nil { + return "", errors.Wrap(err, "get host disk root path") + } + workDir := filepath.Join(hostPath, prefixDir, ctrId, ov.ContainerTargetDir) + if ensure { + if err := volume_mount.EnsureDir(workDir); err != nil { + return "", errors.Wrapf(err, "make %s", workDir) + } + } + return workDir, nil +} + +func (d diskPostOverlay) getPostOverlayWorkDir( + pod volume_mount.IPodInfo, ctrId string, + vm *hostapi.ContainerVolumeMount, + ov *apis.ContainerVolumeMountDiskPostOverlay, + ensure bool, +) (string, error) { + return d.getPostOverlayDirWithPrefix(POST_OVERLAY_PREFIX_WORK_DIR, pod, ctrId, vm, ov, ensure) +} + +func (d diskPostOverlay) getPostOverlayUpperDir( + pod volume_mount.IPodInfo, ctrId string, + vm *hostapi.ContainerVolumeMount, + ov *apis.ContainerVolumeMountDiskPostOverlay, + ensure bool, +) (string, error) { + return d.getPostOverlayDirWithPrefix(POST_OVERLAY_PREFIX_UPPER_DIR, pod, ctrId, vm, ov, ensure) +} + +func (d diskPostOverlay) getPostOverlayMountpoint(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) (string, error) { + ctrMountHostPath, err := d.disk.GetRuntimeMountHostPath(pod, ctrId, vm) + if err != nil { + return "", errors.Wrap(err, "get runtime mount host_path") + } + // remove hostPath sub_directory path + ctrMountHostPath = strings.TrimSuffix(ctrMountHostPath, vm.Disk.SubDirectory) + mergedDir := filepath.Join(ctrMountHostPath, ov.ContainerTargetDir) + if err := volume_mount.EnsureDir(mergedDir); err != nil { + return "", errors.Wrap(err, "make merged mountpoint dir") + } + return mergedDir, nil +} + +func (d diskPostOverlay) mountPostOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { + upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) + } + + workDir, err := d.getPostOverlayWorkDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) + } + + mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) + if err != nil { + return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) + } + + return mountutils.MountOverlay(ov.HostLowerDir, upperDir, workDir, mergedDir) +} + +func (d diskPostOverlay) unmountPostOverlay(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, cleanLayers bool) error { + mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) + if err != nil { + return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) + } + if err := mountutils.Unmount(mergedDir); err != nil { + return errors.Wrapf(err, "unmount %s", mergedDir) + } + if cleanLayers { + upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) + } + if err := volume_mount.RemoveDir(upperDir); err != nil { + return errors.Wrap(err, "remove upper dir") + } + + workDir, err := d.getPostOverlayWorkDir(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) + } + if err := volume_mount.RemoveDir(workDir); err != nil { + return errors.Wrap(err, "remove work dir") + } + } + return nil +} diff --git a/pkg/hostman/container/volume_mount/helper.go b/pkg/hostman/container/volume_mount/helper.go index 6020f8a2887..5addb4fc748 100644 --- a/pkg/hostman/container/volume_mount/helper.go +++ b/pkg/hostman/container/volume_mount/helper.go @@ -9,6 +9,22 @@ import ( "yunion.io/x/onecloud/pkg/util/procutils" ) +func EnsureDir(dir string) error { + out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + if err != nil { + return errors.Wrapf(err, "mkdir -p %s: %s", dir, out) + } + return nil +} + +func RemoveDir(dir string) error { + out, err := procutils.NewRemoteCommandAsFarAsPossible("rm", "-rf", dir).Output() + if err != nil { + return errors.Wrapf(err, "rm -rf %s: %s", dir, out) + } + return nil +} + func ChangeDirOwner(pod IPodInfo, drv IVolumeMount, ctrId string, vol *hostapi.ContainerVolumeMount) error { if vol.FsUser == nil && vol.FsGroup == nil { return errors.Errorf("specify fs_user or fs_group") diff --git a/pkg/hostman/container/volume_mount/text.go b/pkg/hostman/container/volume_mount/text.go index a94a34a5b67..f6935f520e2 100644 --- a/pkg/hostman/container/volume_mount/text.go +++ b/pkg/hostman/container/volume_mount/text.go @@ -48,9 +48,8 @@ func (t text) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *hostapi.Co if ti == nil { return "", httperrors.NewNotEmptyError("text is nil") } - out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", pod.GetVolumesDir()).Output() - if err != nil { - return "", errors.Wrapf(err, "mkdir %s: %s", pod.GetVolumesDir(), out) + if err := EnsureDir(pod.GetVolumesDir()); err != nil { + return "", errors.Wrapf(err, "mkdir %s", pod.GetVolumesDir()) } mntPath := filepath.Join(pod.GetVolumesDir(), fmt.Sprintf("%s-%s", ctrId, strings.ReplaceAll(vm.MountPath, "/", "_"))) if err := t.writeContent(ti, mntPath); err != nil { diff --git a/pkg/hostman/guestman/container/runtime.go b/pkg/hostman/guestman/container/runtime.go index 98cc8cf8141..f668db1cc9b 100644 --- a/pkg/hostman/guestman/container/runtime.go +++ b/pkg/hostman/guestman/container/runtime.go @@ -16,12 +16,10 @@ package container import ( "time" - - "yunion.io/x/onecloud/pkg/hostman/guestman/desc" ) // CommandRunner interface allows to run command in a container. type CommandRunner interface { // RunInContainer synchronously executes the command in the container, and returns the output. - RunInContainer(pod *desc.SGuestDesc, containerId string, cmd []string, timeout time.Duration) ([]byte, error) + RunInContainer(podId string, containerId string, cmd []string, timeout time.Duration) ([]byte, error) } diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 6d8d8625351..3b60e842fa0 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -45,6 +45,8 @@ import ( proberesults "yunion.io/x/onecloud/pkg/hostman/container/prober/results" "yunion.io/x/onecloud/pkg/hostman/container/status" "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" + "yunion.io/x/onecloud/pkg/hostman/container/volume_mount/disk" + _ "yunion.io/x/onecloud/pkg/hostman/container/volume_mount/disk" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/hostman/guestman/pod/runtime" deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" @@ -86,10 +88,10 @@ type containerRunner struct { manager *SGuestManager } -func (cr *containerRunner) RunInContainer(pod *desc.SGuestDesc, containerId string, cmd []string, timeout time.Duration) ([]byte, error) { - srv, ok := cr.manager.GetServer(pod.Uuid) +func (cr *containerRunner) RunInContainer(podId string, containerId string, cmd []string, timeout time.Duration) ([]byte, error) { + srv, ok := cr.manager.GetServer(podId) if !ok { - return nil, errors.Wrapf(httperrors.ErrNotFound, "server %s not found", pod.Uuid) + return nil, errors.Wrapf(httperrors.ErrNotFound, "server %s not found", podId) } s := srv.(*sPodGuestInstance) ctrCriId, err := s.getContainerCRIId(containerId) @@ -126,6 +128,8 @@ type PodInstance interface { ContainerExecSync(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerExecSyncInput) (jsonutils.JSONObject, error) SetContainerResourceLimit(ctrId string, limit *apis.ContainerResources) (jsonutils.JSONObject, error) CommitContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerCommitInput) (jsonutils.JSONObject, error) + AddContainerVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerVolumeMountAddPostOverlayInput) error + RemoveContainerVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerVolumeMountRemovePostOverlayInput) error ReadLogs(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.PodLogOptions, stdout, stderr io.Writer) error @@ -284,7 +288,7 @@ func (s *sPodGuestInstance) ImportServer(pendingDelete bool) { } } else { s.SyncStatus("sync status after host started") - s.getProbeManager().AddPod(s.Desc) + s.getProbeManager().AddPod(s) } } @@ -650,7 +654,7 @@ func (s *sPodGuestInstance) GetVolumesDir() string { } func (s *sPodGuestInstance) GetVolumesOverlayDir() string { - return filepath.Join(s.GetVolumesDir(), "overlay") + return filepath.Join(s.GetVolumesDir(), "_overlay_") } func (s *sPodGuestInstance) GetDiskMountPoint(disk storageman.IDisk) string { @@ -886,7 +890,7 @@ func (s *sPodGuestInstance) _startPod(ctx context.Context, userCred mcclient.Tok return nil, errors.Wrapf(err, "set pod %s cgroup memMB %d, cpu %d", criId, s.GetDesc().Mem, s.GetDesc().Cpu) } - s.getProbeManager().AddPod(s.Desc) + s.getProbeManager().AddPod(s) if err := s.startStat.CreatePodFile(); err != nil { return nil, errors.Wrap(err, "startStat.CreatePodFile") } @@ -931,7 +935,7 @@ func (s *sPodGuestInstance) ensurePodRemoved(ctx context.Context, timeout int64) } } - s.getProbeManager().RemovePod(s.Desc) + s.getProbeManager().RemovePod(s) if err := s.startStat.RemovePodFile(); err != nil { return errors.Wrap(err, "startStat.RemovePodFile") } @@ -2412,3 +2416,36 @@ func (s *sPodGuestInstance) CommitContainer(ctx context.Context, userCred mcclie "image_repository": imgRepo, }), nil } + +func (s *sPodGuestInstance) AddContainerVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerVolumeMountAddPostOverlayInput) error { + isRunning, err := s.IsContainerRunning(ctx, ctrId) + if err != nil { + return errors.Wrap(err, "check container is running") + } + if !isRunning { + return nil + } + ctrSpec := s.GetContainerById(ctrId) + vol := ctrSpec.Spec.VolumeMounts[input.Index] + drv := volume_mount.GetDriver(vol.Type) + diskDrv, ok := drv.(disk.IVolumeMountDisk) + if !ok { + return errors.Errorf("invalid disk volume driver of %s", vol.Type) + } + return diskDrv.MountPostOverlays(s, ctrId, vol, input.PostOverlay) +} + +func (s *sPodGuestInstance) RemoveContainerVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerVolumeMountRemovePostOverlayInput) error { + ctrSpec := s.GetContainerById(ctrId) + vol := ctrSpec.Spec.VolumeMounts[input.Index] + drv := volume_mount.GetDriver(vol.Type) + diskDrv, ok := drv.(disk.IVolumeMountDisk) + if !ok { + return errors.Errorf("invalid disk volume driver of %s", vol.Type) + } + // drv.Mount 不会重复挂载，支持重复调用 + if err := drv.Mount(s, ctrId, vol); err != nil { + return errors.Wrapf(err, "mount volume %s, ctrId %s", jsonutils.Marshal(vol), ctrId) + } + return diskDrv.UnmountPostOverlays(s, ctrId, vol, input.PostOverlay, input.ClearLayers) +} diff --git a/pkg/hostman/guestman/podhandlers/podhandlers.go b/pkg/hostman/guestman/podhandlers/podhandlers.go index 3402a8feecd..767d2c7ab53 100644 --- a/pkg/hostman/guestman/podhandlers/podhandlers.go +++ b/pkg/hostman/guestman/podhandlers/podhandlers.go @@ -114,12 +114,14 @@ func _containerActionHandler(cf containerActionFunc, isSync bool, workerMan *app func AddPodHandlers(prefix string, app *appsrv.Application) { ctrHandlers := map[string]containerActionFunc{ - "create": createContainer, - "delete": deleteContainer, - "sync-status": syncContainerStatus, - "pull-image": pullImage, - "save-volume-mount-to-image": saveVolumeMountToImage, - "commit": commitContainer, + "create": createContainer, + "delete": deleteContainer, + "sync-status": syncContainerStatus, + "pull-image": pullImage, + "save-volume-mount-to-image": saveVolumeMountToImage, + "commit": commitContainer, + "add-volume-mount-post-overlay": containerAddVolumeMountPostOverlay, + "remove-volume-mount-post-overlay": containerRemoveVolumeMountPostOverlay, } for action, f := range ctrHandlers { app.AddHandler("POST", @@ -323,3 +325,19 @@ func commitContainer(ctx context.Context, userCred mcclient.TokenCredential, pod } return pod.CommitContainer(ctx, userCred, ctrId, input) } + +func containerAddVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, containerId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { + input := new(compute.ContainerVolumeMountAddPostOverlayInput) + if err := body.Unmarshal(input); err != nil { + return nil, errors.Wrap(err, "unmarshal to ContainerVolumeMountAddPostOverlayInput") + } + return nil, pod.AddContainerVolumeMountPostOverlay(ctx, userCred, containerId, input) +} + +func containerRemoveVolumeMountPostOverlay(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, containerId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { + input := new(compute.ContainerVolumeMountRemovePostOverlayInput) + if err := body.Unmarshal(input); err != nil { + return nil, errors.Wrap(err, "unmarshal to ContainerMountVolumeRemovePostOverlayInput") + } + return nil, pod.RemoveContainerVolumeMountPostOverlay(ctx, userCred, containerId, input) +} diff --git a/pkg/mcclient/options/compute/containers.go b/pkg/mcclient/options/compute/containers.go index 301f250ab85..69b25a8f0da 100644 --- a/pkg/mcclient/options/compute/containers.go +++ b/pkg/mcclient/options/compute/containers.go @@ -451,3 +451,45 @@ func (o *ContainerCommitOptions) Params() (jsonutils.JSONObject, error) { } return jsonutils.Marshal(input), nil } + +type ContainerAddVolumeMountPostOverlayOptions struct { + ServerIdOptions + INDEX int `help:"INDEX of volume mount"` + MountDesc []string `help:"Mount description, :" short-token:"m"` +} + +func (o *ContainerAddVolumeMountPostOverlayOptions) Params() (jsonutils.JSONObject, error) { + input := &computeapi.ContainerVolumeMountAddPostOverlayInput{ + Index: o.INDEX, + PostOverlay: make([]*apis.ContainerVolumeMountDiskPostOverlay, 0), + } + for _, md := range o.MountDesc { + segs := strings.Split(md, ":") + if len(segs) != 2 { + return nil, errors.Errorf("invalid mount description: %s", md) + } + lowerDir := segs[0] + containerTargetDir := segs[1] + input.PostOverlay = append(input.PostOverlay, &apis.ContainerVolumeMountDiskPostOverlay{ + HostLowerDir: []string{lowerDir}, + ContainerTargetDir: containerTargetDir, + }) + } + return jsonutils.Marshal(input), nil +} + +type ContainerRemoveVolumeMountPostOverlayOptions struct { + ContainerAddVolumeMountPostOverlayOptions + ClearLayers bool `help:"clear overlay upper and work layers"` +} + +func (o *ContainerRemoveVolumeMountPostOverlayOptions) Params() (jsonutils.JSONObject, error) { + params, err := o.ContainerAddVolumeMountPostOverlayOptions.Params() + if err != nil { + return nil, err + } + if o.ClearLayers { + params.(*jsonutils.JSONDict).Add(jsonutils.JSONTrue, "clear_layers") + } + return params, nil +} diff --git a/pkg/util/mountutils/mount.go b/pkg/util/mountutils/mount.go index cec8e10de5c..d93a92d7a81 100644 --- a/pkg/util/mountutils/mount.go +++ b/pkg/util/mountutils/mount.go @@ -30,7 +30,7 @@ import ( "yunion.io/x/onecloud/pkg/util/procutils" ) -func Mount(devPath string, mountPoint string, fsType string) error { +func mountWrap(mountPoint string, action func() error) error { if !fileutils2.Exists(mountPoint) { output, err := procutils.NewCommand("mkdir", "-p", mountPoint).Output() if err != nil { @@ -41,12 +41,30 @@ func Mount(devPath string, mountPoint string, fsType string) error { log.Warningf("mountpoint %s is already mounted", mountPoint) return nil } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", "-t", fsType, devPath, mountPoint).Output(); err != nil { - return errors.Wrapf(err, "mount %s to %s with fs %s: %s", devPath, mountPoint, fsType, string(out)) - } - return nil + return action() +} + +func Mount(devPath string, mountPoint string, fsType string) error { + return mountWrap(mountPoint, func() error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", "-t", fsType, devPath, mountPoint).Output(); err != nil { + return errors.Wrapf(err, "mount %s to %s with fs %s: %s", devPath, mountPoint, fsType, string(out)) + } + return nil + }) +} + +func MountOverlay(lowerDir []string, upperDir string, workDir string, mergedDir string) error { + return mountWrap(mergedDir, func() error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + overlayArgs := []string{"-t", "overlay", "overlay", "-o", fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(lowerDir, ":"), upperDir, workDir), mergedDir} + if out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", overlayArgs...).Output(); err != nil { + return errors.Wrapf(err, "mount %v: %s", overlayArgs, out) + } + return nil + }) } func Unmount(mountPoint string) error { @@ -94,13 +112,21 @@ func cleanProcessUseMountPoint(mountPoint string) error { if err != nil { return errors.Wrapf(err, "get mount point devices: %s", mountPoint) } + errs := []error{} for _, dev := range devs { pids, err := useLsofFindDevProcess(dev) if err != nil { - return errors.Wrapf(err, "use lsof find device %q process", dev) + errs = append(errs, errors.Wrapf(err, "use lsof find device %q process", dev)) } - if err := killProcess(pids); err != nil { - return errors.Wrapf(err, "kill process: %v", pids) + if len(pids) > 0 { + if err := killProcess(pids); err != nil { + errs = append(errs, errors.Wrapf(err, "kill process %q", pids)) + return errors.NewAggregate(errs) + } + } else { + if err != nil { + return errors.NewAggregate(errs) + } } } return nil @@ -122,7 +148,10 @@ func killProcess(pids []int) error { func useLsofFindDevProcess(dev string) ([]int, error) { out, err := procutils.NewRemoteCommandAsFarAsPossible("lsof", "+f", "--", dev).Output() if err != nil { - return nil, errors.Wrapf(err, "'lsof +f -- %s' failed: %s", dev, out) + err = errors.Wrapf(err, "'lsof +f -- %s' failed: %s", dev, out) + if len(out) == 0 { + return nil, err + } } pids := sets.NewInt() for _, line := range strings.Split(string(out), "\n") { @@ -146,7 +175,7 @@ func useLsofFindDevProcess(dev string) ([]int, error) { log.Infof("find process %q use device %q", line, dev) pids.Insert(pid) } - return pids.List(), nil + return pids.List(), err } func getMountPointDevices(mountPoint string) ([]string, error) { @@ -166,7 +195,16 @@ func getMountPointDevices(mountPoint string) ([]string, error) { if point != mountPoint { continue } - devs.Insert(parts[0]) + seg1 := parts[0] + var dev string + switch seg1 { + case "sysfs", "proc", "tmpfs", "overlay": + dev = point + default: + dev = seg1 + } + + devs.Insert(dev) } return devs.List(), nil }