feat(region,host): set container resources limit (#21165)

cmd/climc/shell/compute/containers.go

@@ -44,6 +44,7 @@ func init() {
 	cmd.BatchPerform("syncstatus", new(options.ContainerIdsOptions))
 	cmd.Perform("save-volume-mount-image", new(options.ContainerSaveVolumeMountImage))
 	cmd.Perform("exec-sync", new(options.ContainerExecSyncOptions))
+	cmd.BatchPerform("set-resources-limit", new(options.ContainerSetResourcesLimitOptions))
 
 	type UpdateSpecOptions struct {
 		ID string `help:"ID or name of server" json:"-"`

pkg/apis/compute/container.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/gotypes"
+	"yunion.io/x/pkg/util/sets"
 
 	"yunion.io/x/onecloud/pkg/apis"
 )
@@ -78,6 +79,10 @@ const (
 	CONTAINER_STATUS_PROBE_FAILED = "probe_failed"
 )
 
+var (
+	ContainerRunningStatus = sets.NewString(CONTAINER_STATUS_RUNNING, CONTAINER_STATUS_PROBING)
+)
+
 const (
 	CONTAINER_METADATA_CRI_ID           = "cri_id"
 	CONTAINER_METADATA_RELEASED_DEVICES = "released_devices"

pkg/apis/container.go

@@ -48,6 +48,17 @@ type ContainerSecurityContext struct {
 	RunAsGroup *int64 `json:"run_as_group,omitempty"`
 }
 
+type ContainerResources struct {
+	// CpuCfsQuota can be set to 0.5 that mapping to 0.5*100000 for cpu.cpu_cfs_quota_us
+	CpuCfsQuota *float64 `json:"cpu_cfs_quota,omitempty"`
+	// MemoryLimitMB will be transferred to memory.limit_in_bytes
+	// MemoryLimitMB *int64 `json:"memory_limit_mb,omitempty"`
+	// PidsMax will be set to pids.max
+	PidsMax *int `json:"pids_max"`
+	// DevicesAllow will be set to devices.allow
+	DevicesAllow []string `json:"devices_allow"`
+}
+
 type ContainerSpec struct {
 	// Image to use.
 	Image string `json:"image"`
@@ -68,6 +79,7 @@ type ContainerSpec struct {
 	Lifecyle           *ContainerLifecyle        `json:"lifecyle"`
 	CgroupDevicesAllow []string                  `json:"cgroup_devices_allow"`
 	CgroupPidsMax      int                       `json:"cgroup_pids_max"`
+	ResourcesLimit     *ContainerResources       `json:"resources_limit"`
 	SimulateCpu        bool                      `json:"simulate_cpu"`
 	ShmSizeMB          int                       `json:"shm_size_mb"`
 	SecurityContext    *ContainerSecurityContext `json:"security_context,omitempty"`

pkg/compute/guestdrivers/pod.go

@@ -500,15 +500,23 @@ func (p *SPodDriver) RequestExecContainer(ctx context.Context, userCred mcclient
 	return nil
 }
 
-func (p *SPodDriver) RequestExecSyncContainer(ctx context.Context, userCred mcclient.TokenCredential, container *models.SContainer, input *api.ContainerExecSyncInput) (jsonutils.JSONObject, error) {
+func (p *SPodDriver) requestContainerSyncAction(ctx context.Context, userCred mcclient.TokenCredential, container *models.SContainer, action string, input jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	pod := container.GetPod()
 	host, _ := pod.GetHost()
-	url := fmt.Sprintf("%s/pods/%s/containers/%s/exec-sync", host.ManagerUri, pod.GetId(), container.GetId())
+	url := fmt.Sprintf("%s/pods/%s/containers/%s/%s", host.ManagerUri, pod.GetId(), container.GetId(), action)
 	header := mcclient.GetTokenHeaders(userCred)
-	_, ret, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, jsonutils.Marshal(input), false)
+	_, ret, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, input, false)
 	return ret, err
 }
 
+func (p *SPodDriver) RequestExecSyncContainer(ctx context.Context, userCred mcclient.TokenCredential, container *models.SContainer, input *api.ContainerExecSyncInput) (jsonutils.JSONObject, error) {
+	return p.requestContainerSyncAction(ctx, userCred, container, "exec-sync", jsonutils.Marshal(input))
+}
+
+func (p *SPodDriver) RequestSetContainerResourcesLimit(ctx context.Context, userCred mcclient.TokenCredential, container *models.SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error) {
+	return p.requestContainerSyncAction(ctx, userCred, container, "set-resources-limit", jsonutils.Marshal(limit))
+}
+
 func (p *SPodDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error {
 	// clean disk records in DB
 	return guest.DeleteAllDisksInDB(ctx, userCred)

pkg/compute/models/containers.go

@@ -646,6 +646,38 @@ func (c *SContainer) PerformExecSync(ctx context.Context, userCred mcclient.Toke
 	return c.GetPodDriver().RequestExecSyncContainer(ctx, userCred, c, input)
 }
 
+func (c *SContainer) PerformSetResourcesLimit(ctx context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject, limit *apis.ContainerResources) (jsonutils.JSONObject, error) {
+	if err := c.ValidateResourcesLimit(limit); err != nil {
+		return nil, errors.Wrap(err, "ValidateResourcesLimit")
+	}
+	if _, err := db.Update(c, func() error {
+		c.Spec.ResourcesLimit = limit
+		return nil
+	}); err != nil {
+		return nil, errors.Wrap(err, "Update spec.resources_limit")
+	}
+	if !api.ContainerRunningStatus.Has(c.GetStatus()) {
+		return nil, nil
+	}
+	return c.GetPodDriver().RequestSetContainerResourcesLimit(ctx, userCred, c, limit)
+}
+
+func (c *SContainer) ValidateResourcesLimit(limit *apis.ContainerResources) error {
+	if limit == nil {
+		return httperrors.NewInputParameterError("limit cannot be nil")
+	}
+	pod := c.GetPod()
+	if limit.CpuCfsQuota != nil {
+		if *limit.CpuCfsQuota <= 0 {
+			return httperrors.NewInputParameterError("invalid cpu_cfs_quota %f", *limit.CpuCfsQuota)
+		}
+		if *limit.CpuCfsQuota > float64(pod.VcpuCount) {
+			return httperrors.NewInputParameterError("cpu_cfs_quota %f can't large than %d", *limit.CpuCfsQuota, pod.VcpuCount)
+		}
+	}
+	return nil
+}
+
 type ContainerReleasedDevice struct {
 	*api.ContainerDevice
 	DeviceType  string

pkg/compute/models/pod_driver.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -42,4 +43,5 @@ type IPodDriver interface {
 	RequestPullContainerImage(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
 	RequestSaveVolumeMountImage(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
 	RequestExecSyncContainer(ctx context.Context, userCred mcclient.TokenCredential, ctr *SContainer, input *compute.ContainerExecSyncInput) (jsonutils.JSONObject, error)
+	RequestSetContainerResourcesLimit(ctx context.Context, cred mcclient.TokenCredential, c *SContainer, limit *apis.ContainerResources) (jsonutils.JSONObject, error)
 }

pkg/hostman/guestman/pod.go

@@ -116,6 +116,7 @@ type PodInstance interface {
 	SaveVolumeMountToImage(ctx context.Context, userCred mcclient.TokenCredential, input *hostapi.ContainerSaveVolumeMountToImageInput, ctrId string) (jsonutils.JSONObject, error)
 	ExecContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerExecInput) (*url.URL, error)
 	ContainerExecSync(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerExecSyncInput) (jsonutils.JSONObject, error)
+	SetContainerResourceLimit(ctrId string, limit *apis.ContainerResources) (jsonutils.JSONObject, error)
 
 	ReadLogs(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.PodLogOptions, stdout, stderr io.Writer) error
 }
@@ -930,6 +931,11 @@ func (s *sPodGuestInstance) StartContainer(ctx context.Context, userCred mcclien
 	if err := s.startStat.CreateContainerFile(ctrId); err != nil {
 		return nil, errors.Wrapf(err, "create container startup stat file %s", ctrId)
 	}
+	if input.Spec.ResourcesLimit != nil {
+		if err := s.setContainerResourcesLimit(criId, input.Spec.ResourcesLimit); err != nil {
+			return nil, errors.Wrap(err, "set container resources limit")
+		}
+	}
 	return nil, nil
 }
 
@@ -1201,6 +1207,40 @@ func (s *sPodGuestInstance) setContainerCgroupDevicesAllow(ctrId string, allowSt
 	return s.getCGUtil().SetDevicesAllow(ctrId, allowStrs)
 }
 
+func (s *sPodGuestInstance) SetContainerResourceLimit(ctrId string, limit *apis.ContainerResources) (jsonutils.JSONObject, error) {
+	criId, err := s.getContainerCRIId(ctrId)
+	if err != nil {
+		return nil, errors.Wrap(err, "get container cri id")
+	}
+	return nil, s.setContainerResourcesLimit(criId, limit)
+}
+
+func (s *sPodGuestInstance) setContainerResourcesLimit(ctrId string, limit *apis.ContainerResources) error {
+	cgUtil := s.getCGUtil()
+	/*if limit.MemoryLimitMB != nil {
+		if err := cgUtil.SetMemoryLimitBytes(ctrId, *limit.MemoryLimitMB*1024*1024); err != nil {
+			return errors.Wrapf(err, "set memory limit to %d MB", *limit.MemoryLimitMB)
+		}
+	}*/
+	if limit.CpuCfsQuota != nil {
+		cpuCfsQuotaUs := *limit.CpuCfsQuota * float64(s.getDefaultCPUPeriod())
+		if err := cgUtil.SetCPUCfs(ctrId, int64(cpuCfsQuotaUs), s.getDefaultCPUPeriod()); err != nil {
+			return errors.Wrapf(err, "set cpu cfs quota to %d", int64(cpuCfsQuotaUs))
+		}
+	}
+	if limit.PidsMax != nil {
+		if err := cgUtil.SetPidsMax(ctrId, *limit.PidsMax); err != nil {
+			return errors.Wrapf(err, "set pids.max to %d", *limit.PidsMax)
+		}
+	}
+	if len(limit.DevicesAllow) != 0 {
+		if err := cgUtil.SetDevicesAllow(ctrId, limit.DevicesAllow); err != nil {
+			return errors.Wrapf(err, "set devices.allow %v", limit.DevicesAllow)
+		}
+	}
+	return nil
+}
+
 func (s *sPodGuestInstance) getDefaultCPUPeriod() int64 {
 	return 100000
 }

pkg/hostman/guestman/podhandlers/podhandlers.go

@@ -27,6 +27,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/appsrv"
@@ -146,6 +147,9 @@ func AddPodHandlers(prefix string, app *appsrv.Application) {
 
 	logWorker := appsrv.NewWorkerManager("container-log-worker", 64, appsrv.DEFAULT_BACKLOG, false)
 	app.AddHandler3(newContainerWorkerHandler("GET", fmt.Sprintf("%s/pods/%s/containers/%s/log", prefix, POD_ID, CONTAINER_ID), logWorker, logContainer()))
+
+	syncWorker := appsrv.NewWorkerManager("container-sync-action-worker", 16, appsrv.DEFAULT_BACKLOG, false)
+	app.AddHandler3(newContainerWorkerHandler("POST", fmt.Sprintf("%s/pods/%s/containers/%s/set-resources-limit", prefix, POD_ID, CONTAINER_ID), syncWorker, containerSyncActionHandler(containerSetResourcesLimit)))
 }
 
 func pullImage(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
@@ -297,3 +301,11 @@ func containerExecSync(ctx context.Context, userCred mcclient.TokenCredential, p
 	}
 	return pod.ContainerExecSync(ctx, userCred, containerId, input)
 }
+
+func containerSetResourcesLimit(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, containerId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	input := new(apis.ContainerResources)
+	if err := body.Unmarshal(input); err != nil {
+		return nil, errors.Wrap(err, "unmarshal to ContainerExecSyncInput")
+	}
+	return pod.SetContainerResourceLimit(containerId, input)
+}

pkg/mcclient/options/compute/containers.go

@@ -326,6 +326,32 @@ func (o *ContainerExecOptions) Params() (jsonutils.JSONObject, error) {
 	return jsonutils.Marshal(o.ToAPIInput()), nil
 }
 
+type ContainerSetResourcesLimitOptions struct {
+	ContainerIdsOptions
+	CpuCfsQuota float64 `help:"cpu cfs quota. e.g.:0.5 equals 0.5*100000"`
+	//MemoryLimitMb int64    `help:"memory limit MB"`
+	PidsMax     int      `help:"pids max"`
+	DeviceAllow []string `help:"devices allow"`
+}
+
+func (o *ContainerSetResourcesLimitOptions) Params() (jsonutils.JSONObject, error) {
+	limit := &apis.ContainerResources{}
+	if o.CpuCfsQuota > 0 {
+		limit.CpuCfsQuota = &o.CpuCfsQuota
+	}
+	//if o.MemoryLimitMb > 0 {
+	//	limit.MemoryLimitMB = &o.MemoryLimitMb
+	//}
+	if o.PidsMax > 0 {
+		limit.PidsMax = &o.PidsMax
+	}
+	if len(o.DeviceAllow) > 0 {
+		limit.DevicesAllow = o.DeviceAllow
+	}
+
+	return jsonutils.Marshal(limit), nil
+}
+
 type ContainerExecSyncOptions struct {
 	ServerIdOptions
 	COMMAND string