diff --git a/build/docker/Dockerfile.host-deployer b/build/docker/Dockerfile.host-deployer index 948cf9e3d2d..6ae86ad6b58 100644 --- a/build/docker/Dockerfile.host-deployer +++ b/build/docker/Dockerfile.host-deployer @@ -1,9 +1,19 @@ -FROM registry.cn-beijing.aliyuncs.com/yunionio/host-deployer-base:1.5 +FROM registry.cn-beijing.aliyuncs.com/yunionio/yunionos:v0.1.9-20231129.0 as yunionos +FROM registry.cn-beijing.aliyuncs.com/yunionio/host-deployer-base:1.4.2 MAINTAINER "Yaoqi Wan wanyaoqi@yunionyun.com" ENV TZ UTC RUN mkdir -p /opt/yunion/bin +RUN mkdir -p /yunionos/x86_64 +RUN mkdir -p /yunionos/aarch64 + + +COPY --from=yunionos /yunionos/x86_64/initramfs /yunionos/x86_64/initramfs +COPY --from=yunionos /yunionos/x86_64/kernel /yunionos/x86_64/kernel +COPY --from=yunionos /yunionos/aarch64/kernel /yunionos/aarch64/kernel +COPY --from=yunionos /yunionos/aarch64/initramfs /yunionos/aarch64/initramfs + ADD ./_output/centos-build/bin/host-deployer /opt/yunion/bin/host-deployer diff --git a/cmd/climc/shell/compute/servers.go b/cmd/climc/shell/compute/servers.go index 9f690101e25..79ce72ca99d 100644 --- a/cmd/climc/shell/compute/servers.go +++ b/cmd/climc/shell/compute/servers.go @@ -123,6 +123,8 @@ func init() { cmd.Perform("set-nic-traffic-limit", &options.ServerNicTrafficLimitOptions{}) cmd.Perform("add-sub-ips", &options.ServerAddSubIpsOptions{}) cmd.BatchPerform("set-os-info", &options.ServerSetOSInfoOptions{}) + cmd.BatchPerform("start-rescue", &options.ServerStartOptions{}) + cmd.BatchPerform("stop-rescue", &options.ServerStartOptions{}) cmd.Get("vnc", new(options.ServerVncOptions)) cmd.Get("desc", new(options.ServerIdOptions)) @@ -953,46 +955,4 @@ func init() { } return nil }) - - // ServerStartRescueOptions is used to start a rescue os. - type ServerStartRescueOptions struct { - ID string `help:"ID of server" json:"-"` - QemuVersion string `help:"prefer qemu version" json:"qemu_version"` - } - R(&ServerStartRescueOptions{}, "server-start-rescue ", "Start rescu e a guest server", func(s *mcclient.ClientSession, opts *ServerStartRescueOptions) error { - params, err := baseoptions.StructToParams(opts) - if err != nil { - return err - } - - result, err := modules.Servers.PerformAction(s, opts.ID, "start-rescue", params) - if err != nil { - return err - } - - printObject(result) - - return nil - }) - - // ServerStopRescueOptions is used to stop a rescue os. - type ServerStopRescueOptions struct { - ID string `help:"ID of server" json:"-"` - QemuVersion string `help:"prefer qemu version" json:"qemu_version"` - } - R(&ServerStopRescueOptions{}, "server-stop-rescue", "Stop rescue a guest server", func(s *mcclient.ClientSession, opts *ServerStopRescueOptions) error { - params, err := baseoptions.StructToParams(opts) - if err != nil { - return err - } - - result, err := modules.Servers.PerformAction(s, opts.ID, "stop-rescue", params) - if err != nil { - return err - } - - printObject(result) - - return nil - }) } diff --git a/pkg/apis/compute/guests.go b/pkg/apis/compute/guests.go index 373c1dd0012..224c3a6fca8 100644 --- a/pkg/apis/compute/guests.go +++ b/pkg/apis/compute/guests.go @@ -884,7 +884,7 @@ type GuestJsonDesc struct { IsDaemon bool `json:"is_daemon"` - RescueMode bool `json:"rescue_mode"` + LightMode bool `json:"light_mode"` } type ServerSetBootIndexInput struct { diff --git a/pkg/cloudcommon/db/opslog_const.go b/pkg/cloudcommon/db/opslog_const.go index 8f91c3494af..b03546d9abd 100644 --- a/pkg/cloudcommon/db/opslog_const.go +++ b/pkg/cloudcommon/db/opslog_const.go @@ -319,9 +319,8 @@ const ( ACT_SYNC_TRAFFIC_LIMIT = "sync_traffic_limit" ACT_SYNC_TRAFFIC_LIMIT_FAIL = "sync_traffic_limit_fail" - - ACT_BIND = "bind" - ACT_UNBIND = "unbind" + ACT_BIND = "bind" + ACT_UNBIND = "unbind" ACT_START_RESCUE = "start_rescue" ACT_STOP_RESCUE = "stop_rescue" diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 78f08a192bc..39a595dc234 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -546,14 +546,14 @@ func (drv *SBaseGuestDriver) SyncOsInfo(ctx context.Context, userCred mcclient.T return nil } -func (drv *SBaseGuestDriver) RequestStartRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) error { - return httperrors.ErrNotImplemented +func (self *SBaseGuestDriver) ValidateSetOSInfo(ctx context.Context, userCred mcclient.TokenCredential, _ *models.SGuest, _ *api.ServerSetOSInfoInput) error { + return nil } -func (drv *SBaseGuestDriver) RequestStopRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) error { +func (self *SBaseGuestDriver) RequestStartRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) error { return httperrors.ErrNotImplemented } -func (self *SBaseGuestDriver) ValidateSetOSInfo(ctx context.Context, userCred mcclient.TokenCredential, _ *models.SGuest, _ *api.ServerSetOSInfoInput) error { - return nil +func (self *SBaseGuestDriver) RequestStopRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) error { + return httperrors.ErrNotImplemented } diff --git a/pkg/compute/models/guest_rescue.go b/pkg/compute/models/guest_rescue.go index 743dd3cd330..11908439c4c 100644 --- a/pkg/compute/models/guest_rescue.go +++ b/pkg/compute/models/guest_rescue.go @@ -39,21 +39,6 @@ func (self *SGuest) PerformStartRescue(ctx context.Context, userCred mcclient.To return nil, httperrors.NewInvalidStatusError("vmem size must be greater than 2G") } - // Reset index - disks, err := self.GetGuestDisks() - if err != nil || len(disks) < 1 { - return nil, httperrors.NewInvalidStatusError("guest.GetGuestDisks: %s", err.Error()) - } - for i := 0; i < len(disks); i++ { - if disks[i].BootIndex >= 0 { - // Move to next index, and easy to rollback - err = disks[i].SetBootIndex(disks[i].BootIndex + 1) - if err != nil { - return nil, httperrors.NewInvalidStatusError("guest.SetBootIndex: %s", err.Error()) - } - } - } - // Get baremetal agent host, err := self.GetHost() if err != nil { @@ -84,23 +69,8 @@ func (self *SGuest) PerformStopRescue(ctx context.Context, userCred mcclient.Tok return nil, httperrors.NewInvalidStatusError("guest is not in rescue mode") } - // Recover index - disks, err := self.GetGuestDisks() - if err != nil || len(disks) < 1 { - return nil, httperrors.NewInvalidStatusError("guest.GetGuestDisks: %s", err.Error()) - } - for i := 0; i < len(disks); i++ { - if disks[i].BootIndex >= 0 { - // Rollback index - err = disks[i].SetBootIndex(disks[i].BootIndex - 1) - if err != nil { - return nil, httperrors.NewInvalidStatusError("guest.SetBootIndex: %s", err.Error()) - } - } - } - // Start rescue vm task - err = self.StopRescueTask(ctx, userCred, data.(*jsonutils.JSONDict), "") + err := self.StopRescueTask(ctx, userCred, data.(*jsonutils.JSONDict), "") if err != nil { return nil, httperrors.NewInvalidStatusError("guest.StopGuestRescueTask: %s", err.Error()) } @@ -115,7 +85,7 @@ func (self *SGuest) UpdateRescueMode(mode bool) error { return nil }) if err != nil { - return errors.Wrap(err, "Update RescueMode") + return errors.Wrap(err, "Update LightMode") } return nil } diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 0dec8764fbf..9ef4e99605d 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -245,10 +245,9 @@ type IGuestDriver interface { SyncOsInfo(ctx context.Context, userCred mcclient.TokenCredential, g *SGuest, extVM cloudprovider.IOSInfo) error + ValidateSetOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, input *api.ServerSetOSInfoInput) error RequestStartRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *SHost, guest *SGuest) error RequestStopRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *SHost, guest *SGuest) error - - ValidateSetOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, input *api.ServerSetOSInfoInput) error } var guestDrivers map[string]IGuestDriver diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index fa0c3662b13..1dc51fa4e56 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -4942,7 +4942,7 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *a IsDaemon: self.IsDaemon.Bool(), - RescueMode: self.RescueMode, + LightMode: self.RescueMode, } if len(self.BackupHostId) > 0 { diff --git a/pkg/hostman/diskutils/deploy_iface/deployer.go b/pkg/hostman/diskutils/deploy_iface/deployer.go new file mode 100644 index 00000000000..6b56ff53c75 --- /dev/null +++ b/pkg/hostman/diskutils/deploy_iface/deployer.go @@ -0,0 +1,42 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deploy_iface + +import ( + "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" + "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" +) + +type IDeployer interface { + Connect(desc *apis.GuestDesc) error + Disconnect() error + + GetPartitions() []fsdriver.IDiskPartition + IsLVMPartition() bool + Zerofree() + ResizePartition() error + FormatPartition(fs, uuid string) error + MakePartition(fs string) error + + MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error) + UmountRootfs(fd fsdriver.IRootFsDriver) error + DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool + + DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) + ResizeFs() (res *apis.Empty, err error) + FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) + SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) + ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) +} diff --git a/pkg/hostman/diskutils/deploy_iface/doc.go b/pkg/hostman/diskutils/deploy_iface/doc.go new file mode 100644 index 00000000000..d961433acc0 --- /dev/null +++ b/pkg/hostman/diskutils/deploy_iface/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deploy_iface // import "yunion.io/x/onecloud/pkg/hostman/diskutils/deploy_iface" diff --git a/pkg/hostman/diskutils/fsutils/fsutils.go b/pkg/hostman/diskutils/fsutils/fsutils.go index bb6aafacdfd..a787d463d93 100644 --- a/pkg/hostman/diskutils/fsutils/fsutils.go +++ b/pkg/hostman/diskutils/fsutils/fsutils.go @@ -23,11 +23,14 @@ import ( "strconv" "strings" - "github.com/pkg/errors" - "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/hostman/diskutils/deploy_iface" + "yunion.io/x/onecloud/pkg/hostman/guestfs" + "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" + "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/procutils" "yunion.io/x/onecloud/pkg/util/regutils2" @@ -44,7 +47,7 @@ func IsPartedFsString(fsstr string) bool { }) } -func ParseDiskPartition(dev string, lines []byte) ([][]string, string) { +func ParseDiskPartition(dev string, lines []string) ([][]string, string) { var ( parts = [][]string{} label string @@ -52,7 +55,7 @@ func ParseDiskPartition(dev string, lines []byte) ([][]string, string) { partten = regexp.MustCompile(`(?P\d+)\s+(?P\d+)s\s+(?P\d+)s\s+(?P\d+)s`) ) - for _, line := range strings.Split(string(lines), "\n") { + for _, line := range lines { if len(label) == 0 { m := regutils2.GetParams(labelPartten, line) if len(m) > 0 { @@ -135,7 +138,7 @@ func ResizeDiskFs(diskPath string, sizeMb int) error { log.Errorf("resize disk fs fail, output: %s , error: %s", lines, err) return err } - parts, label := ParseDiskPartition(diskPath, lines) + parts, label := ParseDiskPartition(diskPath, strings.Split(string(lines), "\n")) log.Infof("Parts: %v label: %s", parts, label) maxSector := GetDevSector512Count(path.Base(diskPath)) if label == "gpt" { @@ -199,7 +202,7 @@ func ResizeDiskFs(diskPath string, sizeMb int) error { if label == "msdos" && end >= 4294967296 { end = 4294967295 } - if isSupportResizeFs(part[6]) { + if IsSupportResizeFs(part[6]) { cmds := []string{"parted", "-a", "none", "-s", diskPath, "--", "resizepart", part[0], fmt.Sprintf("%ds", end)} log.Infof("resize disk partition: %s", cmds) output, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output() @@ -215,7 +218,7 @@ func ResizeDiskFs(diskPath string, sizeMb int) error { return nil } -func isSupportResizeFs(fs string) bool { +func IsSupportResizeFs(fs string) bool { if strings.HasPrefix(fs, "linux-swap") { return true } else if strings.HasPrefix(fs, "ext") { @@ -406,3 +409,187 @@ func FormatPartition(path, fs, uuid string) error { } return fmt.Errorf("Unknown fs %s", fs) } + +func DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver, partitions []fsdriver.IDiskPartition) bool { + for i := 0; i < len(partitions); i++ { + if partitions[i].IsMounted() { + if rootfs.DetectIsUEFISupport(partitions[i]) { + return true + } + } else { + if partitions[i].Mount() { + support := rootfs.DetectIsUEFISupport(partitions[i]) + if err := partitions[i].Umount(); err != nil { + log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err) + } + if support { + return true + } + } + } + } + return false +} + +func MountRootfs(readonly bool, partitions []fsdriver.IDiskPartition) (fsdriver.IRootFsDriver, error) { + errs := []error{} + for i := 0; i < len(partitions); i++ { + log.Infof("detect partition %s", partitions[i].GetPartDev()) + mountFunc := partitions[i].Mount + if readonly { + mountFunc = partitions[i].MountPartReadOnly + } + if mountFunc() { + fs, err := guestfs.DetectRootFs(partitions[i]) + if err == nil { + log.Infof("Use rootfs %s, partition %s", fs, partitions[i].GetPartDev()) + return fs, nil + } + errs = append(errs, err) + if err := partitions[i].Umount(); err != nil { + log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err) + } + } + } + if len(partitions) == 0 { + return nil, errors.Wrap(errors.ErrNotFound, "not found any partition") + } + var err error = errors.ErrNotFound + if len(errs) > 0 { + err = errors.Wrapf(errors.ErrNotFound, errors.NewAggregate(errs).Error()) + } + return nil, err +} + +func DeployGuestfs(d deploy_iface.IDeployer, req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) { + root, err := d.MountRootfs(false) + if err != nil { + if errors.Cause(err) == errors.ErrNotFound && req.DeployInfo.IsInit { + // if init deploy, ignore no partition error + log.Errorf("disk.MountRootfs not found partition, not init, quit") + return nil, nil + } + log.Errorf("Failed mounting rootfs for %s disk: %s", req.GuestDesc.Hypervisor, err) + return nil, err + } + defer d.UmountRootfs(root) + ret, err := guestfs.DoDeployGuestFs(root, req.GuestDesc, req.DeployInfo) + if err != nil { + log.Errorf("guestfs.DoDeployGuestFs fail %s", err) + return nil, err + } + if ret == nil { + log.Errorf("guestfs.DoDeployGuestFs return empty results") + return nil, nil + } + return ret, nil +} + +func ResizeFs(d deploy_iface.IDeployer) (*apis.Empty, error) { + unmount := func(root fsdriver.IRootFsDriver) error { + err := d.UmountRootfs(root) + if err != nil { + return errors.Wrap(err, "unmount rootfs") + } + return nil + } + + root, err := d.MountRootfs(false) + if err != nil { + if errors.Cause(err) == errors.ErrNotFound { + return new(apis.Empty), nil + } + return new(apis.Empty), errors.Wrapf(err, "disk.MountRootfs") + } + if !root.IsResizeFsPartitionSupport() { + err := unmount(root) + if err != nil { + return new(apis.Empty), err + } + return new(apis.Empty), errors.ErrNotSupported + } + + // must umount rootfs before resize partition + err = unmount(root) + if err != nil { + return new(apis.Empty), err + } + err = d.ResizePartition() + if err != nil { + return new(apis.Empty), errors.Wrap(err, "resize disk partition") + } + return new(apis.Empty), nil +} + +func FormatFs(d deploy_iface.IDeployer, req *apis.FormatFsParams) (*apis.Empty, error) { + err := d.MakePartition(req.FsFormat) + if err != nil { + return new(apis.Empty), errors.Wrap(err, "MakePartition") + } + err = d.FormatPartition(req.FsFormat, req.Uuid) + if err != nil { + return new(apis.Empty), errors.Wrap(err, "FormatPartition") + } + return new(apis.Empty), nil +} + +func SaveToGlance(d deploy_iface.IDeployer, req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) { + var ( + osInfo string + relInfo *apis.ReleaseInfo + ) + + ret := &apis.SaveToGlanceResponse{ + OsInfo: osInfo, + ReleaseInfo: relInfo, + } + + root, err := d.MountRootfs(false) + if err != nil { + if errors.Cause(err) == errors.ErrNotFound { + return ret, nil + } + return ret, errors.Wrapf(err, "MountKvmRootfs") + } + defer d.UmountRootfs(root) + + osInfo = root.GetOs() + relInfo = root.GetReleaseInfo(root.GetPartition()) + if req.Compress { + err = root.PrepareFsForTemplate(root.GetPartition()) + if err != nil { + log.Errorf("PrepareFsForTemplate %s", err) + } + } + if req.Compress { + d.Zerofree() + } + return ret, err +} + +func ProbeImageInfo(d deploy_iface.IDeployer) (*apis.ImageInfo, error) { + // Fsck is executed during mount + rootfs, err := d.MountRootfs(false) + if err != nil { + if errors.Cause(err) == errors.ErrNotFound { + return new(apis.ImageInfo), nil + } + return new(apis.ImageInfo), errors.Wrapf(err, "d.MountKvmRootfs") + } + partition := rootfs.GetPartition() + imageInfo := &apis.ImageInfo{ + OsInfo: rootfs.GetReleaseInfo(partition), + OsType: rootfs.GetOs(), + IsLvmPartition: d.IsLVMPartition(), + IsReadonly: partition.IsReadonly(), + IsInstalledCloudInit: rootfs.IsCloudinitInstall(), + } + d.UmountRootfs(rootfs) + + // In case of deploy driver is guestfish, we can't mount + // multi partition concurrent, so we need umount rootfs first + imageInfo.IsUefiSupport = d.DetectIsUEFISupport(rootfs) + imageInfo.PhysicalPartitionType = partition.GetPhysicalPartitionType() + log.Infof("ProbeImageInfo response %s", imageInfo) + return imageInfo, nil +} diff --git a/pkg/hostman/diskutils/interface.go b/pkg/hostman/diskutils/interface.go index cb422e782cb..a3f4cf48c7c 100644 --- a/pkg/hostman/diskutils/interface.go +++ b/pkg/hostman/diskutils/interface.go @@ -22,12 +22,17 @@ import ( ) type IDisk interface { - Connect() error + Connect(desc *apis.GuestDesc) error Disconnect() error MountRootfs() (fsdriver.IRootFsDriver, error) UmountRootfs(driver fsdriver.IRootFsDriver) error - ResizePartition() error Cleanup() + + DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) + ResizeFs() (res *apis.Empty, err error) + FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) + SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) + ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) } type DiskParams struct { @@ -48,15 +53,3 @@ func GetIDisk(params DiskParams, driver string, readOnly bool) (IDisk, error) { return NewKVMGuestDisk(params.DiskInfo, driver, readOnly) } } - -type IDeployer interface { - Connect() error - Disconnect() error - - GetPartitions() []fsdriver.IDiskPartition - IsLVMPartition() bool - Zerofree() - ResizePartition() error - FormatPartition(fs, uuid string) error - MakePartition(fs string) error -} diff --git a/pkg/hostman/diskutils/kvm.go b/pkg/hostman/diskutils/kvm.go index 8834a534827..d8f5b9f2600 100644 --- a/pkg/hostman/diskutils/kvm.go +++ b/pkg/hostman/diskutils/kvm.go @@ -23,10 +23,12 @@ import ( "yunion.io/x/pkg/errors" cloudconsts "yunion.io/x/onecloud/pkg/cloudcommon/consts" + "yunion.io/x/onecloud/pkg/hostman/diskutils/deploy_iface" "yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs" "yunion.io/x/onecloud/pkg/hostman/diskutils/nbd" - "yunion.io/x/onecloud/pkg/hostman/guestfs" + "yunion.io/x/onecloud/pkg/hostman/diskutils/qemu_kvm" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" + "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" "yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts" "yunion.io/x/onecloud/pkg/util/qemuimg" ) @@ -35,7 +37,7 @@ type SKVMGuestDisk struct { readOnly bool kvmImagePath string topImagePath string - deployer IDeployer + deployer deploy_iface.IDeployer } func NewKVMGuestDisk(imageInfo qemuimg.SImageInfo, driver string, readOnly bool) (*SKVMGuestDisk, error) { @@ -78,12 +80,19 @@ func (d *SKVMGuestDisk) Cleanup() { } } -func newDeployer(imageInfo qemuimg.SImageInfo, driver string) IDeployer { +var _ deploy_iface.IDeployer = (*qemu_kvm.QemuKvmDriver)(nil) +var _ deploy_iface.IDeployer = (*qemu_kvm.LocalDiskDriver)(nil) + +func newDeployer(imageInfo qemuimg.SImageInfo, driver string) deploy_iface.IDeployer { switch driver { case consts.DEPLOY_DRIVER_NBD: return nbd.NewNBDDriver(imageInfo) case consts.DEPLOY_DRIVER_LIBGUESTFS: return libguestfs.NewLibguestfsDriver(imageInfo) + case consts.DEPLOY_DRIVER_QEMU_KVM: + return qemu_kvm.NewQemuKvmDriver(imageInfo) + case consts.DEPLOY_DRIVER_LOCAL_DISK: + return qemu_kvm.NewLocalDiskDriver() default: return nbd.NewNBDDriver(imageInfo) } @@ -93,36 +102,14 @@ func (d *SKVMGuestDisk) IsLVMPartition() bool { return d.deployer.IsLVMPartition() } -func (d *SKVMGuestDisk) Connect() error { - return d.deployer.Connect() +func (d *SKVMGuestDisk) Connect(guestDesc *apis.GuestDesc) error { + return d.deployer.Connect(guestDesc) } func (d *SKVMGuestDisk) Disconnect() error { return d.deployer.Disconnect() } -func (d *SKVMGuestDisk) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool { - partitions := d.deployer.GetPartitions() - for i := 0; i < len(partitions); i++ { - if partitions[i].IsMounted() { - if rootfs.DetectIsUEFISupport(partitions[i]) { - return true - } - } else { - if partitions[i].Mount() { - support := rootfs.DetectIsUEFISupport(partitions[i]) - if err := partitions[i].Umount(); err != nil { - log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err) - } - if support { - return true - } - } - } - } - return false -} - func (d *SKVMGuestDisk) MountRootfs() (fsdriver.IRootFsDriver, error) { return d.MountKvmRootfs() } @@ -132,34 +119,7 @@ func (d *SKVMGuestDisk) MountKvmRootfs() (fsdriver.IRootFsDriver, error) { } func (d *SKVMGuestDisk) mountKvmRootfs(readonly bool) (fsdriver.IRootFsDriver, error) { - partitions := d.deployer.GetPartitions() - errs := []error{} - for i := 0; i < len(partitions); i++ { - log.Infof("detect partition %s", partitions[i].GetPartDev()) - mountFunc := partitions[i].Mount - if readonly { - mountFunc = partitions[i].MountPartReadOnly - } - if mountFunc() { - fs, err := guestfs.DetectRootFs(partitions[i]) - if err == nil { - log.Infof("Use rootfs %s, partition %s", fs, partitions[i].GetPartDev()) - return fs, nil - } - errs = append(errs, err) - if err := partitions[i].Umount(); err != nil { - log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err) - } - } - } - if len(partitions) == 0 { - return nil, errors.Wrap(errors.ErrNotFound, "not found any partition") - } - var err error = errors.ErrNotFound - if len(errs) > 0 { - err = errors.Wrapf(errors.ErrNotFound, errors.NewAggregate(errs).Error()) - } - return nil, err + return d.deployer.MountRootfs(readonly) } func (d *SKVMGuestDisk) MountKvmRootfsReadOnly() (fsdriver.IRootFsDriver, error) { @@ -180,18 +140,22 @@ func (d *SKVMGuestDisk) UmountRootfs(fd fsdriver.IRootFsDriver) error { return d.UmountKvmRootfs(fd) } -func (d *SKVMGuestDisk) MakePartition(fs string) error { - return d.deployer.MakePartition(fs) +func (d *SKVMGuestDisk) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) { + return d.deployer.DeployGuestfs(req) +} + +func (d *SKVMGuestDisk) ResizeFs() (*apis.Empty, error) { + return d.deployer.ResizeFs() } -func (d *SKVMGuestDisk) FormatPartition(fs, uuid string) error { - return d.deployer.FormatPartition(fs, uuid) +func (d *SKVMGuestDisk) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) { + return d.deployer.FormatFs(req) } -func (d *SKVMGuestDisk) ResizePartition() error { - return d.deployer.ResizePartition() +func (d *SKVMGuestDisk) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) { + return d.deployer.SaveToGlance(req) } -func (d *SKVMGuestDisk) Zerofree() { - d.deployer.Zerofree() +func (d *SKVMGuestDisk) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) { + return d.deployer.ProbeImageInfo(req) } diff --git a/pkg/hostman/diskutils/libguestfs/driver.go b/pkg/hostman/diskutils/libguestfs/driver.go index b37923dc4cc..6c8355cb299 100644 --- a/pkg/hostman/diskutils/libguestfs/driver.go +++ b/pkg/hostman/diskutils/libguestfs/driver.go @@ -33,6 +33,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/guestfs/guestfishpart" "yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart" + "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/qemuimg" ) @@ -73,7 +74,7 @@ func NewLibguestfsDriver(imageInfo qemuimg.SImageInfo) *SLibguestfsDriver { } } -func (d *SLibguestfsDriver) Connect() error { +func (d *SLibguestfsDriver) Connect(*apis.GuestDesc) error { fish, err := guestfsManager.AcquireFish() if err != nil { return err @@ -250,3 +251,38 @@ func (d *SLibguestfsDriver) MakePartition2(fsFormat string) error { } return nil } + +func (d *SLibguestfsDriver) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool { + return fsutils.DetectIsUEFISupport(rootfs, d.GetPartitions()) +} + +func (d *SLibguestfsDriver) MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error) { + return fsutils.MountRootfs(readonly, d.GetPartitions()) +} + +func (d *SLibguestfsDriver) UmountRootfs(fd fsdriver.IRootFsDriver) error { + if part := fd.GetPartition(); part != nil { + return part.Umount() + } + return nil +} + +func (d *SLibguestfsDriver) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) { + return fsutils.DeployGuestfs(d, req) +} + +func (d *SLibguestfsDriver) ResizeFs() (*apis.Empty, error) { + return fsutils.ResizeFs(d) +} + +func (d *SLibguestfsDriver) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) { + return fsutils.SaveToGlance(d, req) +} + +func (d *SLibguestfsDriver) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) { + return fsutils.FormatFs(d, req) +} + +func (d *SLibguestfsDriver) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) { + return fsutils.ProbeImageInfo(d) +} diff --git a/pkg/hostman/diskutils/nbd/driver.go b/pkg/hostman/diskutils/nbd/driver.go index 47692d729bd..6d984e2b0e2 100644 --- a/pkg/hostman/diskutils/nbd/driver.go +++ b/pkg/hostman/diskutils/nbd/driver.go @@ -30,6 +30,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/diskutils/fsutils" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart" + "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" "yunion.io/x/onecloud/pkg/util/procutils" "yunion.io/x/onecloud/pkg/util/qemuimg" "yunion.io/x/onecloud/pkg/util/qemutils" @@ -58,7 +59,7 @@ func init() { lock = new(sync.Mutex) } -func (d *NBDDriver) Connect() error { +func (d *NBDDriver) Connect(*apis.GuestDesc) error { d.nbdDev = GetNBDManager().AcquireNbddev() if len(d.nbdDev) == 0 { return errors.Errorf("Cannot get nbd device") @@ -262,6 +263,41 @@ func (d *NBDDriver) IsLVMPartition() bool { return len(d.lvms) > 0 } +func (d *NBDDriver) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool { + return fsutils.DetectIsUEFISupport(rootfs, d.GetPartitions()) +} + +func (d *NBDDriver) MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error) { + return fsutils.MountRootfs(readonly, d.GetPartitions()) +} + +func (d *NBDDriver) UmountRootfs(fd fsdriver.IRootFsDriver) error { + if part := fd.GetPartition(); part != nil { + return part.Umount() + } + return nil +} + +func (d *NBDDriver) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) { + return fsutils.DeployGuestfs(d, req) +} + +func (d *NBDDriver) ResizeFs() (*apis.Empty, error) { + return fsutils.ResizeFs(d) +} + +func (d *NBDDriver) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) { + return fsutils.SaveToGlance(d, req) +} + +func (d *NBDDriver) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) { + return fsutils.FormatFs(d, req) +} + +func (d *NBDDriver) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) { + return fsutils.ProbeImageInfo(d) +} + func getQemuNbdVersion() (string, error) { output, err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuNbd(), "--version").Output() if err != nil { diff --git a/pkg/hostman/diskutils/nbd/nbdman.go b/pkg/hostman/diskutils/nbd/nbdman.go index d00005e13b8..4c2c7b7a7c6 100644 --- a/pkg/hostman/diskutils/nbd/nbdman.go +++ b/pkg/hostman/diskutils/nbd/nbdman.go @@ -148,15 +148,15 @@ func (m *SNBDManager) findNbdDevices() error { for { nbddev := fmt.Sprintf("/dev/nbd%d", i) if fileutils2.Exists(nbddev) { + i++ if fileutils2.IsBlockDeviceUsed(nbddev) { continue } m.nbdDevs[nbddev] = false // https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-bdi - nbdBdi := fmt.Sprintf("/sys/block/nbd%d/bdi/", i) + nbdBdi := fmt.Sprintf("/sys/block/nbd%d/bdi/", i-1) sysutils.SetSysConfig(nbdBdi+"max_ratio", "0") sysutils.SetSysConfig(nbdBdi+"min_ratio", "0") - i++ } else { break } diff --git a/pkg/hostman/diskutils/qemu_kvm/doc.go b/pkg/hostman/diskutils/qemu_kvm/doc.go new file mode 100644 index 00000000000..1d0de0bd413 --- /dev/null +++ b/pkg/hostman/diskutils/qemu_kvm/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package qemu_kvm // import "yunion.io/x/onecloud/pkg/hostman/diskutils/qemu_kvm" diff --git a/pkg/hostman/diskutils/qemu_kvm/driver.go b/pkg/hostman/diskutils/qemu_kvm/driver.go new file mode 100644 index 00000000000..2ba52fbbbb6 --- /dev/null +++ b/pkg/hostman/diskutils/qemu_kvm/driver.go @@ -0,0 +1,617 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package qemu_kvm + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "sync" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/stringutils" + + "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" + "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" + "yunion.io/x/onecloud/pkg/hostman/monitor" + "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/netutils2" + "yunion.io/x/onecloud/pkg/util/procutils" + "yunion.io/x/onecloud/pkg/util/qemuimg" + "yunion.io/x/onecloud/pkg/util/ssh" + "yunion.io/x/onecloud/pkg/util/sysutils" +) + +var BASE_SSH_PORT = 22222 + +var ( + QEMU_KVM_PATH = "/usr/libexec/qemu-kvm" + + OS_ARCH_AARCH64 = "aarch64" + ARM_INITRD_PATH = "/yunionos/aarch64/initramfs" + ARM_KERNEL_PATH = "/yunionos/aarch64/kernel" + X86_INITRD_PATH = "/yunionos/x86_64/initramfs" + X86_KERNEL_PATH = "/yunionos/x86_64/kernel" + + DEPLOY_ISO = "/opt/cloud/host_deployer_v1.iso" + DEPLOYER_BIN = "/opt/yunion/bin/host-deployer" + YUNIONOS_PASSWD = "mosbaremetal" +) + +type QemuDeployManager struct { + cpuArch string + hugepage bool + hugepageSizeKB int + portsInUse *sync.Map + lastUsedSshPort int + + c chan struct{} +} + +func (m *QemuDeployManager) Acquire() { + log.Infof("acquire QemuDeployManager") + m.c <- struct{}{} +} + +func (m *QemuDeployManager) Release() { + log.Infof("release QemuDeployManager") + <-m.c +} + +func (m *QemuDeployManager) GetFreePortByBase(basePort int) int { + var port = 1 + for { + if netutils2.IsTcpPortUsed("0.0.0.0", basePort+port) { + port += 1 + } else { + if !m.checkAndSetPort(basePort + port) { + continue + } + break + } + } + return port + basePort +} + +func (m *QemuDeployManager) checkAndSetPort(port int) bool { + _, loaded := m.portsInUse.LoadOrStore(port, struct{}{}) + return !loaded +} + +func (m *QemuDeployManager) unsetPort(port int) { + m.portsInUse.Delete(port) +} + +func (m *QemuDeployManager) GetSshFreePort() int { + port := m.GetFreePortByBase(BASE_SSH_PORT + m.lastUsedSshPort) + m.lastUsedSshPort = port - BASE_SSH_PORT + if m.lastUsedSshPort > 10000 { + m.lastUsedSshPort = 0 + } + return port +} + +var manager *QemuDeployManager + +func InitQemuDeployManager(cpuArch string, hugepage bool, hugepageSizeKB int, deployConcurrent int) { + if deployConcurrent <= 0 { + deployConcurrent = 10 + } + + if manager == nil { + manager = &QemuDeployManager{ + cpuArch: cpuArch, + hugepage: hugepage, + hugepageSizeKB: hugepageSizeKB, + portsInUse: new(sync.Map), + c: make(chan struct{}, deployConcurrent), + } + } +} + +type QemuKvmDriver struct { + imageInfo qemuimg.SImageInfo + qemuArchDriver IQemuArchDriver + sshClient *ssh.Client + + partitions []fsdriver.IDiskPartition + lvmPartitions []fsdriver.IDiskPartition +} + +func NewQemuKvmDriver(imageInfo qemuimg.SImageInfo) *QemuKvmDriver { + return &QemuKvmDriver{ + imageInfo: imageInfo, + } +} + +func (d *QemuKvmDriver) Connect(guestDesc *apis.GuestDesc) error { + manager.Acquire() + d.qemuArchDriver = NewCpuArchDriver(manager.cpuArch) + err := d.connect(guestDesc) + if err != nil { + d.qemuArchDriver.CleanGuest() + return err + } + + return nil +} + +func (d *QemuKvmDriver) connect(guestDesc *apis.GuestDesc) error { + var ( + ncpu = 2 + memSizeMB = 1024 // yunionos acquire least 1g mem + disks = make([]string, 0) + ) + + var sshport = manager.GetSshFreePort() + defer manager.unsetPort(sshport) + + if guestDesc != nil { + for i := range guestDesc.Disks { + disks = append(disks, guestDesc.Disks[i].Path) + } + } else { + disks = append(disks, d.imageInfo.Path) + } + + err := d.qemuArchDriver.StartGuest(sshport, ncpu, memSizeMB, manager.hugepage, manager.hugepageSizeKB, disks) + if err != nil { + return err + } + log.Infof("guest started ....") + + cli, err := ssh.NewClient("localhost", sshport, "root", YUNIONOS_PASSWD, "") + if err != nil { + return errors.Wrap(err, "new ssh client") + } + d.sshClient = cli + log.Infof("guest ssh connected") + + out, err := d.sshRun("mount /dev/sr0 /opt") + if err != nil { + return errors.Wrapf(err, "failed mount iso /dev/sr0: %v", out) + } + return nil +} + +func (d *QemuKvmDriver) Disconnect() error { + d.sshClient.Close() + d.qemuArchDriver.CleanGuest() + d.qemuArchDriver = nil + return nil +} + +func (d *QemuKvmDriver) GetPartitions() []fsdriver.IDiskPartition { + return nil +} + +func (d *QemuKvmDriver) IsLVMPartition() bool { + return false +} + +func (d *QemuKvmDriver) Zerofree() {} + +func (d *QemuKvmDriver) ResizePartition() error { + return nil +} + +func (d *QemuKvmDriver) FormatPartition(fs, uuid string) error { + return nil +} + +func (d *QemuKvmDriver) MakePartition(fs string) error { + return nil +} + +func (d *QemuKvmDriver) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool { + return false +} + +func (d *QemuKvmDriver) MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error) { + return nil, nil +} + +func (d *QemuKvmDriver) UmountRootfs(fd fsdriver.IRootFsDriver) error { + return nil +} + +func (d *QemuKvmDriver) sshRun(cmd string) ([]string, error) { + log.Infof("QemuKvmDriver start command %s", cmd) + return d.sshClient.Run(cmd) +} + +func (d *QemuKvmDriver) DeployGuestfs(req *apis.DeployParams) (*apis.DeployGuestFsResponse, error) { + defer func() { + logStr, _ := d.sshRun("test -f /log && cat /log") + log.Infof("DeployGuestfs log: %v", strings.Join(logStr, "\n")) + }() + + params, _ := json.Marshal(req) + cmd := fmt.Sprintf("%s --deploy-action deploy_guest_fs --deploy-params '%s'", DEPLOYER_BIN, params) + out, err := d.sshRun(cmd) + if err != nil { + return nil, errors.Wrapf(err, "run deploy_guest_fs failed %s", out) + } + log.Infof("DeployGuestfs log: %s", strings.Join(out, "\n")) + + responseStrs, err := d.sshRun("test -f /response && cat /response") + if err != nil { + return nil, errors.Wrapf(err, "ssh gather errors failed") + } + var res = new(apis.DeployGuestFsResponse) + if len(responseStrs[0]) > 0 { + err := json.Unmarshal([]byte(responseStrs[0]), res) + if err != nil { + return nil, errors.Wrapf(err, "failed unmarshal deploy response %s", responseStrs[0]) + } + return res, nil + } + + errStrs, err := d.sshRun("test -f /error && cat /error") + if err != nil { + return nil, errors.Wrapf(err, "ssh gather errors failed") + } + var retErr error = nil + if len(errStrs[0]) > 0 { + retErr = errors.Errorf(errStrs[0]) + } + return res, retErr +} + +func (d *QemuKvmDriver) ResizeFs() (*apis.Empty, error) { + defer func() { + logStr, _ := d.sshRun("test -f /log && cat /log") + log.Infof("ResizeFs log: %v", strings.Join(logStr, "\n")) + }() + + cmd := fmt.Sprintf("%s --deploy-action resize_fs", DEPLOYER_BIN) + out, err := d.sshRun(cmd) + if err != nil { + return nil, errors.Wrapf(err, "run resize_fs failed %s", out) + } + log.Infof("ResizeFs log: %s", strings.Join(out, "\n")) + + errStrs, err := d.sshRun("test -f /error && cat /error") + if err != nil { + return nil, errors.Wrapf(err, "ssh gather errors failed") + } + var retErr error = nil + if len(errStrs[0]) > 0 { + retErr = errors.Errorf(errStrs[0]) + } + return new(apis.Empty), retErr +} + +func (d *QemuKvmDriver) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) { + defer func() { + logStr, _ := d.sshRun("test -f /log && cat /log") + log.Infof("FormatFs log: %v", strings.Join(logStr, "\n")) + }() + + params, _ := json.Marshal(req) + cmd := fmt.Sprintf("%s --deploy-action format_fs --deploy-params '%s'", DEPLOYER_BIN, params) + out, err := d.sshRun(cmd) + if err != nil { + return nil, errors.Wrapf(err, "run format_fs failed %s", out) + } + log.Infof("FormatFs log: %s", strings.Join(out, "\n")) + + errStrs, err := d.sshRun("test -f /error && cat /error") + if err != nil { + return nil, errors.Wrapf(err, "ssh gather errors failed") + } + var retErr error = nil + if len(errStrs[0]) > 0 { + retErr = errors.Errorf(errStrs[0]) + } + return new(apis.Empty), retErr +} + +func (d *QemuKvmDriver) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) { + defer func() { + logStr, _ := d.sshRun("test -f /log && cat /log") + log.Infof("SaveToGlance log: %s", strings.Join(logStr, "\n")) + }() + + params, _ := json.Marshal(req) + cmd := fmt.Sprintf("%s --deploy-action save_to_glance --deploy-params '%s'", DEPLOYER_BIN, params) + out, err := d.sshRun(cmd) + if err != nil { + return nil, errors.Wrapf(err, "run save_to_glance failed %s", out) + } + log.Infof("SaveToGlance log: %s", strings.Join(out, "\n")) + + responseStrs, err := d.sshRun("test -f /response && cat /response") + if err != nil { + return nil, errors.Wrapf(err, "ssh gather errors failed") + } + var res = new(apis.SaveToGlanceResponse) + if len(responseStrs[0]) > 0 { + err := json.Unmarshal([]byte(responseStrs[0]), res) + if err != nil { + return nil, errors.Wrapf(err, "failed unmarshal deploy response %s", responseStrs[0]) + } + return res, nil + } + + errStrs, err := d.sshRun("test -f /error && cat /error") + if err != nil { + return nil, errors.Wrapf(err, "ssh gather errors failed") + } + var retErr error = nil + if len(errStrs[0]) > 0 { + retErr = errors.Errorf(errStrs[0]) + } + return res, retErr +} + +func (d *QemuKvmDriver) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) { + defer func() { + logStr, _ := d.sshRun("test -f /log && cat /log") + log.Infof("ProbeImageInfo log: %v", strings.Join(logStr, "\n")) + }() + + params, _ := json.Marshal(req) + cmd := fmt.Sprintf("%s --deploy-action probe_image_info --deploy-params '%s'", DEPLOYER_BIN, params) + out, err := d.sshRun(cmd) + if err != nil { + return nil, errors.Wrapf(err, "run probe_image_info failed %s", out) + } + log.Infof("ProbeImageInfo log: %s", strings.Join(out, "\n")) + + responseStrs, err := d.sshRun("test -f /response && cat /response") + if err != nil { + return nil, errors.Wrapf(err, "ssh gather errors failed") + } + var res = new(apis.ImageInfo) + if len(responseStrs[0]) > 0 { + err := json.Unmarshal([]byte(responseStrs[0]), res) + if err != nil { + return nil, errors.Wrapf(err, "failed unmarshal deploy response %s", responseStrs[0]) + } + return res, nil + } + + errStrs, err := d.sshRun("test -f /error && cat /error") + if err != nil { + return nil, errors.Wrapf(err, "ssh gather errors failed") + } + var retErr error = nil + if len(errStrs[0]) > 0 { + retErr = errors.Errorf(errStrs[0]) + } + return res, retErr +} + +// wrap strings +func __(v string, vs ...interface{}) string { + return fmt.Sprintf(" "+v, vs...) +} + +type QemuBaseDriver struct { + proc *procutils.Command + outb io.ReadCloser + errb io.ReadCloser + + hmp *monitor.HmpMonitor + hugepagePath string + pidPath string +} + +func (d *QemuBaseDriver) CleanGuest() { + defer manager.Release() + + if d.hmp != nil { + d.hmp.IsConnected() + d.hmp.Quit(func(string) {}) + d.hmp = nil + } + + if d.pidPath != "" && fileutils2.Exists(d.pidPath) { + pid, _ := fileutils2.FileGetContents(d.pidPath) + if len(pid) > 0 { + out, err := procutils.NewCommand("kill", pid).Output() + log.Infof("kill process %s %v", out, err) + } + } + + if d.hugepagePath != "" { + err, out := procutils.NewCommand("umount", d.hugepagePath).Output() + if err != nil { + log.Errorf("failed umount %s %s : %s", d.hugepagePath, err, out) + } else { + procutils.NewCommand("rm", "-rf", d.hugepagePath).Run() + } + d.hugepagePath = "" + } +} + +type QemuX86Driver struct { + QemuBaseDriver +} + +func (d *QemuX86Driver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool, pageSizeKB int, disks []string) error { + uuid := stringutils.UUID4() + socketPath := fmt.Sprintf("/tmp/hmp_%s.socket", uuid) + + cmd := QEMU_KVM_PATH + if sysutils.IsKvmSupport() { + cmd += __("-enable-kvm") + } + + cmd += __("-cpu host") + cmd += __("-M pc") + + d.pidPath = fmt.Sprintf("/tmp/%s.pid", uuid) + cmd += __("-nodefaults") + cmd += __("-daemonize") + cmd += __("-monitor unix:%s,server,nowait", socketPath) + cmd += __("-pidfile %s", d.pidPath) + cmd += __("-vnc none") + cmd += __("-smp %d", ncpu) + + //if hugePage { + // if pageSizeKB/1024 > memSizeMB { + // memSizeMB = pageSizeKB / 1024 + // } + // + // hugepagePath := fmt.Sprintf("/dev/hugepages/host-deployer/%s", uuid) + // out, err := procutils.NewCommand("mkdir", "-p", hugepagePath).Output() + // if err != nil { + // return errors.Wrapf(err, "mkdir %s failed: %s", hugepagePath, out) + // } + // d.hugepagePath = hugepagePath + // + // mountCmd := fmt.Sprintf("mount -t hugetlbfs -o pagesize=%dK,size=%dM hugetlbfs-%s %s", pageSizeKB, memSizeMB, uuid, hugepagePath) + // out, err = procutils.NewCommand("bash", "-c", mountCmd).Output() + // if err != nil { + // return errors.Wrapf(err, "mount %s failed: %s", mountCmd, out) + // } + // + // cmd += __("-m %dM", memSizeMB) + // cmd += __("-object memory-backend-file,id=mem,prealloc=on,mem-path=%s,size=%dM,share=on", hugepagePath, memSizeMB) + // cmd += __("-numa node,memdev=mem") + //} else { + cmd += __("-m %dM", memSizeMB) + //} + + cmd += __("-initrd %s", X86_INITRD_PATH) + cmd += __("-kernel %s", X86_KERNEL_PATH) + cmd += __("-device VGA") + cmd += __("-device virtio-serial-pci") + cmd += __("-netdev user,id=hostnet0,hostfwd=tcp::%d-:22", sshPort) + cmd += __("-device virtio-net-pci,netdev=hostnet0") + cmd += __("-device virtio-scsi-pci,id=scsi") + for i, diskPath := range disks { + cmd += __("-drive file=%s,if=none,id=drive_%d,cache=none,aio=native,file.locking=off", diskPath, i) + cmd += __("-device scsi-hd,drive=drive_%d,bus=scsi.0,id=drive_%d", i, i) + } + cmd += __("-drive id=ide0-cd0,if=none,media=cdrom,file=%s", DEPLOY_ISO) + cmd += __("-device ide-cd,drive=ide0-cd0,bus=ide.1") + + log.Infof("start guest %s", cmd) + out, err := procutils.NewCommand("bash", "-c", cmd).Output() + if err != nil { + log.Errorf("failed start guest %s: %s", out, err) + return errors.Wrapf(err, "failed start guest %s", out) + } + + var c = make(chan error) + onMonitorConnected := func() { + log.Infof("monitor connected") + c <- nil + } + onMonitorDisConnect := func(e error) { + log.Errorf("monitor disconnect %s", e) + } + onMonitorConnectFailed := func(e error) { + log.Errorf("monitor connect failed %s", e) + c <- e + } + m := monitor.NewHmpMonitor("", "", onMonitorDisConnect, onMonitorConnectFailed, onMonitorConnected) + if err = m.ConnectWithSocket(socketPath); err != nil { + return errors.Wrapf(err, "connect socket %s failed", socketPath) + } + if err = <-c; err != nil { + return errors.Wrap(err, "monitor connect failed") + } + d.hmp = m + return nil +} + +type QemuARMDriver struct { + QemuBaseDriver +} + +func (d *QemuARMDriver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool, pageSizeKB int, disks []string) error { + uuid := stringutils.UUID4() + socketPath := fmt.Sprintf("/tmp/hmp_%s.socket", uuid) + + cmd := QEMU_KVM_PATH + if sysutils.IsKvmSupport() { + cmd += __("-enable-kvm") + } + + cmd += __("-cpu host") + cmd += __("-M virt") + + d.pidPath = fmt.Sprintf("/tmp/%s.pid", uuid) + cmd += __("-nodefaults") + cmd += __("-daemonize") + cmd += __("-monitor unix:%s,server,nowait", socketPath) + cmd += __("-pidfile %s", d.pidPath) + cmd += __("-vnc none") + cmd += __("-smp %d", ncpu) + + cmd += __("-m %dM", memSizeMB) + cmd += __("-initrd %s", ARM_INITRD_PATH) + cmd += __("-kernel %s", ARM_KERNEL_PATH) + cmd += __("-drive if=pflash,format=raw,unit=0,file=/opt/cloud/contrib/OVMF.fd,readonly=on") + + cmd += __("-device virtio-serial-pci") + cmd += __("-netdev user,id=hostnet0,hostfwd=tcp::%d-:22", sshPort) + cmd += __(" -device virtio-net-pci,netdev=hostnet0") + cmd += __("-device virtio-scsi-pci,id=scsi") + for i, diskPath := range disks { + cmd += __("-drive file=%s,if=none,id=drive_%d,cache=none,aio=native,file.locking=off", diskPath, i) + cmd += __("-device scsi-hd,drive=drive_%d,bus=scsi.0,id=drive_%d", i, i) + } + cmd += __("-drive if=none,file=%s,id=cd0,media=cdrom", DEPLOY_ISO) + cmd += __("-device scsi-cd,drive=cd0,share-rw=true") + + log.Infof("start guest %s", cmd) + out, err := procutils.NewCommand("bash", "-c", cmd).Output() + if err != nil { + log.Errorf("failed start guest %s: %s", out, err) + return errors.Wrapf(err, "failed start guest %s", out) + } + + var c = make(chan error) + onMonitorConnected := func() { + log.Infof("monitor connected") + c <- nil + } + onMonitorDisConnect := func(e error) { + log.Errorf("monitor disconnect %s", e) + } + onMonitorConnectFailed := func(e error) { + log.Errorf("monitor connect failed %s", e) + c <- e + } + m := monitor.NewHmpMonitor("", "", onMonitorDisConnect, onMonitorConnectFailed, onMonitorConnected) + if err = m.ConnectWithSocket(socketPath); err != nil { + return errors.Wrapf(err, "connect socket %s failed", socketPath) + } + if err = <-c; err != nil { + return errors.Wrap(err, "monitor connect failed") + } + d.hmp = m + return nil +} + +type IQemuArchDriver interface { + StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool, pageSizeKB int, disks []string) error + CleanGuest() +} + +func NewCpuArchDriver(cpuArch string) IQemuArchDriver { + if cpuArch == OS_ARCH_AARCH64 { + return &QemuARMDriver{} + } + + return &QemuX86Driver{} +} diff --git a/pkg/hostman/diskutils/qemu_kvm/local_driver.go b/pkg/hostman/diskutils/qemu_kvm/local_driver.go new file mode 100644 index 00000000000..bbd60cdad9b --- /dev/null +++ b/pkg/hostman/diskutils/qemu_kvm/local_driver.go @@ -0,0 +1,140 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package qemu_kvm + +import ( + "fmt" + "strings" + "time" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/hostman/diskutils/fsutils" + "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" + "yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart" + "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" + "yunion.io/x/onecloud/pkg/util/procutils" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +type LocalDiskDriver struct { + partitions []fsdriver.IDiskPartition + lvmPartitions []fsdriver.IDiskPartition +} + +func NewLocalDiskDriver() *LocalDiskDriver { + return &LocalDiskDriver{ + partitions: make([]fsdriver.IDiskPartition, 0), + lvmPartitions: make([]fsdriver.IDiskPartition, 0), + } +} + +func (d *LocalDiskDriver) Connect(desc *apis.GuestDesc) error { + out, err := procutils.NewCommand("sh", "-c", "cat /proc/partitions | grep -v name | awk '{print $4}'").Output() + if err != nil { + return errors.Wrap(err, "cat proc partitions") + } + lines := strings.Split(string(out), "\n") + partDevs := make([]string, 0) + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "sd") { + partDevs = append(partDevs, line) + } + } + sortPartDevs := stringutils2.NewSortedStrings(partDevs) + for _, partDev := range sortPartDevs { + part := kvmpart.NewKVMGuestDiskPartition(fmt.Sprintf("/dev/%s", partDev), "", false) + d.partitions = append(d.partitions, part) + log.Infof("found part dev %s", part.GetPartDev()) + } + d.setupLVMS() + if len(d.lvmPartitions) > 0 { + d.partitions = append(d.partitions, d.lvmPartitions...) + } + return nil +} + +func (d *LocalDiskDriver) Disconnect() error { + return nil +} + +func (d *LocalDiskDriver) GetPartitions() []fsdriver.IDiskPartition { + return d.partitions +} + +func (d *LocalDiskDriver) IsLVMPartition() bool { + return len(d.lvmPartitions) > 0 +} + +func (d *LocalDiskDriver) Zerofree() { + startTime := time.Now() + for _, part := range d.partitions { + part.Zerofree() + } + log.Infof("Zerofree %d partitions takes %f seconds", len(d.partitions), time.Now().Sub(startTime).Seconds()) +} + +func (d *LocalDiskDriver) ResizePartition() error { + if d.IsLVMPartition() { + // do not resize LVM partition + return nil + } + return fsutils.ResizeDiskFs("/dev/sda", 0) +} + +func (d *LocalDiskDriver) FormatPartition(fs, uuid string) error { + return fsutils.FormatPartition("/dev/sda1", fs, uuid) +} + +func (d *LocalDiskDriver) MakePartition(fs string) error { + return fsutils.Mkpartition("/dev/sda", fs) +} + +func (d *LocalDiskDriver) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool { + return fsutils.DetectIsUEFISupport(rootfs, d.GetPartitions()) +} + +func (d *LocalDiskDriver) MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error) { + return fsutils.MountRootfs(readonly, d.GetPartitions()) +} + +func (d *LocalDiskDriver) UmountRootfs(fd fsdriver.IRootFsDriver) error { + if part := fd.GetPartition(); part != nil { + return part.Umount() + } + return nil +} + +func (d *LocalDiskDriver) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) { + return fsutils.DeployGuestfs(d, req) +} + +func (d *LocalDiskDriver) ResizeFs() (*apis.Empty, error) { + return fsutils.ResizeFs(d) +} + +func (d *LocalDiskDriver) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) { + return fsutils.SaveToGlance(d, req) +} + +func (d *LocalDiskDriver) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) { + return fsutils.FormatFs(d, req) +} + +func (d *LocalDiskDriver) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) { + return fsutils.ProbeImageInfo(d) +} diff --git a/pkg/hostman/diskutils/qemu_kvm/lvm.go b/pkg/hostman/diskutils/qemu_kvm/lvm.go new file mode 100644 index 00000000000..01dda5dca92 --- /dev/null +++ b/pkg/hostman/diskutils/qemu_kvm/lvm.go @@ -0,0 +1,138 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package qemu_kvm + +import ( + "encoding/json" + "fmt" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func (d *LocalDiskDriver) setupLVMS() { + for _, part := range d.partitions { + vg, err := d.findVg(part.GetPartDev()) + if err != nil { + log.Infof("failed find vg %s", err) + continue + } + if vg == nil { + continue + } + + log.Infof("found vg %s from %s", vg.VgName, part.GetPartDev()) + err = d.vgActive(vg.VgName) + if err != nil { + log.Infof("failed active vg %s: %s", vg.VgName, err) + continue + } + lvs, err := d.getVgLvs(vg.VgName) + if err != nil { + log.Infof("failed get vg lvs %s: %s", vg.VgName, err) + continue + } + log.Infof("found lvs %v from vg %s", lvs, vg.VgName) + for _, lv := range lvs { + lvmpart := kvmpart.NewKVMGuestDiskPartition(lv.LvPath, "", true) + d.lvmPartitions = append(d.lvmPartitions, lvmpart) + log.Infof("found lvm part dev %v", lvmpart.GetPartDev()) + } + } +} + +func (d *LocalDiskDriver) vgActive(vgname string) error { + _, err := procutils.NewCommand("vgchange", "-ay", vgname).Output() + if err != nil { + return err + } + return nil +} + +type LvProps struct { + LvName string + LvPath string +} + +type LvNames struct { + Report []struct { + LV []struct { + LVName string `json:"lv_name"` + LVPath string `json:"lv_path"` + } `json:"lv"` + } `json:"report"` +} + +func (d *LocalDiskDriver) getVgLvs(vg string) ([]LvProps, error) { + cmd := fmt.Sprintf("lvs --reportformat json -o lv_name,lv_path %s 2>/dev/null", vg) + out, err := procutils.NewCommand("sh", "-c", cmd).Output() + if err != nil { + return nil, errors.Wrap(err, "find vg lvs") + } + var res LvNames + err = json.Unmarshal(out, &res) + if err != nil { + return nil, errors.Wrap(err, "unmarshal lvs") + } + if len(res.Report) != 1 { + return nil, errors.Errorf("unexpect res %v", res) + } + lvs := make([]LvProps, len(res.Report[0].LV)) + for i := 0; i < len(res.Report[0].LV); i++ { + if res.Report[0].LV[i].LVName == "" { + continue + } + lvs = append(lvs, LvProps{res.Report[0].LV[i].LVName, res.Report[0].LV[i].LVPath}) + } + return lvs, nil +} + +type VgProps struct { + VgName string + VgUuid string +} + +type VgReports struct { + Report []struct { + VG []struct { + VgName string `json:"vg_name"` + VgUuid string `json:"vg_uuid"` + } `json:"vg"` + } `json:"report"` +} + +func (d *LocalDiskDriver) findVg(partDev string) (*VgProps, error) { + cmd := fmt.Sprintf("vgs --reportformat json -o vg_name,vg_uuid --devices %s 2>/dev/null", partDev) + out, err := procutils.NewCommand("sh", "-c", cmd).Output() + if err != nil { + return nil, errors.Wrap(err, "find vg lvs") + } + var vgReports VgReports + err = json.Unmarshal(out, &vgReports) + if err != nil { + return nil, errors.Wrapf(err, "unmarshal vgprops %s", out) + } + if len(vgReports.Report) == 1 && len(vgReports.Report[0].VG) == 1 { + var vgProps VgProps + vgProps.VgName = vgReports.Report[0].VG[0].VgName + vgProps.VgUuid = vgReports.Report[0].VG[0].VgUuid + return &vgProps, nil + } + + return nil, nil +} diff --git a/pkg/hostman/diskutils/vddk.go b/pkg/hostman/diskutils/vddk.go index 97184d25144..e77c5e68b3e 100644 --- a/pkg/hostman/diskutils/vddk.go +++ b/pkg/hostman/diskutils/vddk.go @@ -147,7 +147,7 @@ func (vd *VDDKDisk) Cleanup() { } } -func (vd *VDDKDisk) Connect() error { +func (vd *VDDKDisk) Connect(*apis.GuestDesc) error { flatFile, err := vd.ConnectBlockDevice() if err != nil { return errors.Wrap(err, "ConnectBlockDevice") @@ -156,7 +156,7 @@ func (vd *VDDKDisk) Connect() error { if err != nil { return errors.Wrap(err, "NewKVMGuestDisk") } - return vd.kvmDisk.Connect() + return vd.kvmDisk.Connect(nil) } func (vd *VDDKDisk) Disconnect() error { @@ -441,8 +441,24 @@ func (vd *VDDKDisk) DisconnectBlockDevice() error { return fmt.Errorf("vddk disk has not connected") } -func (vd *VDDKDisk) ResizePartition() error { - return vd.kvmDisk.ResizePartition() +func (vd *VDDKDisk) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) { + return vd.kvmDisk.DeployGuestfs(req) +} + +func (d *VDDKDisk) ResizeFs() (*apis.Empty, error) { + return d.kvmDisk.ResizeFs() +} + +func (d *VDDKDisk) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) { + return d.kvmDisk.FormatFs(req) +} + +func (d *VDDKDisk) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) { + return d.kvmDisk.SaveToGlance(req) +} + +func (d *VDDKDisk) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) { + return d.kvmDisk.ProbeImageInfo(req) } type VDDKPartition struct { diff --git a/pkg/hostman/guestfs/fsdriver/linux.go b/pkg/hostman/guestfs/fsdriver/linux.go index 1b7e1f7e45a..378fa3fdbf3 100644 --- a/pkg/hostman/guestfs/fsdriver/linux.go +++ b/pkg/hostman/guestfs/fsdriver/linux.go @@ -497,6 +497,9 @@ func (l *sLinuxRootFs) GetArch(rootFs IDiskPartition) string { if fileInfo.IsDir() { continue } + if fileInfo.Mode()&os.ModeSymlink != 0 { + continue + } rp, err := filepath.EvalSymlinks(p) if err != nil { log.Errorf("readlink of %s: %s", p, err) diff --git a/pkg/hostman/guestfs/kvmpart/kvmpart.go b/pkg/hostman/guestfs/kvmpart/kvmpart.go index d9ce52bf949..01e70cbb53e 100644 --- a/pkg/hostman/guestfs/kvmpart/kvmpart.go +++ b/pkg/hostman/guestfs/kvmpart/kvmpart.go @@ -154,7 +154,7 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error { if output, err := procutils.NewCommand("mkdir", "-p", p.mountPath).Output(); err != nil { return errors.Wrapf(err, "mkdir %s failed: %s", p.mountPath, output) } - var cmds = []string{"mount", "-t"} + var cmds = []string{"mount"} var opt, fsType string if readonly { opt = "ro" @@ -168,7 +168,9 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error { } else if fsType == "hfsplus" && !readonly { opt = "force,rw" } - cmds = append(cmds, fsType) + if len(fsType) > 0 { + cmds = append(cmds, "-t", fsType) + } if len(opt) > 0 { cmds = append(cmds, "-o", opt) } @@ -189,13 +191,35 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error { } retrier := func(utils.FibonacciRetrier) (bool, error) { - output, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output() - if err == nil { + var errChan = make(chan error) + go func() { + output, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output() + if err == nil { + errChan <- nil + } else { + log.Errorf("mount fail: %s %s", err, output) + time.Sleep(time.Millisecond * time.Duration(100+rand.Intn(400))) + errChan <- err + } + }() + select { + case err := <-errChan: + if err != nil { + return false, err + } return true, nil - } else { - log.Errorf("mount fail: %s %s", err, output) - time.Sleep(time.Millisecond * time.Duration(100+rand.Intn(400))) - return false, errors.Wrap(err, "") + case <-time.After(time.Second * 3): + if fsType == "ntfs-3g" { + if err = procutils.NewCommand("mountpoint", p.mountPath).Run(); err != nil { + // mountpath is not a mountpoint + return false, err + } else { + // ntfs already mounted, but maybe in buildroot yunion os Failed to daemonize. + return true, nil + } + } else { + return false, errors.Errorf("mount timeout") + } } } _, err = utils.NewFibonacciRetrierMaxTries(3, retrier).Start(context.Background()) diff --git a/pkg/hostman/guestfs/kvmpart/localfs.go b/pkg/hostman/guestfs/kvmpart/localfs.go index 29a945342d2..521039fb6fd 100644 --- a/pkg/hostman/guestfs/kvmpart/localfs.go +++ b/pkg/hostman/guestfs/kvmpart/localfs.go @@ -224,15 +224,51 @@ func (f *SLocalGuestFS) Chmod(sPath string, mode uint32, caseInsensitive bool) e return nil } +func (f *SLocalGuestFS) updateUserEtcShadow(username string) error { + sPath := f.GetLocalPath("/etc/shadow", false) + if !fileutils2.Exists(sPath) { + return nil + } + content, err := fileutils2.FileGetContents(sPath) + if err != nil { + return errors.Wrap(err, "read /etc/shadow") + } + + var ( + minimumDays = "0" // -m 0 + maximumDays = "99999" // -M 99999 + ) + + lines := strings.Split(string(content), "\n") + for i, line := range lines { + fields := strings.Split(line, ":") + if len(fields) >= 7 && fields[0] == username { + fields[3] = minimumDays + fields[4] = maximumDays + fields[5] = "" // password warning period + fields[6] = "" // password inactivity period + fields[7] = "" // account expiration date + line = strings.Join(fields, ":") + lines[i] = line + break + } + } + newContent := strings.Join(lines, "\n") + err = fileutils2.FilePutContents(sPath, newContent, false) + if err != nil { + return errors.Wrapf(err, "read %s, put %s to /etc/shadow", content, newContent) + } + + return nil +} + func (f *SLocalGuestFS) CheckOrAddUser(user, homeDir string, isSys bool) (realHomeDir string, err error) { var exist bool if exist, realHomeDir, err = f.checkUser(user); err != nil || exist { if exist { - cmd := []string{"chage", "-R", f.mountPath, "-E", "-1", "-m", "0", "-M", "99999", "-I", "-1", user} - command := procutils.NewCommand(cmd[0], cmd[1:]...) - _, err = command.Output() + err = f.updateUserEtcShadow(user) if err != nil { - err = errors.Wrap(err, "chage") + err = errors.Wrap(err, "updateUserEtcShadow") return } if !f.Exists(realHomeDir, false) { diff --git a/pkg/hostman/guestfs/sshpart/sshpart.go b/pkg/hostman/guestfs/sshpart/sshpart.go index 9149c7e147e..aa6dc5cee8f 100644 --- a/pkg/hostman/guestfs/sshpart/sshpart.go +++ b/pkg/hostman/guestfs/sshpart/sshpart.go @@ -40,17 +40,17 @@ type SSHPartition struct { term *ssh.Client partDev string mountPath string - part *disktool.Partition + isLVM bool } var _ fsdriver.IDiskPartition = &SSHPartition{} -func NewSSHPartition(term *ssh.Client, part *disktool.Partition) *SSHPartition { +func NewSSHPartition(term *ssh.Client, partDev string, isLVM bool) *SSHPartition { p := new(SSHPartition) p.term = term - p.partDev = part.GetDev() + p.partDev = partDev + p.isLVM = isLVM p.mountPath = fmt.Sprintf("/tmp/%s", strings.Replace(p.partDev, "/", "_", -1)) - p.part = part return p } @@ -616,7 +616,7 @@ func MountSSHRootfs(tool *disktool.SSHPartitionTool, term *ssh.Client, layouts [ return nil, nil, fmt.Errorf("Not found root disk partitions") } for _, part := range parts { - dev := NewSSHPartition(term, part) + dev := NewSSHPartition(term, part.GetDev(), false) if !dev.Mount() { continue } diff --git a/pkg/hostman/guestman/desc/desc.go b/pkg/hostman/guestman/desc/desc.go index 13f02fa4b42..42824ae9bab 100644 --- a/pkg/hostman/guestman/desc/desc.go +++ b/pkg/hostman/guestman/desc/desc.go @@ -343,7 +343,7 @@ type SGuestControlDesc struct { EncryptKeyId string - RescueMode bool // rescue mode + LightMode bool // light mode } type SGuestMetaDesc struct { diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index fd92dc5d115..1e80b51a62e 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -787,7 +787,7 @@ func (s *SKVMGuestInstance) StartMonitorWithImportGuestSocketFile(ctx context.Co }, // on monitor connected s.onReceiveQMPEvent, // on reveive qmp event ) - return mon.ConnectWithSocket(socketFile) + return mon.ConnectWithSocket(socketFile, 0) } func (s *SKVMGuestInstance) StartMonitor(ctx context.Context, cb func()) error { @@ -1651,13 +1651,13 @@ func (s *SKVMGuestInstance) SaveSourceDesc(guestDesc *desc.SGuestDesc) error { } // Save rescue desc if exist - if s.SourceDesc.RescueMode { - err := s.GetRescueDesc() - if err != nil { - log.Errorf("get rescue desc failed %s", err) - return errors.Wrap(err, "get rescue desc") - } - } + //if s.SourceDesc.LightMode { + // err := s.GetRescueDesc() + // if err != nil { + // log.Errorf("get rescue desc failed %s", err) + // return errors.Wrap(err, "get rescue desc") + // } + //} if err := fileutils2.FilePutContents( s.GetSourceDescFilePath(), jsonutils.Marshal(s.SourceDesc).String(), false, @@ -1679,29 +1679,29 @@ func (s *SKVMGuestInstance) GetVpcNIC() *desc.SGuestNetwork { return nil } -func (s *SKVMGuestInstance) GetRescueDesc() error { - if !s.SourceDesc.RescueMode { - return errors.Errorf("guest %s not in rescue mode", s.Id) - } - - s.SourceDesc.RescueInitdPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS) - s.SourceDesc.RescueKernelPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL) - s.SourceDesc.RescueDiskPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_SYS_DISK_NAME) - if s.manager.GetHost().IsAarch64() { - s.SourceDesc.RescueInitdPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS_ARM64) - s.SourceDesc.RescueKernelPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL_ARM64) - } - - // Address - bus, slot, found := s.findUnusedSlotForController(desc.CONTROLLER_TYPE_PCI_ROOT, 0) - if !found { - return errors.Errorf("no valid pci address found ?") - } - s.SourceDesc.RescueDiskDeviceBus = uint(bus) - s.SourceDesc.RescueDiskDeviceSlot = uint(slot) - - return nil -} +//func (s *SKVMGuestInstance) GetRescueDesc() error { +// if !s.SourceDesc.LightMode { +// return errors.Errorf("guest %s not in rescue mode", s.Id) +// } +// +// s.SourceDesc.RescueInitrdPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS) +// s.SourceDesc.RescueKernelPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL) +// s.SourceDesc.RescueDiskPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_SYS_DISK_NAME) +// if s.manager.GetHost().IsAarch64() { +// s.SourceDesc.RescueInitrdPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS_ARM64) +// s.SourceDesc.RescueKernelPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL_ARM64) +// } +// +// // Address +// bus, slot, found := s.findUnusedSlotForController(desc.CONTROLLER_TYPE_PCI_ROOT, 0) +// if !found { +// return errors.Errorf("no valid pci address found ?") +// } +// s.SourceDesc.RescueDiskDeviceBus = uint(bus) +// s.SourceDesc.RescueDiskDeviceSlot = uint(slot) +// +// return nil +//} type guestStartTask struct { s *SKVMGuestInstance diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index 8b8eae192c8..e1f01900681 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -558,12 +558,9 @@ function nic_mtu() { } // set rescue flag to input - if s.Desc.RescueMode { - input.RescueInitdPath = s.SourceDesc.RescueInitdPath - input.RescueKernelPath = s.SourceDesc.RescueKernelPath - input.RescueDiskPath = s.SourceDesc.RescueDiskPath - input.RescueDiskDeviceBus = s.SourceDesc.RescueDiskDeviceBus - input.RescueDiskDeviceSlot = s.SourceDesc.RescueDiskDeviceSlot + if s.Desc.LightMode { + input.RescueInitrdPath = s.getRescueInitrdPath() + input.RescueKernelPath = s.getRescueKernelPath() } qemuOpts, err := qemu.GenerateStartOptions(input) @@ -578,6 +575,20 @@ function nic_mtu() { return cmd, nil } +func (s *SKVMGuestInstance) getRescueInitrdPath() string { + if s.manager.GetHost().IsAarch64() { + return path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS_ARM64) + } + return path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS) +} + +func (s *SKVMGuestInstance) getRescueKernelPath() string { + if s.manager.GetHost().IsAarch64() { + return path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL_ARM64) + } + return path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL) +} + func (s *SKVMGuestInstance) slaveDiskPrepare(input *qemu.GenerateStartOptionsInput, diskUri string) error { for i := 0; i < len(input.GuestDesc.Disks); i++ { diskPath := input.GuestDesc.Disks[i].Path diff --git a/pkg/hostman/guestman/qemu/generate.go b/pkg/hostman/guestman/qemu/generate.go index 33427908cdd..95ae585869d 100644 --- a/pkg/hostman/guestman/qemu/generate.go +++ b/pkg/hostman/guestman/qemu/generate.go @@ -219,25 +219,10 @@ func generateScsiOptions(scsi *desc.SGuestVirtioScsi) string { return opt } -func generateInitrdOptions(drvOpt QemuOptions, initrdPath, kernel, sys_img string, rescueDiskDeviceBus, rescueDiskDeviceSlot uint, nics []*desc.SGuestNetwork) []string { +func generateInitrdOptions(drvOpt QemuOptions, initrdPath, kernel string) []string { opts := make([]string, 0) - opts = append(opts, fmt.Sprintf("-initrd %s", initrdPath)) - opts = append(opts, fmt.Sprintf("-kernel %s", kernel)) - - // create temp disk info - driveString := fmt.Sprintf("file=%s,if=none,id=initrd,cache=none,aio=native,file.locking=off", sys_img) - opts = append(opts, drvOpt.Drive(driveString)) - deviceString := fmt.Sprintf("virtio-blk-pci,drive=initrd,iothread=iothread0,bus=pci.%d,addr=0x%02x,id=initrd,bootindex=1", rescueDiskDeviceBus, rescueDiskDeviceSlot) - opts = append(opts, drvOpt.Device(deviceString)) - - // add ip config - //var ips []string - //for _, nic := range nics { - // ips = append(ips, fmt.Sprintf("ip=%s:%s:%s:%s:%s:%s:off,", nic.Ip, "", nic.Gateway, netutils.Masklen2Mask(nic.Masklen).String(), "", nic.Ifname)) - //} - //appendIps := strings.Join(ips, ",") - // - //opts = append(opts, fmt.Sprintf("-append %s", appendIps)) + opts = append(opts, drvOpt.Initrd(initrdPath)) + opts = append(opts, drvOpt.Kernel(kernel)) return opts } @@ -659,11 +644,8 @@ type GenerateStartOptionsInput struct { EncryptKeyPath string - RescueInitdPath string // rescue initramfs path - RescueKernelPath string // rescue kernel path - RescueDiskPath string // rescue disk path - RescueDiskDeviceBus uint - RescueDiskDeviceSlot uint + RescueInitrdPath string // rescue initramfs path + RescueKernelPath string // rescue kernel path } func (input *GenerateStartOptionsInput) HasBootIndex() bool { @@ -788,15 +770,11 @@ func GenerateStartOptions( } // generate initrd and kernel options - if input.RescueInitdPath != "" { + if input.GuestDesc.LightMode { opts = append(opts, generateInitrdOptions( drvOpt, - input.RescueInitdPath, + input.RescueInitrdPath, input.RescueKernelPath, - input.RescueDiskPath, - input.RescueDiskDeviceBus, - input.RescueDiskDeviceSlot, - input.GuestDesc.Nics, )...) } @@ -835,7 +813,7 @@ func GenerateStartOptions( } // isolated devices - if len(input.GuestDesc.IsolatedDevices) > 0 { + if len(input.GuestDesc.IsolatedDevices) > 0 && !input.GuestDesc.LightMode { opts = append(opts, generateIsolatedDeviceOptions(input.GuestDesc)...) } diff --git a/pkg/hostman/guestman/qemu/qemu.go b/pkg/hostman/guestman/qemu/qemu.go index d3059a40ac0..84c410f3bbd 100644 --- a/pkg/hostman/guestman/qemu/qemu.go +++ b/pkg/hostman/guestman/qemu/qemu.go @@ -106,6 +106,8 @@ type QemuOptions interface { Pidfile(file string) string USB() string VNC(port uint, usePasswd bool) string + Initrd(initrdPath string) string + Kernel(kernelPath string) string } var ( @@ -328,6 +330,14 @@ func (o baseOptions) USB() string { return "-usb" } +func (o baseOptions) Initrd(initrdPath string) string { + return "-initrd " + initrdPath +} + +func (o baseOptions) Kernel(kernelPath string) string { + return "-kernel " + kernelPath +} + func (o baseOptions) VNC(port uint, usePasswd bool) string { opt := fmt.Sprintf("-vnc :%d", port) if usePasswd { diff --git a/pkg/hostman/hostdeployer/consts/consts.go b/pkg/hostman/hostdeployer/consts/consts.go index a7a063f48a0..1a23b02881b 100644 --- a/pkg/hostman/hostdeployer/consts/consts.go +++ b/pkg/hostman/hostdeployer/consts/consts.go @@ -17,4 +17,6 @@ package consts const ( DEPLOY_DRIVER_NBD = "nbd" DEPLOY_DRIVER_LIBGUESTFS = "libguestfs" + DEPLOY_DRIVER_QEMU_KVM = "qemu-kvm" + DEPLOY_DRIVER_LOCAL_DISK = "local" ) diff --git a/pkg/hostman/hostdeployer/deployserver/deployserver.go b/pkg/hostman/hostdeployer/deployserver/deployserver.go index 815a897a9f1..18e4fd704b1 100644 --- a/pkg/hostman/hostdeployer/deployserver/deployserver.go +++ b/pkg/hostman/hostdeployer/deployserver/deployserver.go @@ -16,6 +16,7 @@ package deployserver import ( "context" + "encoding/json" "fmt" "net" "os" @@ -29,11 +30,12 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + commonconsts "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/service" "yunion.io/x/onecloud/pkg/hostman/diskutils" "yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs" "yunion.io/x/onecloud/pkg/hostman/diskutils/nbd" - "yunion.io/x/onecloud/pkg/hostman/guestfs" + "yunion.io/x/onecloud/pkg/hostman/diskutils/qemu_kvm" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" "yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts" @@ -92,32 +94,20 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP } defer disk.Cleanup() - if err := disk.Connect(); err != nil { + if err := disk.Connect(req.GuestDesc); err != nil { log.Errorf("Failed to connect %s disk: %s", req.GuestDesc.Hypervisor, err) return new(deployapi.DeployGuestFsResponse), errors.Wrap(err, "Connect") } defer disk.Disconnect() - root, err := disk.MountRootfs() - if err != nil { - if errors.Cause(err) == errors.ErrNotFound && req.DeployInfo.IsInit { - // if init deploy, ignore no partition error - log.Errorf("disk.MountRootfs not found partition, not init, quit") - return new(deployapi.DeployGuestFsResponse), nil - } - log.Errorf("Failed mounting rootfs for %s disk: %s", req.GuestDesc.Hypervisor, err) - return new(deployapi.DeployGuestFsResponse), err + + ret, err := disk.DeployGuestfs(req) + if ret == nil { + ret = new(deployapi.DeployGuestFsResponse) } - defer disk.UmountRootfs(root) - ret, err := guestfs.DoDeployGuestFs(root, req.GuestDesc, req.DeployInfo) if err != nil { - log.Errorf("guestfs.DoDeployGuestFs fail %s", err) - return new(deployapi.DeployGuestFsResponse), err + log.Errorf("failed deploy guest fs: %s", err) } - if ret == nil { - log.Errorf("guestfs.DoDeployGuestFs return empty results") - return new(deployapi.DeployGuestFsResponse), nil - } - return ret, nil + return ret, err } func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsParams) (res *deployapi.Empty, err error) { @@ -142,44 +132,12 @@ func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsPara return new(deployapi.Empty), errors.Wrap(err, "GetIDisk fail") } defer disk.Cleanup() - if err := disk.Connect(); err != nil { + if err := disk.Connect(nil); err != nil { return new(deployapi.Empty), errors.Wrap(err, "disk connect failed") } defer disk.Disconnect() - unmount := func(root fsdriver.IRootFsDriver) error { - err := disk.UmountRootfs(root) - if err != nil { - return errors.Wrap(err, "unmount rootfs") - } - return nil - } - - root, err := disk.MountRootfs() - if err != nil { - if errors.Cause(err) == errors.ErrNotFound { - return new(deployapi.Empty), nil - } - return new(deployapi.Empty), errors.Wrapf(err, "disk.MountRootfs") - } - if !root.IsResizeFsPartitionSupport() { - err := unmount(root) - if err != nil { - return new(deployapi.Empty), err - } - return new(deployapi.Empty), errors.ErrNotSupported - } - - // must umount rootfs before resize partition - err = unmount(root) - if err != nil { - return new(deployapi.Empty), err - } - err = disk.ResizePartition() - if err != nil { - return new(deployapi.Empty), errors.Wrap(err, "resize disk partition") - } - return new(deployapi.Empty), nil + return disk.ResizeFs() } func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsParams) (*deployapi.Empty, error) { @@ -190,16 +148,9 @@ func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsPara } defer gd.Cleanup() - if err := gd.Connect(); err == nil { + if err := gd.Connect(nil); err == nil { defer gd.Disconnect() - if err := gd.MakePartition(req.FsFormat); err == nil { - err = gd.FormatPartition(req.FsFormat, req.Uuid) - if err != nil { - return new(deployapi.Empty), errors.Wrap(err, "FormatPartition") - } - } else { - return new(deployapi.Empty), errors.Wrap(err, "MakePartition") - } + return gd.FormatFs(req) } else { log.Errorf("failed connect kvm disk %#v: %s", apiDiskInfo(req.DiskInfo), err) } @@ -208,75 +159,20 @@ func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsPara func (*DeployerServer) SaveToGlance(ctx context.Context, req *deployapi.SaveToGlanceParams) (*deployapi.SaveToGlanceResponse, error) { log.Infof("********* %#v save to glance", apiDiskInfo(req.DiskInfo)) - var ( - osInfo string - relInfo *deployapi.ReleaseInfo - ) + kvmDisk, err := diskutils.NewKVMGuestDisk(apiDiskInfo(req.GetDiskInfo()), DeployOption.ImageDeployDriver, false) if err != nil { return new(deployapi.SaveToGlanceResponse), errors.Wrap(err, "NewKVMGuestDisk") } defer kvmDisk.Cleanup() - ret := &deployapi.SaveToGlanceResponse{ - OsInfo: osInfo, - ReleaseInfo: relInfo, - } - - err = kvmDisk.Connect() + err = kvmDisk.Connect(nil) if err != nil { - return ret, errors.Wrapf(err, "kvmDisk.Connect %#v", apiDiskInfo(req.DiskInfo)) + return new(deployapi.SaveToGlanceResponse), errors.Wrapf(err, "kvmDisk.Connect %#v", apiDiskInfo(req.DiskInfo)) } defer kvmDisk.Disconnect() - root, err := kvmDisk.MountKvmRootfs() - if err != nil { - if errors.Cause(err) == errors.ErrNotFound { - return ret, nil - } - return ret, errors.Wrapf(err, "MountKvmRootfs") - } - defer kvmDisk.UmountKvmRootfs(root) - - osInfo = root.GetOs() - relInfo = root.GetReleaseInfo(root.GetPartition()) - if req.Compress { - err = root.PrepareFsForTemplate(root.GetPartition()) - if err != nil { - log.Errorf("PrepareFsForTemplate %s", err) - } - } - if req.Compress { - kvmDisk.Zerofree() - } - return ret, err -} - -func (*DeployerServer) getImageInfo(kvmDisk *diskutils.SKVMGuestDisk) (*deployapi.ImageInfo, error) { - // Fsck is executed during mount - rootfs, err := kvmDisk.MountKvmRootfs() - if err != nil { - if errors.Cause(err) == errors.ErrNotFound { - return new(deployapi.ImageInfo), nil - } - return new(deployapi.ImageInfo), errors.Wrapf(err, "kvmDisk.MountKvmRootfs") - } - partition := rootfs.GetPartition() - imageInfo := &deployapi.ImageInfo{ - OsInfo: rootfs.GetReleaseInfo(partition), - OsType: rootfs.GetOs(), - IsLvmPartition: kvmDisk.IsLVMPartition(), - IsReadonly: partition.IsReadonly(), - IsInstalledCloudInit: rootfs.IsCloudinitInstall(), - } - kvmDisk.UmountKvmRootfs(rootfs) - - // In case of deploy driver is guestfish, we can't mount - // multi partition concurrent, so we need umount rootfs first - imageInfo.IsUefiSupport = kvmDisk.DetectIsUEFISupport(rootfs) - imageInfo.PhysicalPartitionType = partition.GetPhysicalPartitionType() - log.Infof("ProbeImageInfo response %s", imageInfo) - return imageInfo, nil + return kvmDisk.SaveToGlance(req) } func (s *DeployerServer) ProbeImageInfo(ctx context.Context, req *deployapi.ProbeImageInfoPramas) (*deployapi.ImageInfo, error) { @@ -287,13 +183,13 @@ func (s *DeployerServer) ProbeImageInfo(ctx context.Context, req *deployapi.Prob } defer kvmDisk.Cleanup() - if err := kvmDisk.Connect(); err != nil { + if err := kvmDisk.Connect(nil); err != nil { log.Errorf("Failed to connect kvm disk %#v: %s", apiDiskInfo(req.DiskInfo), err) return new(deployapi.ImageInfo), errors.Wrap(err, "Disk connector failed to connect image") } defer kvmDisk.Disconnect() - return s.getImageInfo(kvmDisk) + return kvmDisk.ProbeImageInfo(req) } var connectedEsxiDisks = map[string]*diskutils.VDDKDisk{} @@ -365,6 +261,22 @@ func NewDeployService() *SDeployService { } func (s *SDeployService) RunService() { + if DeployOption.DeployAction != "" { + res, err := StartLocalDeploy(DeployOption.DeployAction) + if err != nil { + if e := fileutils2.FilePutContents("/error", err.Error(), false); e != nil { + log.Errorf("failed put errors to file: %s", e) + } + } + if res != nil { + resStr, _ := json.Marshal(res) + if e := fileutils2.FilePutContents("/response", string(resStr), false); e != nil { + log.Errorf("failed put response to file: %s", e) + } + } + return + } + s.grpcServer = grpc.NewServer() deployapi.RegisterDeployAgentServer(s.grpcServer, &DeployerServer{}) if fileutils2.Exists(DeployOption.DeployServerSocketPath) { @@ -399,12 +311,71 @@ func (s *SDeployService) FixPathEnv() error { } func (s *SDeployService) PrepareEnv() error { + if !fileutils2.Exists(DeployOption.DeployTempDir) { + err := os.MkdirAll(DeployOption.DeployTempDir, 0755) + if err != nil { + return errors.Errorf("fail to create %s: %s", DeployOption.DeployTempDir, err) + } + } + commonconsts.SetDeployTempDir(DeployOption.DeployTempDir) + commonconsts.SetAllowVmSELinux(DeployOption.AllowVmSELinux) + if err := s.FixPathEnv(); err != nil { return err } - if err := nbd.Init(); err != nil { - return errors.Wrap(err, "nbd.Init") + if err := fsdriver.Init(DeployOption.PrivatePrefixes, DeployOption.CloudrootDir); err != nil { + log.Fatalln(err) + } + if DeployOption.ImageDeployDriver == consts.DEPLOY_DRIVER_LIBGUESTFS { + if err := libguestfs.Init(3); err != nil { + log.Fatalln(err) + } + } + + if DeployOption.ImageDeployDriver != consts.DEPLOY_DRIVER_QEMU_KVM { + if err := nbd.Init(); err != nil { + return errors.Wrap(err, "nbd.Init") + } + } else { + // prepare for yunionos don't have but necessary files + out, err := procutils.NewCommand("cp", "-rf", "/usr/bin/chntpw.static", "/opt/yunion/bin/chntpw.static").Output() + if err != nil { + return errors.Wrapf(err, "cp files failed %s", out) + } + out, err = procutils.NewCommand("cp", "-rf", "/usr/bin/.chntpw.static.bin", "/opt/yunion/bin/.chntpw.static.bin").Output() + if err != nil { + return errors.Wrapf(err, "cp files failed %s", out) + } + out, err = procutils.NewCommand("mkdir", "-p", "/opt/yunion/bin/bundles").Output() + if err != nil { + return errors.Wrapf(err, "cp files failed %s", out) + } + out, err = procutils.NewCommand("cp", "-rf", "/usr/bin/bundles/chntpw.static", "/opt/yunion/bin/bundles/chntpw.static").Output() + if err != nil { + return errors.Wrapf(err, "cp files failed %s", out) + } + out, err = procutils.NewCommand("cp", "-rf", "/usr/sbin/zerofree", "/opt/yunion/bin/zerofree").Output() + if err != nil { + return errors.Wrapf(err, "cp files failed %s", out) + } + + cmd := fmt.Sprintf("mkisofs -l -J -L -R -r -v -hide-rr-moved -o %s -graft-points vmware-vddk=/opt/vmware-vddk yunion=/opt/yunion", qemu_kvm.DEPLOY_ISO) + out, err = procutils.NewCommand("bash", "-c", cmd).Output() + if err != nil { + return errors.Wrapf(err, "mkisofs failed %s", out) + } + + cpuArch, err := procutils.NewCommand("uname", "-m").Output() + if err != nil { + return errors.Wrap(err, "get cpu architecture") + } + qemu_kvm.InitQemuDeployManager( + strings.TrimSpace(string(cpuArch)), + DeployOption.HugepagesOption == "native", + DeployOption.HugepageSizeMb*1024, + DeployOption.DeployConcurrent, + ) } // create /dev/lvm_remote @@ -454,6 +425,14 @@ func (s *SDeployService) InitService() { } } }) + + if DeployOption.DeployAction != "" { + if err := LocalInitEnv(); err != nil { + log.Fatalf("local init env %s", err) + } + return + } + log.Infof("exec socket path: %s", DeployOption.ExecutorSocketPath) if DeployOption.EnableRemoteExecutor { execlient.Init(DeployOption.ExecutorSocketPath) @@ -464,14 +443,7 @@ func (s *SDeployService) InitService() { if err := s.PrepareEnv(); err != nil { log.Fatalln(err) } - if err := fsdriver.Init(DeployOption.PrivatePrefixes, DeployOption.CloudrootDir); err != nil { - log.Fatalln(err) - } - if DeployOption.ImageDeployDriver == consts.DEPLOY_DRIVER_LIBGUESTFS { - if err := libguestfs.Init(3); err != nil { - log.Fatalln(err) - } - } + s.O = &DeployOption.BaseOptions if len(DeployOption.DeployServerSocketPath) == 0 { log.Fatalf("missing deploy server socket path") diff --git a/pkg/hostman/hostdeployer/deployserver/localdeploy.go b/pkg/hostman/hostdeployer/deployserver/localdeploy.go new file mode 100644 index 00000000000..a5909d18251 --- /dev/null +++ b/pkg/hostman/hostdeployer/deployserver/localdeploy.go @@ -0,0 +1,152 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deployserver + +import ( + "encoding/json" + "os" + "strings" + + "github.com/sirupsen/logrus" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/hostman/diskutils" + "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" + deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" + "yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts" + "yunion.io/x/onecloud/pkg/util/qemuimg" + "yunion.io/x/onecloud/pkg/util/winutils" +) + +type LocalDeploy struct{} + +func (d *LocalDeploy) DeployGuestFs(req *deployapi.DeployParams) (res *deployapi.DeployGuestFsResponse, err error) { + localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false) + if err != nil { + return nil, errors.Wrap(err, "new local disk") + } + if err := localDisk.Connect(req.GuestDesc); err != nil { + return nil, errors.Wrapf(err, "local disk connect") + } + defer localDisk.Disconnect() + return localDisk.DeployGuestfs(req) +} + +func (d *LocalDeploy) ResizeFs(req *deployapi.ResizeFsParams) (res *deployapi.Empty, err error) { + localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false) + if err != nil { + return nil, errors.Wrap(err, "new local disk") + } + if err := localDisk.Connect(nil); err != nil { + return nil, errors.Wrapf(err, "local disk connect") + } + defer localDisk.Disconnect() + return localDisk.ResizeFs() +} + +func (d *LocalDeploy) FormatFs(req *deployapi.FormatFsParams) (*deployapi.Empty, error) { + localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false) + if err != nil { + return nil, errors.Wrap(err, "new local disk") + } + if err := localDisk.Connect(nil); err != nil { + return nil, errors.Wrapf(err, "local disk connect") + } + defer localDisk.Disconnect() + return localDisk.FormatFs(req) +} + +func (d *LocalDeploy) SaveToGlance(req *deployapi.SaveToGlanceParams) (*deployapi.SaveToGlanceResponse, error) { + localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false) + if err != nil { + return nil, errors.Wrap(err, "new local disk") + } + if err := localDisk.Connect(nil); err != nil { + return nil, errors.Wrapf(err, "local disk connect") + } + defer localDisk.Disconnect() + return localDisk.SaveToGlance(req) +} + +func (d *LocalDeploy) ProbeImageInfo(req *deployapi.ProbeImageInfoPramas) (*deployapi.ImageInfo, error) { + localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false) + if err != nil { + return nil, errors.Wrap(err, "new local disk") + } + if err := localDisk.Connect(nil); err != nil { + return nil, errors.Wrapf(err, "local disk connect") + } + defer localDisk.Disconnect() + return localDisk.ProbeImageInfo(req) +} + +func LocalInitEnv() error { + f, _ := os.OpenFile("/log", os.O_CREATE|os.O_WRONLY, 0777) + logrus.SetOutput(f) + + // /bin:/sbin:/usr/bin:/usr/sbin + var paths = []string{ + "/bin", + "/sbin", + "/usr/bin", + "/usr/sbin", + "/opt/yunion/bin", // for zerofree command + } + err := os.Setenv("PATH", strings.Join(paths, ":")) + if err != nil { + return errors.Wrap(err, "set env path") + } + + if err := fsdriver.Init(DeployOption.PrivatePrefixes, DeployOption.CloudrootDir); err != nil { + return errors.Wrap(err, "init fsdriver") + } + winutils.SetChntpwPath("/opt/yunion/bin/chntpw.static") + return nil +} + +func StartLocalDeploy(deployAction string) (interface{}, error) { + localDeployer := LocalDeploy{} + switch deployAction { + case "deploy_guest_fs": + params := new(deployapi.DeployParams) + if err := json.Unmarshal([]byte(DeployOption.DeployParams), params); err != nil { + return nil, errors.Wrap(err, "unmarshal params") + } + return localDeployer.DeployGuestFs(params) + case "resize_fs": + return localDeployer.ResizeFs(nil) + case "format_fs": + params := new(deployapi.FormatFsParams) + if err := json.Unmarshal([]byte(DeployOption.DeployParams), params); err != nil { + return nil, errors.Wrap(err, "unmarshal params") + } + return localDeployer.FormatFs(params) + case "save_to_glance": + params := new(deployapi.SaveToGlanceParams) + if err := json.Unmarshal([]byte(DeployOption.DeployParams), params); err != nil { + return nil, errors.Wrap(err, "unmarshal params") + } + return localDeployer.SaveToGlance(params) + case "probe_image_info": + params := new(deployapi.ProbeImageInfoPramas) + if err := json.Unmarshal([]byte(DeployOption.DeployParams), params); err != nil { + return nil, errors.Wrap(err, "unmarshal params") + } + return localDeployer.ProbeImageInfo(params) + default: + return nil, errors.Errorf("unknown deploy action") + } +} diff --git a/pkg/hostman/hostdeployer/deployserver/options.go b/pkg/hostman/hostdeployer/deployserver/options.go index 29af97eb434..2683cdeccaf 100644 --- a/pkg/hostman/hostdeployer/deployserver/options.go +++ b/pkg/hostman/hostdeployer/deployserver/options.go @@ -17,11 +17,7 @@ package deployserver import ( "os" - "yunion.io/x/log" - - "yunion.io/x/onecloud/pkg/cloudcommon/consts" common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" - "yunion.io/x/onecloud/pkg/util/fileutils2" ) type SDeployOptions struct { @@ -31,12 +27,19 @@ type SDeployOptions struct { ChntpwPath string `help:"path to chntpw tool" default:"/usr/local/bin/chntpw.static"` CloudrootDir string `help:"User cloudroot home dir" default:"/opt"` - ImageDeployDriver string `help:"Image deploy driver" default:"nbd" choices:"nbd|libguestfs"` + ImageDeployDriver string `help:"Image deploy driver" default:"qemu-kvm" choices:"qemu-kvm|nbd|libguestfs"` CommonConfigFile string `help:"common config file for container"` DeployTempDir string `help:"temp dir for deployer" default:"/opt/cloud/workspace/run/deploy"` AllowVmSELinux bool `help:"turn off vm selinux" default:"false" json:"allow_vm_selinux"` + + HugepagesOption string `help:"Hugepages option: disable|native|transparent" default:"transparent"` + HugepageSizeMb int `help:"hugepage size mb default 1G" default:"1024"` + + DeployAction string `help:"local deploy action"` + DeployParams string `help:"params for deploy action"` + DeployConcurrent int `help:"qemu-kvm deploy driver concurrent" default:"3"` } var DeployOption SDeployOptions @@ -52,15 +55,6 @@ func Parse() (hostOpts SDeployOptions) { // keep base options hostOpts.BaseOptions.BaseOptions = baseOpt } - if !fileutils2.Exists(hostOpts.DeployTempDir) { - err := os.MkdirAll(hostOpts.DeployTempDir, 0755) - if err != nil { - log.Fatalf("fail to create %s: %s", hostOpts.DeployTempDir, err) - return - } - } - consts.SetDeployTempDir(hostOpts.DeployTempDir) - consts.SetAllowVmSELinux(hostOpts.AllowVmSELinux) return hostOpts } diff --git a/pkg/hostman/monitor/monitor.go b/pkg/hostman/monitor/monitor.go index b09d27adf0b..126cb6a117b 100644 --- a/pkg/hostman/monitor/monitor.go +++ b/pkg/hostman/monitor/monitor.go @@ -186,7 +186,7 @@ func (self *BlockJob) CalcOffset(preOffset int64) { type Monitor interface { Connect(host string, port int) error - ConnectWithSocket(address string) error + ConnectWithSocket(address string, timeout time.Duration) error Disconnect() IsConnected() bool @@ -307,6 +307,12 @@ func (m *SBaseMonitor) onConnectSuccess(conn net.Conn) { m.rwc = conn } +func (m *SBaseMonitor) SetReadDeadlineTimeout(duration time.Duration) { + if m.rwc != nil { + m.rwc.SetReadDeadline(time.Now().Add(duration)) + } +} + func (m *SBaseMonitor) Connect(host string, port int) error { return m.connect("tcp", fmt.Sprintf("%s:%d", host, port)) } diff --git a/pkg/hostman/monitor/qmp.go b/pkg/hostman/monitor/qmp.go index f77b85fda55..dfc554d7610 100644 --- a/pkg/hostman/monitor/qmp.go +++ b/pkg/hostman/monitor/qmp.go @@ -311,7 +311,7 @@ func (m *QmpMonitor) Query(cmd *Command, cb qmpMonitorCallBack) { } } -func (m *QmpMonitor) ConnectWithSocket(address string) error { +func (m *QmpMonitor) ConnectWithSocket(address string, timeout time.Duration) error { err := m.SBaseMonitor.connect("unix", address) if err != nil { return err diff --git a/pkg/util/winutils/winutils.go b/pkg/util/winutils/winutils.go index 663db8ee765..9d92a113240 100644 --- a/pkg/util/winutils/winutils.go +++ b/pkg/util/winutils/winutils.go @@ -311,6 +311,7 @@ func (w *SWinRegTool) listRegistry(spath string, keySeg []string) ([]string, []s } func (w *SWinRegTool) cmdRegistry(spath string, ops []string, retcode []int) bool { + log.Infof("cmdRegistry %s -e %v", GetChntpwPath(), spath) proc := procutils.NewCommand(GetChntpwPath(), "-e", spath) stdin, err := proc.StdinPipe() if err != nil { @@ -369,7 +370,7 @@ func (w *SWinRegTool) cmdRegistry(spath string, ops []string, retcode []int) boo case err := <-done: if err != nil { if exitStatus, ok := proc.GetExitStatus(err); ok { - log.Errorf("exit status: %d", exitStatus) + log.Errorf("ops %v exit status: %d", ops, exitStatus) if in, _ := utils.InArray(exitStatus, retcode); in { return true }