Skip to content

Commit

Permalink
Handle the review comments in PR 4407
Browse files Browse the repository at this point in the history 
- changed the function to retrieveDeviceNodeName() and call it only at
  the start of domainmgr Run()
- remove the ctx.hvTypeKube and status.IsDNidNode checks in the
  domainmgr.go code; also remove the status.DomainConfigDeleted. we now
  rely on normal domain handling of delete/cleanup work flow
- fixed a bug where nodeName with underscore, which is not allowed in
  kubernetes names
- changed the zedmanager/handleclusterstatus.go code to PoC code base,
  and commented out one line for later PR to handle
- implemented the scheme when kubevirt can not contact kubernetes
  API-server or the cluster does not have the POD/VMI being scheduled
  yet, we return the 'Unknown' status now. It keeps a starting 'unknown'
  timestamp per application
- also if the 'unknown' status lasts longer than 5 minutes, it changes
  into 'Halt' status back to domainmgr
- updated 'zedkube.md' section 'Handle Domain Apps Status in domainmgr'
  for the above behavior

Signed-off-by: Naiming Shen <[email protected]>
  • Loading branch information
@naiming-zededa
naiming-zededa committed Dec 28, 2024
1 parent 7db2529 commit ee49db4
Show file tree
Hide file tree
Showing 7 changed files with 134 additions and 153 deletions.
86 changes: 14 additions & 72 deletions pkg/pillar/cmd/domainmgr/domainmgr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,12 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar
}
log.Noticef("processed GCComplete")

// Get the EdgeNode info, needed for kubevirt clustering
err = domainCtx.retrieveDeviceNodeName()
if err != nil {
log.Fatal(err)
}

if !domainCtx.setInitialUsbAccess {
log.Functionf("GCComplete but not setInitialUsbAccess => first boot")
// Enable USB keyboard and storage
Expand Down Expand Up @@ -1102,13 +1108,6 @@ func maybeRetryBoot(ctx *domainContext, status *types.DomainStatus) {
return
}

err := ctx.retrieveNodeNameAndUUID()
if err != nil {
log.Errorf("maybeRetryBoot(%s) retrieveNodeNameAndUUID failed: %s",
status.Key(), err)
return
}

if status.Activated && status.BootFailed {
log.Functionf("maybeRetryBoot(%s) clearing bootFailed since Activated",
status.Key())
Expand Down Expand Up @@ -1365,7 +1364,6 @@ func handleCreate(ctx *domainContext, key string, config *types.DomainConfig) {
State: types.INSTALLED,
VmConfig: config.VmConfig,
Service: config.Service,
IsDNidNode: config.IsDNidNode,
}

status.VmConfig.CPUs = make([]int, 0)
Expand Down Expand Up @@ -1575,13 +1573,6 @@ func doActivate(ctx *domainContext, config types.DomainConfig,
log.Functionf("doActivate(%v) for %s",
config.UUIDandVersion, config.DisplayName)

err := ctx.retrieveNodeNameAndUUID()
if err != nil {
log.Errorf("doActivate(%s) retrieveNodeNameAndUUID failed: %s",
status.Key(), err)
return
}

if ctx.cpuPinningSupported {
if err := assignCPUs(ctx, &config, status); err != nil {
log.Warnf("failed to assign CPUs for %s", config.DisplayName)
Expand Down Expand Up @@ -1807,18 +1798,6 @@ func doActivateTail(ctx *domainContext, status *types.DomainStatus,
log.Errorf("domain start for %s: %s", status.DomainName, err)
status.SetErrorNow(err.Error())

// HvKube case
if ctx.hvTypeKube && !status.IsDNidNode {
log.Noticef("doActivateTail(%v) we are not DNiD, skip delete app", status.DomainName)
return
}

// Only send delete if DomainConfig is not deleted
// detail see the zedkube.md section 'Handling Domain Deletion in Domainmgr'
if ctx.hvTypeKube && !status.DomainConfigDeleted {
log.Noticef("doActivateTail(%v) DomainConfig exists, skip delete app", status.DomainName)
return
}
// Delete
if err := hyper.Task(status).Delete(status.DomainName); err != nil {
log.Errorf("failed to delete domain: %s (%v)", status.DomainName, err)
Expand Down Expand Up @@ -1849,17 +1828,6 @@ func doActivateTail(ctx *domainContext, status *types.DomainStatus,
log.Errorf("doActivateTail(%v) failed for %s: %s",
status.UUIDandVersion, status.DisplayName, err)

if ctx.hvTypeKube && !status.IsDNidNode {
log.Noticef("doActivateTail(%v) we are not DNiD, skip delete app", status.DomainName)
return
}
// Only send delete if DomainConfig is not deleted
// detail see the zedkube.md section 'Handling Domain Deletion in Domainmgr'
if ctx.hvTypeKube && !status.DomainConfigDeleted {
log.Noticef("doActivateTail(%v) DomainConfig exists, skip delete app", status.DomainName)
return
}

// Delete
if err := hyper.Task(status).Delete(status.DomainName); err != nil {
log.Errorf("failed to delete domain: %s (%v)", status.DomainName, err)
Expand Down Expand Up @@ -1961,17 +1929,6 @@ func doInactivate(ctx *domainContext, status *types.DomainStatus, impatient bool
}

if status.DomainId != 0 {
if ctx.hvTypeKube && !status.IsDNidNode {
log.Noticef("doInactivate(%v) we are not DNiD, skip delete app", status.DomainName)
return
}
// Only send delete if DomainConfig is not deleted
// detail see the zedkube.md section 'Handling Domain Deletion in Domainmgr'
if ctx.hvTypeKube && !status.DomainConfigDeleted {
log.Noticef("doInactivate(%v) DomainConfig exists, skip delete app", status.DomainName)
return
}

if err := hyper.Task(status).Delete(status.DomainName); err != nil {
log.Errorf("Failed to delete domain %s (%v)", status.DomainName, err)
} else {
Expand Down Expand Up @@ -2559,16 +2516,6 @@ func handleDelete(ctx *domainContext, key string, status *types.DomainStatus) {
// No point in publishing metrics any more
ctx.pubDomainMetric.Unpublish(status.Key())

if ctx.hvTypeKube && !status.IsDNidNode {
log.Noticef("handleDelete(%v) we are not DNiD, skip delete app", status.DomainName)
return
}

// set the DomainConfigDeleted for kubernetes to remove the domain
// detail see the zedkube.md section 'Handling Domain Deletion in Domainmgr'
status.DomainConfigDeleted = true
log.Noticef("handleDelete(%v) DomainConfigDeleted", status.DomainName)

err := hyper.Task(status).Delete(status.DomainName)
if err != nil {
log.Errorln(err)
Expand Down Expand Up @@ -2617,10 +2564,6 @@ func DomainShutdown(ctx *domainContext, status types.DomainStatus, force bool) e
// Stop the domain
log.Functionf("Stopping domain - %s", status.DomainName)

if ctx.hvTypeKube && !status.IsDNidNode {
log.Noticef("DomainShutdown(%v) we are not DNiD, skip delete app", status.DomainName)
return nil
}
err = hyper.Task(&status).Stop(status.DomainName, force)

return err
Expand Down Expand Up @@ -3710,15 +3653,14 @@ func lookupCapabilities(ctx *domainContext) (*types.Capabilities, error) {
return &capabilities, nil
}

func (ctx *domainContext) retrieveNodeNameAndUUID() error {
if ctx.nodeName == "" {
NodeInfo, err := ctx.subEdgeNodeInfo.Get("global")
if err != nil {
log.Errorf("retrieveNodeNameAndUUID: can't get edgeNodeInfo %v", err)
return err
}
enInfo := NodeInfo.(types.EdgeNodeInfo)
ctx.nodeName = strings.ToLower(enInfo.DeviceName)
func (ctx *domainContext) retrieveDeviceNodeName() error {
NodeInfo, err := ctx.subEdgeNodeInfo.Get("global")
if err != nil {
log.Errorf("retrieveDeviceNodeName: can't get edgeNodeInfo %v", err)
return err
}
enInfo := NodeInfo.(types.EdgeNodeInfo)
ctx.nodeName = strings.ReplaceAll(strings.ToLower(enInfo.DeviceName), "_", "-")
log.Noticef("retrieveDeviceNodeName: devicename, NodeInfo %v", NodeInfo) // XXX
return nil
}
13 changes: 6 additions & 7 deletions pkg/pillar/cmd/zedagent/parseconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3214,13 +3214,8 @@ func parseEdgeNodeClusterConfig(getconfigCtx *getconfigContext,
ctx := getconfigCtx.zedagentCtx
zcfgCluster := config.GetCluster()
if zcfgCluster == nil {
log.Functionf("parseEdgeNodeClusterConfig: No EdgeNodeClusterConfig, Unpublishing")
pub := ctx.pubEdgeNodeClusterConfig
items := pub.GetAll()
if len(items) > 0 {
log.Functionf("parseEdgeNodeClusterConfig: Unpublishing EdgeNodeClusterConfig")
ctx.pubEdgeNodeClusterConfig.Unpublish("global")
}
log.Functionf("parseEdgeNodeClusterConfig: Unpublishing EdgeNodeClusterConfig")
ctx.pubEdgeNodeClusterConfig.Unpublish("global")
return
}
ipAddr, ipNet, err := net.ParseCIDR(zcfgCluster.GetClusterIpPrefix())
Expand All @@ -3231,6 +3226,10 @@ func parseEdgeNodeClusterConfig(getconfigCtx *getconfigContext,
ipNet.IP = ipAddr

joinServerIP := net.ParseIP(zcfgCluster.GetJoinServerIp())
if joinServerIP == nil {
log.Errorf("handleEdgeNodeConfigItem: parse JoinServerIP failed")
return
}
var isJoinNode bool
// deduce the bootstrap node status from clusterIPPrefix and joinServerIP
if ipAddr.Equal(joinServerIP) { // deduce the bootstrap node status from
Expand Down
2 changes: 1 addition & 1 deletion pkg/pillar/cmd/zedkube/applogs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ func (z *zedkube) getnodeNameAndUUID() error {
return err
}
enInfo := NodeInfo.(types.EdgeNodeInfo)
z.nodeName = strings.ToLower(enInfo.DeviceName)
z.nodeName = strings.ReplaceAll(strings.ToLower(enInfo.DeviceName), "_", "-")
z.nodeuuid = enInfo.DeviceID.String()
}
return nil
Expand Down
45 changes: 31 additions & 14 deletions pkg/pillar/cmd/zedmanager/handleclusterapp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,38 +6,55 @@ package zedmanager
import "github.com/lf-edge/eve/pkg/pillar/types"

func handleENClusterAppStatusCreate(ctxArg interface{}, key string, configArg interface{}) {
log.Functionf("handleENClusterAppStatusCreate(%s)", key)
log.Noticef("handleENClusterAppStatusCreate(%s)", key)
ctx := ctxArg.(*zedmanagerContext)
status := configArg.(types.ENClusterAppStatus)
handleENClusterAppStatusImpl(ctx, key, &status)
}

func handleENClusterAppStatusModify(ctxArg interface{}, key string, configArg interface{}, oldConfigArg interface{}) {
log.Functionf("handleENClusterAppStatusModify(%s)", key)
log.Noticef("handleENClusterAppStatusModify(%s)", key)
ctx := ctxArg.(*zedmanagerContext)
status := configArg.(types.ENClusterAppStatus)
handleENClusterAppStatusImpl(ctx, key, &status)
}

func handleENClusterAppStatusDelete(ctxArg interface{}, key string, configArg interface{}) {
log.Functionf("handleENClusterAppStatusDelete(%s)", key)
log.Noticef("handleENClusterAppStatusDelete(%s)", key)
ctx := ctxArg.(*zedmanagerContext)
//status := configArg.(types.ENClusterAppStatus)
handleENClusterAppStatusImpl(ctx, key, nil)
status := configArg.(types.ENClusterAppStatus)
handleENClusterAppStatusImpl(ctx, key, &status)
}

func handleENClusterAppStatusImpl(ctx *zedmanagerContext, key string, status *types.ENClusterAppStatus) {

log.Functionf("handleENClusterAppStatusImpl(%s) for app-status %v", key, status)
pub := ctx.pubAppInstanceStatus
items := pub.GetAll()
for _, st := range items {
aiStatus := st.(types.AppInstanceStatus)
if aiStatus.UUIDandVersion.UUID.String() == key {
log.Functionf("handleENClusterAppStatusImpl(%s) found ai status, update", key)
aiStatus := lookupAppInstanceStatus(ctx, key)
log.Noticef("handleENClusterAppStatusImpl(%s) for app-status %v aiStatus %v", key, status, aiStatus)

if status.ScheduledOnThisNode {
if aiStatus == nil {
// This could happen if app failover to other node and failing back to this designated node.
// One scenario is node reboot. Kubernetes told us that app is scheduled on this node.
aiConfig := lookupAppInstanceConfig(ctx, key, false)
if aiConfig == nil {
log.Errorf("handleENClusterAppStatusImpl(%s) AppInstanceConfig missing for app", key)
return
}
// XXX this will be handled in later PR in clustering and zedmanager code
//handleCreateAppInstanceStatus(ctx, *aiConfig)
} else {
// Nothing to do, we already have aiStatus
log.Functionf("handleENClusterAppStatusImpl(%s) for app-status %v aiStatus %v", key, status, aiStatus)
return
}
} else { // not scheduled here.

updateAIStatusUUID(ctx, aiStatus.UUIDandVersion.UUID.String())
break
// if aiStatus is not present, nothing to do
if aiStatus != nil {
// If I am not scheduled here, just unpublish the AIStatus.
// We probably had app running on this node earlier before failover.
unpublishAppInstanceStatus(ctx, aiStatus)
}

}
}
8 changes: 2 additions & 6 deletions pkg/pillar/docs/zedkube.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,9 @@ kubenodeop handles NodeDrainRequest objects which zedkube subscribes to, initiat

## Applications under Kubevirt Mode

### Handling Domain Deletion in Domainmgr
### Handle Domain Apps Status in domainmgr

In normal cases of EVE application launching and running, the domainmgr handles the configuration creation, starts the domain, and monitors the domain's running status. If the starting and monitoring status is not in the running state, then there is something wrong with the runtime process, and the domain is normally stopped and deleted by the domainmgr. Domainmgr keeps a timer, usually 10 minutes, to retry starting the domain again later.

When the application is launched and managed in KubeVirt mode, the Kubernetes cluster is provisioned for this application, being a VMI (Virtual Machine Instance) replicaSet object or a Pod replicaSet object. It uses a declarative approach to manage the desired state of the applications. The configurations are saved in the Kubernetes database for the Kubernetes controller to use to ensure the objects eventually achieve the correct state if possible. Any particular VMI/Pod state of a domain may not be in working condition at the time when EVE domainmgr checks. In the domainmgr code running in KubeVirt mode, it normally skips the hyper.Task().Delete() or hyper.Task().Stop() in domainmgr.go, and lets the Kubernetes cluster have a chance to work its way to bring up the application to the running state.

The exception to the above is in the case of the application itself being removed from the AppInstanceConfig, in which case, the DomainStatus of this application will be deleted, and we have a new boolean DomainConfigDeleted to be set if the DomainStatus is pending for deletion. When the DomainStatus of DomainConfigDeleted is set, the code in domainmgr will allow the Stop() or Delete() operations for Kubernetes to remove the replicaSet of the application.
When the application is launched and managed in KubeVirt mode, the Kubernetes cluster is provisioned for this application, being a VMI (Virtual Machine Instance) replicaSet object or a Pod replicaSet object. It uses a declarative approach to manage the desired state of the applications. The configurations are saved in the Kubernetes database for the Kubernetes controller to use to ensure the objects eventually achieve the correct state if possible. Any particular VMI/Pod state of a domain may not be in working condition at the time when EVE domainmgr checks. In the domainmgr code running in KubeVirt mode, if it can not contact the Kubernetes API server to query about the application, or if the application itself has not be started yet in the cluster, the kubervirt.go will return the 'Unknown' status back. It will keep a 'Unknown' status starting timestamp per application. If the 'Unknown' status lasts longer then 5 minutes, the status functions in kubevirt.go will return 'Halt' status back to domainmgr. The timestamp will be cleared once it can get the application status from the kubernetes.

## Kubernetes Node Draining

Expand Down
Loading

0 comments on commit ee49db4

Please sign in to comment.