From 59a87fd494529f350f95285b5e85c34e2bf2abbc Mon Sep 17 00:00:00 2001 From: "malik.hadjri" Date: Fri, 22 Nov 2024 16:47:29 -0500 Subject: [PATCH 1/3] DEVPROD-12451 Update last communication time during host reprovisioning --- model/host/host.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/model/host/host.go b/model/host/host.go index e295c2316fd..f1530485ec3 100644 --- a/model/host/host.go +++ b/model/host/host.go @@ -1663,6 +1663,7 @@ func (h *Host) MarkAsReprovisioning(ctx context.Context) error { needsAgentMonitor = h.StartedBy == evergreen.User } + now := time.Now() err := UpdateOne(ctx, bson.M{ IdKey: h.Id, NeedsReprovisionKey: h.NeedsReprovision, @@ -1670,11 +1671,12 @@ func (h *Host) MarkAsReprovisioning(ctx context.Context) error { }, bson.M{ "$set": bson.M{ - AgentStartTimeKey: utility.ZeroTime, - ProvisionedKey: false, - StatusKey: evergreen.HostProvisioning, - NeedsNewAgentKey: needsAgent, - NeedsNewAgentMonitorKey: needsAgentMonitor, + AgentStartTimeKey: utility.ZeroTime, + ProvisionedKey: false, + StatusKey: evergreen.HostProvisioning, + NeedsNewAgentKey: needsAgent, + NeedsNewAgentMonitorKey: needsAgentMonitor, + LastCommunicationTimeKey: now, }, }, ) @@ -1687,6 +1689,7 @@ func (h *Host) MarkAsReprovisioning(ctx context.Context) error { h.Status = evergreen.HostProvisioning h.NeedsNewAgent = needsAgent h.NeedsNewAgentMonitor = needsAgentMonitor + h.LastCommunicationTime = now return nil } From ba1151735620f0a055fbcd1df5864ae0aa353de6 Mon Sep 17 00:00:00 2001 From: "malik.hadjri" Date: Fri, 22 Nov 2024 17:15:58 -0500 Subject: [PATCH 2/3] test --- service/host_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/service/host_test.go b/service/host_test.go index 55b97a2d31d..e3fb189041d 100644 --- a/service/host_test.go +++ b/service/host_test.go @@ -3,6 +3,7 @@ package service import ( "context" "fmt" + "github.com/evergreen-ci/evergreen/units" "io" "net/http" "net/http/httptest" @@ -61,8 +62,9 @@ func TestModifyHostStatusWithUpdateStatus(t *testing.T) { t.Run("SuccessfullyUnquarantinesHostAndMarksAsReprovisioning", func(t *testing.T) { user := user.DBUser{Id: "user"} h := host.Host{ - Id: "h2", - Status: evergreen.HostQuarantined, + Id: "h2", + Provider: evergreen.ProviderNameStatic, + Status: evergreen.HostQuarantined, Distro: distro.Distro{ BootstrapSettings: distro.BootstrapSettings{ Method: distro.BootstrapMethodSSH, @@ -70,6 +72,7 @@ func TestModifyHostStatusWithUpdateStatus(t *testing.T) { }, }, NumAgentCleanupFailures: 10, + LastCommunicationTime: time.Now().Add(-24 * time.Hour), } require.NoError(h.Insert(ctx)) @@ -79,10 +82,16 @@ func TestModifyHostStatusWithUpdateStatus(t *testing.T) { assert.Equal(h.Status, evergreen.HostProvisioning) assert.Equal(host.ReprovisionToNew, h.NeedsReprovision) + // Verify that host monitoring job does not immediately re-quarantine host + // due to long time since last communication + j := units.NewHostMonitoringCheckJob(env, &h, "job_id") + j.Run(ctx) + dbHost, err := host.FindOneId(ctx, h.Id) require.NoError(err) require.NotNil(t, dbHost) assert.Equal(0, dbHost.NumAgentCleanupFailures) + assert.Equal(evergreen.HostProvisioning, dbHost.Status) }) t.Run("FailsToDecommissionStaticHosts", func(t *testing.T) { user := user.DBUser{Id: "user"} From e508088a88b2e01b2ddc9e6c5d5836edc9758ce7 Mon Sep 17 00:00:00 2001 From: "malik.hadjri" Date: Fri, 22 Nov 2024 17:21:39 -0500 Subject: [PATCH 3/3] test --- service/host_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/host_test.go b/service/host_test.go index e3fb189041d..5a03c90e8e9 100644 --- a/service/host_test.go +++ b/service/host_test.go @@ -3,7 +3,6 @@ package service import ( "context" "fmt" - "github.com/evergreen-ci/evergreen/units" "io" "net/http" "net/http/httptest" @@ -19,6 +18,7 @@ import ( "github.com/evergreen-ci/evergreen/model/event" "github.com/evergreen-ci/evergreen/model/host" "github.com/evergreen-ci/evergreen/model/user" + "github.com/evergreen-ci/evergreen/units" "github.com/evergreen-ci/gimlet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require"