Merge branch 'main' into DEVPROD-12257

agent/command/s3_put.go

@@ -379,11 +379,17 @@ retryLoop:
 				}
 				filesList, err = b.Build()
 				if err != nil {
-					return errors.Wrapf(err, "processing local files include filter '%s'",
-						strings.Join(s3pc.LocalFilesIncludeFilter, " "))
+					// Skip erroring since local files include filter should treat files as optional.
+					if strings.Contains(err.Error(), utility.WalkThroughError) {
+						logger.Task().Warningf("Error while building file list: %s", err.Error())
+						return nil
+					} else {
+						return errors.Wrapf(err, "processing local files include filter '%s'",
+							strings.Join(s3pc.LocalFilesIncludeFilter, " "))
+					}
 				}
 				if len(filesList) == 0 {
-					logger.Task().Infof("File filter '%s' matched no files.", strings.Join(s3pc.LocalFilesIncludeFilter, " "))
+					logger.Task().Warningf("File filter '%s' matched no files.", strings.Join(s3pc.LocalFilesIncludeFilter, " "))
 					return nil
 				}
 			}

docs/Project-Configuration/Task-Runtime-Behavior.md

@@ -133,6 +133,8 @@ If `share_procs` is true, the task group will not clean up processes for the
 entire duration of the task. It will only clean up those processes once the
 entire task group is finished.
 
+Check the Agent Logs on a task to see logs about the process cleanup.
+
 ### Task Directory Cleanup
 
 The task working directory is removed when a task finishes as part of cleaning

docs/Reference/Task-Statuses.md

@@ -1,6 +1,6 @@
-# Task Statuses
+# Statuses
 
-This document provides an overview of the various task statuses used in our
+This document provides an overview of the various statuses used in our
 system, along with their meanings, usage contexts, and the logic that determines
 these statuses.
 
@@ -20,6 +20,7 @@ these statuses.
   - [Status Determination Logic](#status-determination-logic)
   - [Display Statuses](#display-statuses)
 - [Task Icon Reference](#task-icon-reference)
+- [Version Statuses](#version-statuses)
 - [Notes](#notes)
 
 ---
@@ -29,7 +30,7 @@ these statuses.
 These statuses indicate tasks that are currently in progress or scheduled to
 run.
 
-### `undispatched`
+#### `undispatched`
 
 **Description**: Indicates one of the following:
 
@@ -42,7 +43,7 @@ run.
 
 ---
 
-### `dispatched`
+#### `dispatched`
 
 **Description**: An [agent](./Glossary.md) has received the task, but the agent
 has not yet notified the system that it's running the task.
@@ -52,7 +53,7 @@ execution.
 
 ---
 
-### `started`
+#### `started`
 
 **Description**: The task is currently running on an agent.
 
@@ -64,7 +65,7 @@ execution.
 
 These statuses indicate that a task has completed its execution.
 
-### `success`
+#### `success`
 
 **Description**: The task has finished successfully without any errors.
 
@@ -75,7 +76,7 @@ These statuses indicate that a task has completed its execution.
 
 ---
 
-### `failed`
+#### `failed`
 
 **Description**: The task has finished but encountered failures. This status
 covers any failure reason, which can be detailed in the task's end details.
@@ -299,6 +300,49 @@ are the icons used for each task status:
 
 ## ![Task Status Legend](../images/task_status_icon_legend.png)
 
+### Version Statuses
+
+Version statuses are very similar to task statuses, and are listed below for additional clarity. 
+
+
+#### `created`
+
+**Description**: Indicates one of the following:
+
+1. **Not Scheduled to Run**: The version is not activated
+   (`version.activated == false`), i.e. no tasks are scheduled.
+2. **Scheduled to Run**: The version is activated (`version.activated == true`) but
+   tasks have not yet been dispatched to an agent.
+
+This is similar to the `undispatched/dispatched` statuses for tasks.
+
+**Usage**: Reflects version that has been created but has run no tasks, regardless of whether they're scheduled to run.
+
+---
+
+#### `started`
+
+**Description**: The version has tasks that are currently running on an agent.
+
+**Usage**: Indicates active execution of version tasks.
+
+---
+
+#### `success`
+
+**Description**: All scheduled version tasks have completed successfully.
+
+**Usage**: Denotes version that have met all requirements and passed all tests for scheduled tasks.
+
+---
+
+#### `failed`
+
+**Description**: The version has finished but all tasks have not succeeded. This
+covers any failure reason, which can be detailed in the individual task's end details.
+
+**Usage**: General failure status for versions that did not complete successfully.
+
 ## Notes
 
 - **Display Statuses**: Statuses like `unscheduled`, `will-run`, `blocked`, and

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/evergreen-ci/poplar v0.0.0-20241014191612-891426af27cb
 	github.com/evergreen-ci/shrub v0.0.0-20231121224157-600e066f9de6
 	github.com/evergreen-ci/timber v0.0.0-20240509150854-9d66df03b40e
-	github.com/evergreen-ci/utility v0.0.0-20241104181620-267066777913
+	github.com/evergreen-ci/utility v0.0.0-20241121161208-c965546993da
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-github/v52 v52.0.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510

go.sum

@@ -267,8 +267,8 @@ github.com/evergreen-ci/timber v0.0.0-20240509150854-9d66df03b40e/go.mod h1:dOoy
 github.com/evergreen-ci/utility v0.0.0-20211026201827-97b21fa2660a/go.mod h1:fuEDytmDhOv+UCUwRPG/qD7mjVkUgx37KEv+thUgHVk=
 github.com/evergreen-ci/utility v0.0.0-20220404192535-d16eb64796e6/go.mod h1:wSui4nRyDZzm2db7Ju7ZzBAelLyVLmcTPJEIh1IdSrc=
 github.com/evergreen-ci/utility v0.0.0-20230104160902-3f0e05a638bd/go.mod h1:vkCEVgfCMIDajzbG/PHZURszI2Y1SuFqNWX9EUxmLLI=
-github.com/evergreen-ci/utility v0.0.0-20241104181620-267066777913 h1:3rQZj320TrlMKANWd69q80EJzKA5cChbnF+jiYYBUA4=
-github.com/evergreen-ci/utility v0.0.0-20241104181620-267066777913/go.mod h1:SHfOAE51SKTA4NLJFvm046A4CNLDr0gfRTQoTM5l1Zc=
+github.com/evergreen-ci/utility v0.0.0-20241121161208-c965546993da h1:KSHVyL7d0Ymmaa0hJ1TV13+ngn610ZU3AWRXtbfHr7g=
+github.com/evergreen-ci/utility v0.0.0-20241121161208-c965546993da/go.mod h1:SHfOAE51SKTA4NLJFvm046A4CNLDr0gfRTQoTM5l1Zc=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=

graphql/query_resolver.go

@@ -944,7 +944,6 @@ func (r *queryResolver) Waterfall(ctx context.Context, options WaterfallOptions)
 		if limitOpt > model.MaxWaterfallVersionLimit {
 			return nil, InputValidationError.Send(ctx, fmt.Sprintf("limit exceeds max limit of %d", model.MaxWaterfallVersionLimit))
 		}
-
 		limit = limitOpt
 	}
 
@@ -974,8 +973,7 @@ func (r *queryResolver) Waterfall(ctx context.Context, options WaterfallOptions)
 		} else if found == nil {
 			graphql.AddError(ctx, PartialError.Send(ctx, fmt.Sprintf("version on or before date '%s' not found", eod.Format(time.DateOnly))))
 		} else {
-			// Offset the order number so the specified version lands nearer to the center of the page.
-			maxOrderOpt = found.RevisionOrderNumber + limit/2 + 1
+			maxOrderOpt = found.RevisionOrderNumber + 1
 		}
 	}
 
@@ -1126,12 +1124,12 @@ func (r *queryResolver) Version(ctx context.Context, versionID string) (*restMod
 func (r *queryResolver) Image(ctx context.Context, imageID string) (*restModel.APIImage, error) {
 	config, err := evergreen.GetConfig(ctx)
 	if err != nil {
-		return nil, InternalServerError.Send(ctx, fmt.Sprintf("getting evergreen configuration: '%s'", err.Error()))
+		return nil, InternalServerError.Send(ctx, fmt.Sprintf("getting evergreen configuration: %s", err.Error()))
 	}
 	c := thirdparty.NewRuntimeEnvironmentsClient(config.RuntimeEnvironments.BaseURL, config.RuntimeEnvironments.APIKey)
 	result, err := c.GetImageInfo(ctx, imageID)
 	if err != nil {
-		return nil, InternalServerError.Send(ctx, fmt.Sprintf("getting image info: '%s'", err.Error()))
+		return nil, InternalServerError.Send(ctx, fmt.Sprintf("getting image info: %s", err.Error()))
 	}
 	apiImage := restModel.APIImage{}
 	apiImage.BuildFromService(*result)

graphql/tests/query/waterfall/results.json

@@ -452,22 +452,6 @@
         "data": {
           "waterfall": {
             "versions": [
-              {
-                "version": {
-                  "createTime": "2020-01-02T01:43:00-05:00",
-                  "id": "evergreen_version2",
-                  "order": 42
-                },
-                "inactiveVersions": null
-              },
-              {
-                "version": {
-                  "createTime": "2019-12-31T19:00:00-05:00",
-                  "id": "evergreen_version1",
-                  "order": 41
-                },
-                "inactiveVersions": null
-              },
               {
                 "version": null,
                 "inactiveVersions": [
@@ -489,7 +473,7 @@
             ],
             "pagination": {
               "nextPageOrder": 39,
-              "prevPageOrder": 42
+              "prevPageOrder": 40
             }
           }
         }

model/project_event.go

@@ -30,8 +30,11 @@ type ProjectSettings struct {
 // event.
 func NewProjectSettingsFromEvent(e ProjectSettingsEvent) ProjectSettings {
 	return ProjectSettings{
-		ProjectRef:         e.ProjectRef,
-		GitHubAppAuth:      e.GitHubAppAuth,
+		ProjectRef: e.ProjectRef,
+		GitHubAppAuth: githubapp.GithubAppAuth{
+			AppID:      e.GitHubAppAuth.AppID,
+			PrivateKey: e.GitHubAppAuth.PrivateKey,
+		},
 		GithubHooksEnabled: e.GithubHooksEnabled,
 		Vars: ProjectVars{
 			Vars:          e.Vars.Vars,
@@ -53,6 +56,21 @@ type ProjectEventVars struct {
 	AdminOnlyVars map[string]bool   `bson:"admin_only_vars" json:"admin_only_vars"`
 }
 
+// ProjectEventGitHubAppAuth contains the GitHub app auth data relevant to
+// project modification events.
+type ProjectEventGitHubAppAuth struct {
+	AppID int64 `bson:"app_id" json:"app_id"`
+	//  PriavetKey contains a redacted placeholder for the private key.
+	PrivateKey []byte `bson:"private_key" json:"private_key"`
+}
+
+// redactPrivateKey redacts the GitHub app's private key so that it's not
+// exposed via the UI or GraphQL.
+func (g *ProjectEventGitHubAppAuth) redactPrivateKey() *ProjectEventGitHubAppAuth {
+	g.PrivateKey = []byte(evergreen.RedactedValue)
+	return g
+}
+
 // ProjectSettingsEvent contains the event data about a single revision of a
 // project's settings.
 type ProjectSettingsEvent struct {
@@ -63,12 +81,12 @@ type ProjectSettingsEvent struct {
 	// project variables are not stored in the database for security reasons.
 	// However, the project event model needs to keep track of which project
 	// variables changed, hence the need for a separate model in that situation.
-	ProjectRef         ProjectRef              `bson:"proj_ref" json:"proj_ref"`
-	GithubHooksEnabled bool                    `bson:"github_hooks_enabled" json:"github_hooks_enabled"`
-	Aliases            []ProjectAlias          `bson:"aliases" json:"aliases"`
-	Subscriptions      []event.Subscription    `bson:"subscriptions" json:"subscriptions"`
-	GitHubAppAuth      githubapp.GithubAppAuth `bson:"github_app_auth" json:"github_app_auth"`
-	Vars               ProjectEventVars        `bson:"vars" json:"vars"`
+	ProjectRef         ProjectRef                `bson:"proj_ref" json:"proj_ref"`
+	GithubHooksEnabled bool                      `bson:"github_hooks_enabled" json:"github_hooks_enabled"`
+	Aliases            []ProjectAlias            `bson:"aliases" json:"aliases"`
+	Subscriptions      []event.Subscription      `bson:"subscriptions" json:"subscriptions"`
+	GitHubAppAuth      ProjectEventGitHubAppAuth `bson:"github_app_auth" json:"github_app_auth"`
+	Vars               ProjectEventVars          `bson:"vars" json:"vars"`
 
 	// The following boolean fields are flags that indicate that a given
 	// field is nil instead of [], since this information is lost when
@@ -89,7 +107,10 @@ func NewProjectSettingsEvent(p ProjectSettings) ProjectSettingsEvent {
 		GithubHooksEnabled: p.GithubHooksEnabled,
 		Aliases:            p.Aliases,
 		Subscriptions:      p.Subscriptions,
-		GitHubAppAuth:      p.GitHubAppAuth,
+		GitHubAppAuth: ProjectEventGitHubAppAuth{
+			AppID:      p.GitHubAppAuth.AppID,
+			PrivateKey: p.GitHubAppAuth.PrivateKey,
+		},
 		Vars: ProjectEventVars{
 			Vars:          p.Vars.Vars,
 			PrivateVars:   p.Vars.PrivateVars,
@@ -175,7 +196,7 @@ func getRedactedVarsCopy(vars map[string]string, modifiedVarNames map[string]str
 // This intentionally makes a copy to avoid potentially modifying the original
 // GitHub app auth, which may be shared by the actual project settings. That
 // way, callers can still access the unredacted auth credentials.
-func getRedactedGitHubAppCopy(auth githubapp.GithubAppAuth, isGHAppKeyModified bool, placeholder string) githubapp.GithubAppAuth {
+func getRedactedGitHubAppCopy(auth ProjectEventGitHubAppAuth, isGHAppKeyModified bool, placeholder string) ProjectEventGitHubAppAuth {
 	if len(auth.PrivateKey) == 0 {
 		return auth
 	}
@@ -256,16 +277,16 @@ func (p *ProjectChangeEvents) RedactGitHubPrivateKey() {
 		}
 		if len(changeEvent.After.GitHubAppAuth.PrivateKey) > 0 && len(changeEvent.Before.GitHubAppAuth.PrivateKey) > 0 {
 			if string(changeEvent.After.GitHubAppAuth.PrivateKey) == string(changeEvent.Before.GitHubAppAuth.PrivateKey) {
-				changeEvent.After.GitHubAppAuth.RedactPrivateKey()
-				changeEvent.Before.GitHubAppAuth.RedactPrivateKey()
+				changeEvent.After.GitHubAppAuth.redactPrivateKey()
+				changeEvent.Before.GitHubAppAuth.redactPrivateKey()
 			} else {
 				changeEvent.After.GitHubAppAuth.PrivateKey = []byte(evergreen.RedactedAfterValue)
 				changeEvent.Before.GitHubAppAuth.PrivateKey = []byte(evergreen.RedactedBeforeValue)
 			}
 		} else if len(changeEvent.After.GitHubAppAuth.PrivateKey) > 0 {
-			changeEvent.After.GitHubAppAuth.RedactPrivateKey()
+			changeEvent.After.GitHubAppAuth.redactPrivateKey()
 		} else if len(changeEvent.Before.GitHubAppAuth.PrivateKey) > 0 {
-			changeEvent.Before.GitHubAppAuth.RedactPrivateKey()
+			changeEvent.Before.GitHubAppAuth.redactPrivateKey()
 		}
 		event.EventLogEntry.Data = changeEvent
 	}

model/project_parser.go

@@ -162,36 +162,6 @@ func (pp *ParserProject) MarshalBSON() ([]byte, error) {
 	return mgobson.Marshal(pp)
 }
 
-// RetryMarshalBSON marshals the BSON and attempts to unmarshal it back to make sure
-// it is valid. It only retries when it fails at reading the BSON, not if it encountered
-// an error while marshalling.
-func (pp *ParserProject) RetryMarshalBSON(retries int) ([]byte, error) {
-	return pp.retryMarshalBSON(retries, retries)
-}
-
-func (pp *ParserProject) retryMarshalBSON(maxRetries, retries int) ([]byte, error) {
-	projBytes, err := bson.Marshal(pp)
-	if err != nil {
-		return nil, errors.Wrap(err, "marshalling project")
-	}
-	_, err = GetProjectFromBSON(projBytes)
-	if err != nil {
-		if retries > 0 {
-			return pp.retryMarshalBSON(maxRetries, retries-1)
-		}
-		return nil, errors.Wrap(err, "unmarshalling project to verify it's integrity")
-	}
-	// TODO (DEVPROD-12560): Remove this log line, potentially the whole retry if it's
-	// never been logged since that means the retries are never actually happening.
-	if retries < maxRetries {
-		grip.Debug(message.Fields{
-			"message": "parser project marshalling succeeded after retries",
-			"retries": maxRetries - retries,
-		})
-	}
-	return projBytes, nil
-}
-
 func (pp *ParserProject) MarshalYAML() (interface{}, error) {
 	for i, pt := range pp.Tasks {
 		for j := range pt.Commands {

model/project_parser_test.go

@@ -2990,7 +2990,7 @@ func TestMarshalBSON(t *testing.T) {
 		Identifier: utility.ToStringPtr("small"),
 	}
 
-	encoded, err := pp.RetryMarshalBSON(5)
+	encoded, err := pp.MarshalBSON()
 	require.NoError(t, err)
 	require.NotEmpty(t, encoded)
 

rest/route/agent.go

@@ -587,7 +587,7 @@ func (h *getParserProjectHandler) Run(ctx context.Context) gimlet.Responder {
 			Message:    fmt.Sprintf("parser project '%s' not found", v.Id),
 		})
 	}
-	projBytes, err := pp.RetryMarshalBSON(5)
+	projBytes, err := pp.MarshalBSON()
 	if err != nil {
 		return gimlet.MakeJSONInternalErrorResponder(errors.Wrap(err, "marshalling project bytes to bson"))
 	}