-
Notifications
You must be signed in to change notification settings - Fork 95
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[RB] Fix fetching remote build outputs (#7770)
Remote bazel can be used to build a target on a remote runner, and then the CLI will fetch the build outputs to be used locally. A couple things were broken: * The CLI uses certain build events to identify and fetch build outputs, but [this PR](#4490) moved target-level events from the `Event` field to `TargetGroups`. This PR updates the CLI to use the new field * Verifies that the build uses `--remote_upload_local_results` so that remote build outputs can be fetched from the cache * Command parsing failed if there were startup commands * The error code returned when we build remotely and run locally
- Loading branch information
1 parent
095a257
commit 089a79a
Showing
3 changed files
with
136 additions
and
70 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,13 @@ | ||
| package remote_bazel_test | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "math" | ||
| "os" | ||
| "os/exec" | ||
| "path/filepath" | ||
| "strings" | ||
| "testing" | ||
| "time" | ||
|
|
@@ -87,6 +90,33 @@ func waitForInvocationStatus(t *testing.T, ctx context.Context, bb bbspb.BuildBu | |
| require.FailNowf(t, "timeout", "Timed out waiting for invocation to reach expected status %v", expectedStatus) | ||
| } | ||
|
|
||
| func clonePrivateTestRepo(t *testing.T) { | ||
| repoName := "private-test-repo" | ||
| // If you need to re-generate this PAT, it should only have read access to | ||
| // `private-test-repo`, and should be saved as a BB secret in all environments. | ||
| username := "maggie-lou" | ||
| personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") | ||
| repoURLWithToken := fmt.Sprintf("https://%s:%[email protected]/buildbuddy-io/private-test-repo.git", username, personalAccessToken) | ||
|
|
||
| // Use a dir that is persisted on recycled runners | ||
| rootDir := "/root/workspace/remote-bazel-integration-test" | ||
| err := os.Setenv("HOME", rootDir) | ||
| require.NoError(t, err) | ||
|
|
||
| err = os.MkdirAll(rootDir, 0755) | ||
| require.NoError(t, err) | ||
|
|
||
| if _, err := os.Stat(fmt.Sprintf("%s/%s", rootDir, repoName)); os.IsNotExist(err) { | ||
| output := testshell.Run(t, rootDir, fmt.Sprintf("git clone %s --filter=blob:none --depth=1", repoURLWithToken)) | ||
| require.NotContains(t, output, "fatal") | ||
| } | ||
|
|
||
| repoDir := fmt.Sprintf("%s/%s", rootDir, repoName) | ||
| err = os.Chdir(repoDir) | ||
| require.NoError(t, err) | ||
| testshell.Run(t, repoDir, "git pull") | ||
| } | ||
|
|
||
| func TestWithPublicRepo(t *testing.T) { | ||
| // Use a dir that is persisted on recycled runners | ||
| rootDir := "/root/workspace/remote-bazel-integration-test" | ||
|
|
@@ -154,36 +184,17 @@ func TestWithPublicRepo(t *testing.T) { | |
| } | ||
|
|
||
| func TestWithPrivateRepo(t *testing.T) { | ||
| repoName := "private-test-repo" | ||
| // If you need to re-generate this PAT, it should only have read access to | ||
| // `private-test-repo`, and should be saved as a BB secret in all environments. | ||
| username := "maggie-lou" | ||
| personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") | ||
| repoURLWithToken := fmt.Sprintf("https://%s:%[email protected]/buildbuddy-io/private-test-repo.git", username, personalAccessToken) | ||
| clonePrivateTestRepo(t) | ||
|
|
||
| // Use a dir that is persisted on recycled runners | ||
| rootDir := "/root/workspace/remote-bazel-integration-test" | ||
| err := os.Setenv("HOME", rootDir) | ||
| require.NoError(t, err) | ||
|
|
||
| err = os.MkdirAll(rootDir, 0755) | ||
| require.NoError(t, err) | ||
|
|
||
| if _, err := os.Stat(fmt.Sprintf("%s/%s", rootDir, repoName)); os.IsNotExist(err) { | ||
| output := testshell.Run(t, rootDir, fmt.Sprintf("git clone %s --filter=blob:none --depth=1", repoURLWithToken)) | ||
| require.NotContains(t, output, "fatal") | ||
| } | ||
|
|
||
| err = os.Chdir(fmt.Sprintf("%s/%s", rootDir, repoName)) | ||
| require.NoError(t, err) | ||
| personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") | ||
|
|
||
| // Run a server and executor locally to run remote bazel against | ||
| env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken) | ||
|
|
||
| // Create a workflow for the same repo - will be used to fetch the git token | ||
| dbh := env.GetDBHandle() | ||
| require.NotNil(t, dbh) | ||
| err = dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ | ||
| err := dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ | ||
| RepoURL: "https://github.com/buildbuddy-io/private-test-repo", | ||
| GroupID: env.GroupID1, | ||
| }) | ||
|
|
@@ -265,28 +276,9 @@ func runLocalServerAndExecutor(t *testing.T, githubToken string) (*rbetest.Env, | |
| } | ||
|
|
||
| func TestCancel(t *testing.T) { | ||
| repoName := "private-test-repo" | ||
| // If you need to re-generate this PAT, it should only have read access to | ||
| // `private-test-repo`, and should be saved as a BB secret in all environments. | ||
| username := "maggie-lou" | ||
| personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") | ||
| repoURLWithToken := fmt.Sprintf("https://%s:%[email protected]/buildbuddy-io/private-test-repo.git", username, personalAccessToken) | ||
|
|
||
| // Use a dir that is persisted on recycled runners | ||
| rootDir := "/root/workspace/remote-bazel-integration-test" | ||
| err := os.Setenv("HOME", rootDir) | ||
| require.NoError(t, err) | ||
|
|
||
| err = os.MkdirAll(rootDir, 0755) | ||
| require.NoError(t, err) | ||
| clonePrivateTestRepo(t) | ||
|
|
||
| if _, err := os.Stat(fmt.Sprintf("%s/%s", rootDir, repoName)); os.IsNotExist(err) { | ||
| output := testshell.Run(t, rootDir, fmt.Sprintf("git clone %s --filter=blob:none --depth=1", repoURLWithToken)) | ||
| require.NotContains(t, output, "fatal") | ||
| } | ||
|
|
||
| err = os.Chdir(fmt.Sprintf("%s/%s", rootDir, repoName)) | ||
| require.NoError(t, err) | ||
| personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") | ||
|
|
||
| // Run a server and executor locally to run remote bazel against | ||
| env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken) | ||
|
|
@@ -299,7 +291,7 @@ func TestCancel(t *testing.T) { | |
| // Create a workflow for the same repo - will be used to fetch the git token | ||
| dbh := env.GetDBHandle() | ||
| require.NotNil(t, dbh) | ||
| err = dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ | ||
| err := dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ | ||
| RepoURL: "https://github.com/buildbuddy-io/private-test-repo", | ||
| GroupID: env.GroupID1, | ||
| }) | ||
|
|
@@ -366,3 +358,73 @@ func TestCancel(t *testing.T) { | |
|
|
||
| waitForInvocationStatus(t, ctx, bbClient, reqCtx, invocationID, inspb.InvocationStatus_DISCONNECTED_INVOCATION_STATUS) | ||
| } | ||
|
|
||
| func TestFetchRemoteBuildOutputs(t *testing.T) { | ||
| clonePrivateTestRepo(t) | ||
|
|
||
| // Run a server and executor locally to run remote bazel against | ||
| personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") | ||
| env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken) | ||
|
|
||
| // Create a workflow for the same repo - will be used to fetch the git token | ||
| dbh := env.GetDBHandle() | ||
| require.NotNil(t, dbh) | ||
| err := dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ | ||
| RepoURL: "https://github.com/buildbuddy-io/private-test-repo", | ||
| GroupID: env.GroupID1, | ||
| }) | ||
| require.NoError(t, err) | ||
|
|
||
| // Run remote bazel | ||
| randomStr := time.Now().String() | ||
| exitCode, err := remotebazel.HandleRemoteBazel([]string{ | ||
| fmt.Sprintf("--remote_runner=%s", bbServer.GRPCAddress()), | ||
| // Have the ci runner use the "none" isolation type because it's simpler | ||
| // to setup than a firecracker runner | ||
| "--runner_exec_properties=workload-isolation-type=none", | ||
| "--runner_exec_properties=container-image=", | ||
| // Ensure the build is happening on a clean runner, because if the build | ||
| // artifact is locally cached, we won't upload it to the remote cache | ||
| // and we won't be able to fetch it. | ||
| "--runner_exec_properties=instance_name=" + randomStr, | ||
| // Pass a startup flag to test parsing | ||
| "--digest_function=BLAKE3", | ||
| "build", | ||
| ":hello_world_go", | ||
| fmt.Sprintf("--remote_header=x-buildbuddy-api-key=%s", env.APIKey1)}) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, exitCode) | ||
|
|
||
| // Check that the remote build output was fetched locally. | ||
| // The outputs will be downloaded to a directory that may change with the platform, | ||
| // so recursively search for the build output named `hello_world_go`. | ||
| findFile := func(rootDir, targetFile string) (string, error) { | ||
| var outputPath string | ||
| err := filepath.WalkDir(rootDir, func(path string, d os.DirEntry, err error) error { | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if !d.IsDir() && d.Name() == targetFile { | ||
| outputPath = path | ||
| return filepath.SkipAll // Stop searching further once the file is found | ||
| } | ||
|
|
||
| return nil | ||
| }) | ||
| return outputPath, err | ||
| } | ||
| downloadedOutputPath, err := findFile(remotebazel.BuildBuddyArtifactDir, "hello_world_go") | ||
| require.NoError(t, err) | ||
|
|
||
| // Make sure we can successfully run the fetched binary. | ||
| err = os.Chmod(downloadedOutputPath, 0755) | ||
| require.NoError(t, err) | ||
|
|
||
| var buf bytes.Buffer | ||
| cmd := exec.Command(downloadedOutputPath) | ||
| cmd.Stdout = &buf | ||
| err = cmd.Run() | ||
| require.NoError(t, err) | ||
| require.Equal(t, "Hello! I'm a go program.\n", buf.String()) | ||
| } | ||