Skip to content

Commit

Permalink
Merge branch 'master' of ssh://github.com/buildbuddy-io/buildbuddy
Browse files Browse the repository at this point in the history
  • Loading branch information
tylerwilliams committed Sep 30, 2021
2 parents ab37886 + b421eeb commit 6bb53ac
Show file tree
Hide file tree
Showing 4 changed files with 26 additions and 16 deletions.
20 changes: 19 additions & 1 deletion enterprise/server/cmd/executor/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net"
"net/http"
"os"
"time"

"github.com/buildbuddy-io/buildbuddy/enterprise/server/auth"
"github.com/buildbuddy-io/buildbuddy/enterprise/server/backends/gcs_cache"
Expand Down Expand Up @@ -293,7 +294,24 @@ func main() {
if err != nil {
log.Fatalf("Error initializing executor registration: %s", err)
}
reg.Start(rootContext)

warmupDone := make(chan struct{})
go func() {
executionServer.Warmup()
close(warmupDone)
}()
go func() {
if executorConfig.StartupWarmupMaxWaitSecs != 0 {
warmupMaxWait := time.Duration(executorConfig.StartupWarmupMaxWaitSecs) * time.Second
select {
case <-warmupDone:
case <-time.After(warmupMaxWait):
log.Warningf("Warmup did not finish within %s, resuming startup", warmupMaxWait)
}
}
log.Infof("Registering executor with server.")
reg.Start(rootContext)
}()

go func() {
localServer.Serve(localListener)
Expand Down
5 changes: 4 additions & 1 deletion enterprise/server/remote_execution/executor/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,14 +97,17 @@ func NewExecutor(env environment.Env, id string, options *Options) (*Executor, e
} else {
return nil, status.FailedPreconditionError("Missing health checker in env")
}
go s.runnerPool.WarmupDefaultImage()
return s, nil
}

func (s *Executor) Name() string {
return s.name
}

func (s *Executor) Warmup() {
s.runnerPool.WarmupDefaultImage()
}

func diffTimestamps(startPb, endPb *tspb.Timestamp) time.Duration {
start, _ := ptypes.Timestamp(startPb)
end, _ := ptypes.Timestamp(endPb)
Expand Down
16 changes: 2 additions & 14 deletions enterprise/server/remote_execution/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -519,8 +519,8 @@ func (p *Pool) WarmupDefaultImage() {
start := time.Now()
config := p.env.GetConfigurator().GetExecutorConfig()
executorProps := platform.GetExecutorProperties(config)
// Give the pull up to 1 minute to succeed and 1 minute to create a warm up container.
// In practice I saw clean pulls take about 30 seconds.
// Give the pull up to 2 minute to succeed.
// In practice warmup take about 30 seconds for docker and 75 seconds for firecracker.
timeout := 2 * time.Minute
if config.WarmupTimeoutSecs > 0 {
timeout = time.Duration(config.WarmupTimeoutSecs) * time.Second
Expand Down Expand Up @@ -564,18 +564,6 @@ func (p *Pool) WarmupDefaultImage() {
return err
}
log.Infof("Warmup: %s pulled default image %q in %s", containerType, image, time.Since(start))
tmpDir, err := os.MkdirTemp("", "buildbuddy-warmup-*")
if err != nil {
return err
}
defer os.Remove(tmpDir)
if err = c.Create(egCtx, tmpDir); err != nil {
return err
}
if err := c.Remove(egCtx); err != nil {
return err
}
log.Infof("Warmup: %s finished in %s.", containerType, time.Since(start))
return nil
})
}
Expand Down
1 change: 1 addition & 0 deletions server/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,7 @@ type ExecutorConfig struct {
EnableCASFS bool `yaml:"enable_casfs" usage:"Whether FUSE based CAS filesystem is enabled."`
DefaultImage string `yaml:"default_image" usage:"The default docker image to use to warm up executors or if no platform property is set. Ex: gcr.io/flame-public/executor-docker-default:enterprise-v1.5.4"`
WarmupTimeoutSecs int64 `yaml:"warmup_timeout_secs" usage:"The default time (in seconds) to wait for an executor to warm up i.e. download the default docker image. Default is 120s"`
StartupWarmupMaxWaitSecs int64 `yaml:"startup_warmup_max_wait_secs" usage:"Maximum time to block startup while waiting for default image to be pulled. Default is no wait."`
}

type ContainerRegistryConfig struct {
Expand Down

0 comments on commit 6bb53ac

Please sign in to comment.