Rudimentary and hacky implementation of test suites

api/v1/group_routes_test.go

@@ -45,7 +45,7 @@ func getControlSurface(tb testing.TB, testState *lib.TestRunState) *ControlSurfa
 	execScheduler, err := execution.NewScheduler(testState, local.NewController())
 	require.NoError(tb, err)
 
-	me, err := engine.NewMetricsEngine(testState, true)
+	me, err := engine.NewMetricsEngine(testState.Registry, testState.Logger)
 	require.NoError(tb, err)
 
 	ctx, cancel := context.WithCancel(context.Background())

api/v1/setup_teardown_routes_test.go

@@ -141,7 +141,7 @@ func TestSetupData(t *testing.T) {
 
 			execScheduler, err := execution.NewScheduler(testState, local.NewController())
 			require.NoError(t, err)
-			metricsEngine, err := engine.NewMetricsEngine(testState, true)
+			metricsEngine, err := engine.NewMetricsEngine(testState.Registry, testState.Logger)
 			require.NoError(t, err)
 
 			globalCtx, globalCancel := context.WithCancel(context.Background())

api/v1/status_routes_test.go

@@ -116,7 +116,7 @@ func TestPatchStatus(t *testing.T) {
 			execScheduler, err := execution.NewScheduler(testState, local.NewController())
 			require.NoError(t, err)
 
-			metricsEngine, err := engine.NewMetricsEngine(testState, true)
+			metricsEngine, err := engine.NewMetricsEngine(testState.Registry, testState.Logger)
 			require.NoError(t, err)
 
 			globalCtx, globalCancel := context.WithCancel(context.Background())

cmd/agent.go

@@ -85,12 +85,72 @@ func getMetricsHook(
 	}
 }
 
+func loadAndConfigureDistTest(gs *state.GlobalState, distTest *distributed.Test) (*loadedAndConfiguredTest, error) {
+	var options lib.Options
+	if err := json.Unmarshal(distTest.Options, &options); err != nil {
+		return nil, err
+	}
+
+	arc, err := lib.ReadArchive(bytes.NewReader(distTest.Archive))
+	if err != nil {
+		return nil, err
+	}
+
+	registry := metrics.NewRegistry()
+	piState := &lib.TestPreInitState{
+		Logger: gs.Logger,
+		RuntimeOptions: lib.RuntimeOptions{
+			NoThresholds:      null.BoolFrom(true),
+			NoSummary:         null.BoolFrom(true),
+			Env:               arc.Env,
+			CompatibilityMode: null.StringFrom(arc.CompatibilityMode),
+		},
+		Registry:       registry,
+		BuiltinMetrics: metrics.RegisterBuiltinMetrics(registry),
+	}
+
+	initRunner, err := js.NewFromArchive(piState, arc)
+	if err != nil {
+		return nil, err
+	}
+
+	test := &loadedTest{
+		pwd:            arc.Pwd,
+		sourceRootPath: arc.Filename,
+		source: &loader.SourceData{
+			Data: distTest.Archive,
+			URL:  arc.FilenameURL,
+		},
+		fs:           afero.NewMemMapFs(), // TODO: figure out what should be here
+		fileSystems:  arc.Filesystems,
+		preInitState: piState,
+		initRunner:   initRunner,
+	}
+
+	pseudoConsoldatedConfig := applyDefault(Config{Options: options})
+	for _, thresholds := range pseudoConsoldatedConfig.Thresholds {
+		if err = thresholds.Parse(); err != nil {
+			return nil, err
+		}
+	}
+	derivedConfig, err := deriveAndValidateConfig(pseudoConsoldatedConfig, initRunner.IsExecutable, gs.Logger)
+	if err != nil {
+		return nil, err
+	}
+
+	return &loadedAndConfiguredTest{
+		loadedTest:         test,
+		consolidatedConfig: pseudoConsoldatedConfig,
+		derivedConfig:      derivedConfig,
+	}, nil
+}
+
 // TODO: a whole lot of cleanup, refactoring, error handling and hardening
-func getCmdAgent(gs *state.GlobalState) *cobra.Command { //nolint: funlen
+func getCmdAgent(gs *state.GlobalState) *cobra.Command {
 	c := &cmdsRunAndAgent{gs: gs}
 
-	c.loadConfiguredTest = func(cmd *cobra.Command, args []string) (
-		*loadedAndConfiguredTest, execution.Controller, error,
+	c.loadConfiguredTests = func(cmd *cobra.Command, args []string) (
+		[]*loadedAndConfiguredTest, execution.Controller, error,
 	) {
 		conn, err := grpc.Dial(args[0], grpc.WithInsecure())
 		if err != nil {
@@ -108,74 +168,25 @@ func getCmdAgent(gs *state.GlobalState) *cobra.Command { //nolint: funlen
 			return nil, nil, err
 		}
 
-		c.metricsEngineHook = getMetricsHook(gs.Ctx, resp.InstanceID, client, gs.Logger)
-
-		controller, err := distributed.NewAgentController(gs.Ctx, resp.InstanceID, client, gs.Logger)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		var options lib.Options
-		if err := json.Unmarshal(resp.Options, &options); err != nil {
-			return nil, nil, err
-		}
-
-		arc, err := lib.ReadArchive(bytes.NewReader(resp.Archive))
-		if err != nil {
-			return nil, nil, err
-		}
-
-		registry := metrics.NewRegistry()
-		piState := &lib.TestPreInitState{
-			Logger: gs.Logger,
-			RuntimeOptions: lib.RuntimeOptions{
-				NoThresholds:      null.BoolFrom(true),
-				NoSummary:         null.BoolFrom(true),
-				Env:               arc.Env,
-				CompatibilityMode: null.StringFrom(arc.CompatibilityMode),
-			},
-			Registry:       registry,
-			BuiltinMetrics: metrics.RegisterBuiltinMetrics(registry),
-		}
-
-		initRunner, err := js.NewFromArchive(piState, arc)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		test := &loadedTest{
-			pwd:            arc.Pwd,
-			sourceRootPath: arc.Filename,
-			source: &loader.SourceData{
-				Data: resp.Archive,
-				URL:  arc.FilenameURL,
-			},
-			fs:           afero.NewMemMapFs(), // TODO: figure out what should be here
-			fileSystems:  arc.Filesystems,
-			preInitState: piState,
-			initRunner:   initRunner,
-		}
-
-		pseudoConsoldatedConfig := applyDefault(Config{Options: options})
-		for _, thresholds := range pseudoConsoldatedConfig.Thresholds {
-			if err = thresholds.Parse(); err != nil {
+		var configuredTests []*loadedAndConfiguredTest
+		for _, test := range resp.Tests {
+			cTest, err := loadAndConfigureDistTest(gs, test)
+			if err != nil {
 				return nil, nil, err
 			}
+			configuredTests = append(configuredTests, cTest)
 		}
-		derivedConfig, err := deriveAndValidateConfig(pseudoConsoldatedConfig, initRunner.IsExecutable, gs.Logger)
+
+		c.metricsEngineHook = getMetricsHook(gs.Ctx, resp.InstanceID, client, gs.Logger)
+
+		controller, err := distributed.NewAgentController(gs.Ctx, resp.InstanceID, client, gs.Logger)
 		if err != nil {
 			return nil, nil, err
 		}
 
-		configuredTest := &loadedAndConfiguredTest{
-			loadedTest:         test,
-			consolidatedConfig: pseudoConsoldatedConfig,
-			derivedConfig:      derivedConfig,
-		}
-
 		gs.Flags.Address = "" // TODO: fix, this is a hack so agents don't start an API server
 
-		return configuredTest, controller, nil // TODO
+		return configuredTests, controller, nil // TODO
 	}
 
 	agentCmd := &cobra.Command{

cmd/archive.go

@@ -16,7 +16,11 @@ type cmdArchive struct {
 }
 
 func (c *cmdArchive) run(cmd *cobra.Command, args []string) error {
-	test, err := loadAndConfigureLocalTest(c.gs, cmd, args, getPartialConfig)
+	preInitState, err := getPreInitState(c.gs, cmd)
+	if err != nil {
+		return err
+	}
+	test, err := loadAndConfigureLocalTest(c.gs, preInitState, cmd, args[0], getPartialConfig)
 	if err != nil {
 		return err
 	}
@@ -88,7 +92,7 @@ func getCmdArchive(gs *state.GlobalState) *cobra.Command {
 
 An archive is a fully self-contained test run, and can be executed identically elsewhere.`,
 		Example: exampleText,
-		Args:    cobra.ExactArgs(1),
+		Args:    cobra.ExactArgs(1), // TODO: remove and support test suites?
 		RunE:    c.run,
 	}
 

cmd/cloud.go

@@ -84,7 +84,11 @@ func (c *cmdCloud) run(cmd *cobra.Command, args []string) error {
 	)
 	printBar(c.gs, progressBar)
 
-	test, err := loadAndConfigureLocalTest(c.gs, cmd, args, getPartialConfig)
+	preInitState, err := getPreInitState(c.gs, cmd)
+	if err != nil {
+		return err
+	}
+	test, err := loadAndConfigureLocalTest(c.gs, preInitState, cmd, args[0], getPartialConfig)
 	if err != nil {
 		return err
 	}

cmd/coordinator.go

@@ -28,36 +28,52 @@ type cmdCoordinator struct {
 func (c *cmdCoordinator) run(cmd *cobra.Command, args []string) (err error) {
 	ctx, runAbort := execution.NewTestRunContext(c.gs.Ctx, c.gs.Logger)
 
-	test, err := loadAndConfigureLocalTest(c.gs, cmd, args, getPartialConfig)
+	tests, err := loadAndConfigureLocalTests(c.gs, cmd, args, getPartialConfig)
 	if err != nil {
 		return err
 	}
 
-	// Only consolidated options, not derived
-	testRunState, err := test.buildTestRunState(test.consolidatedConfig.Options)
+	// TODO: refactor at some point, this limits us to handleSummary() from the first test
+	firstTest := tests[0]
+	// TODO: refactor - this is safe, preInitState is the same for all tests,
+	// but it's still icky to get it that way
+	preInitState := firstTest.preInitState
+	metricsEngine, err := engine.NewMetricsEngine(preInitState.Registry, c.gs.Logger)
 	if err != nil {
 		return err
 	}
 
-	shouldProcessMetrics := !testRunState.RuntimeOptions.NoSummary.Bool || !testRunState.RuntimeOptions.NoThresholds.Bool
-	metricsEngine, err := engine.NewMetricsEngine(testRunState, shouldProcessMetrics)
-	if err != nil {
-		return err
+	testArchives := make([]*lib.Archive, len(tests))
+	for i, test := range tests {
+		runState, err := test.buildTestRunState(test.consolidatedConfig.Options)
+		if err != nil {
+			return err
+		}
+
+		// We get the thresholds from all tests
+		testArchives[i] = runState.Runner.MakeArchive()
+		err = metricsEngine.InitSubMetricsAndThresholds(
+			test.derivedConfig.Options,
+			preInitState.RuntimeOptions.NoThresholds.Bool,
+		)
+		if err != nil {
+			return err
+		}
 	}
 
 	coordinator, err := distributed.NewCoordinatorServer(
-		c.instanceCount, test.initRunner.MakeArchive(), metricsEngine, c.gs.Logger,
+		c.instanceCount, testArchives, metricsEngine, c.gs.Logger,
 	)
 	if err != nil {
 		return err
 	}
 
-	if !testRunState.RuntimeOptions.NoSummary.Bool {
+	if !preInitState.RuntimeOptions.NoSummary.Bool {
 		defer func() {
 			c.gs.Logger.Debug("Generating the end-of-test summary...")
-			summaryResult, serr := test.initRunner.HandleSummary(ctx, &lib.Summary{
+			summaryResult, serr := firstTest.initRunner.HandleSummary(ctx, &lib.Summary{
 				Metrics:         metricsEngine.ObservedMetrics,
-				RootGroup:       test.initRunner.GetDefaultGroup(),
+				RootGroup:       firstTest.initRunner.GetDefaultGroup(),
 				TestRunDuration: coordinator.GetCurrentTestRunDuration(),
 				NoColor:         c.gs.Flags.NoColor,
 				UIState: lib.UIState{
@@ -74,9 +90,9 @@ func (c *cmdCoordinator) run(cmd *cobra.Command, args []string) (err error) {
 		}()
 	}
 
-	if !testRunState.RuntimeOptions.NoThresholds.Bool {
+	if !preInitState.RuntimeOptions.NoThresholds.Bool {
 		getCurrentTestDuration := coordinator.GetCurrentTestRunDuration
-		finalizeThresholds := metricsEngine.StartThresholdCalculations(runAbort, getCurrentTestDuration)
+		finalizeThresholds := metricsEngine.StartThresholdCalculations(nil, runAbort, getCurrentTestDuration)
 
 		defer func() {
 			// This gets called after all of the outputs have stopped, so we are

cmd/inspect.go

@@ -19,9 +19,14 @@ func getCmdInspect(gs *state.GlobalState) *cobra.Command {
 		Use:   "inspect [file]",
 		Short: "Inspect a script or archive",
 		Long:  `Inspect a script or archive.`,
-		Args:  cobra.ExactArgs(1),
+		Args:  cobra.ExactArgs(1), // TODO: remove and support test suites?
 		RunE: func(cmd *cobra.Command, args []string) error {
-			test, err := loadLocalTest(gs, cmd, args)
+			preInitState, err := getPreInitState(gs, cmd)
+			if err != nil {
+				return err
+			}
+
+			test, err := loadLocalTest(gs, preInitState, cmd, args[0])
 			if err != nil {
 				return err
 			}