Merge branch 'vsukhin/feature/stream-service-logs' into sandbox/twrc

Signed-off-by: Vladislav Sukhin <[email protected]> # Conflicts: # cmd/kubectl-testkube/commands/testworkflows/run.go # cmd/tcl/testworkflow-toolkit/commands/execute.go # go.mod # go.sum
kubeshop · Nov 28, 2024 · c220ccf · c220ccf
2 parents 6ea76ad + 9c03fe4
commit c220ccf
Show file tree

Hide file tree

Showing 129 changed files with 7,038 additions and 1,226 deletions.
diff --git a/.builds-darwin.goreleaser.yml b/.builds-darwin.goreleaser.yml
@@ -20,5 +20,6 @@ builds:
       - -X main.builtBy=goreleaser
       - -X github.com/kubeshop/testkube/pkg/telemetry.TestkubeMeasurementID={{.Env.ANALYTICS_TRACKING_ID}}
       - -X github.com/kubeshop/testkube/pkg/telemetry.TestkubeMeasurementSecret={{.Env.ANALYTICS_API_KEY}}
+      - -X github.com/kubeshop/testkube/pkg/diagnostics/validators/license.KeygenOfflinePublicKey={{.Env.KEYGEN_PUBLIC_KEY}}
 archives:
   - format: binary
diff --git a/.builds-linux.goreleaser.yml b/.builds-linux.goreleaser.yml
@@ -33,6 +33,7 @@ builds:
       - -X main.builtBy=goreleaser
       - -X github.com/kubeshop/testkube/pkg/telemetry.TestkubeMeasurementID={{.Env.ANALYTICS_TRACKING_ID}}
       - -X github.com/kubeshop/testkube/pkg/telemetry.TestkubeMeasurementSecret={{.Env.ANALYTICS_API_KEY}}
+      - -X github.com/kubeshop/testkube/pkg/diagnostics/validators/license.KeygenOfflinePublicKey={{.Env.KEYGEN_PUBLIC_KEY}}
 archives:
   - format: binary
 

diff --git a/.builds-windows.goreleaser.yml b/.builds-windows.goreleaser.yml
@@ -21,5 +21,6 @@ builds:
       - -X main.builtBy=goreleaser
       - -X github.com/kubeshop/testkube/pkg/telemetry.TestkubeMeasurementID={{.Env.ANALYTICS_TRACKING_ID}}
       - -X github.com/kubeshop/testkube/pkg/telemetry.TestkubeMeasurementSecret={{.Env.ANALYTICS_API_KEY}}
+      - -X github.com/kubeshop/testkube/pkg/diagnostics/validators/license.KeygenOfflinePublicKey={{.Env.KEYGEN_PUBLIC_KEY}}
 archives:
   - format: binary
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -52,7 +52,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/[email protected].0
+      uses: github/codeql-action/[email protected].5
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -82,4 +82,4 @@ jobs:
           test-${{ runner.os }}-go-
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/[email protected].0
+      uses: github/codeql-action/[email protected].5
diff --git a/.github/workflows/docker-build-api-executors-tag.yaml b/.github/workflows/docker-build-api-executors-tag.yaml
@@ -19,7 +19,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up Docker Buildx
         id: buildx
@@ -94,7 +94,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up Docker Buildx
         id: buildx
@@ -178,7 +178,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -249,7 +249,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -317,7 +317,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -442,7 +442,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -510,7 +510,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -669,7 +669,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -741,7 +741,7 @@ jobs:
         uses: docker/setup-qemu-action@v3
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up Docker Buildx
         id: buildx
@@ -790,7 +790,7 @@ jobs:
         uses: actions/checkout@v4
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -863,7 +863,7 @@ jobs:
           fetch-depth: 0
 
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -942,25 +942,10 @@ jobs:
         with:
           strip_v: false
 
-      - name: Editing helm-release repo with version based on a Tag pushed.
-        run: |
-
-          # Setting up Git:
-          git clone https://kubeshop-bot:[email protected]/kubeshop/helm-charts
-          cd ./helm-charts
-          git config user.name "kubeshop-bot"
-          git config user.email "[email protected]"
-
-          export GH_TOKEN
-          export RELEASE_VERSION
-          export SERVICE
-
-          git fetch origin main
-          git checkout main
-          cd ./scripts
-          ./chart_releaser.sh --helm-chart-folder testkube-$SERVICE
-
-        env:
-          SERVICE: api
-          GH_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
-          RELEASE_VERSION: ${{ steps.tag.outputs.tag }}
+      - name: Repository Dispatch
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.CI_BOT_TOKEN }}
+          repository: kubeshop/helm-charts
+          event-type: release-testkube-api-charts
+          client-payload: '{"RELEASE_VERSION": "${{ steps.tag.outputs.tag }}", "SERVICE": "api"}'
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
@@ -79,6 +79,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
           ANALYTICS_TRACKING_ID: "${{secrets.TESTKUBE_CLI_GA_MEASUREMENT_ID}}"
           ANALYTICS_API_KEY: "${{secrets.TESTKUBE_CLI_GA_MEASUREMENT_SECRET}}"
+          KEYGEN_PUBLIC_KEY: "${{secrets.KEYGEN_PUBLIC_KEY}}"
           # Your GoReleaser Pro key, if you are using the 'goreleaser-pro' distribution
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
           DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}"
@@ -185,7 +186,7 @@ jobs:
         with:
           fetch-depth: 0
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
       - name: Download Artifacts for Linux
         uses: actions/download-artifact@master
         with:

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -80,6 +80,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
           ANALYTICS_TRACKING_ID: "${{secrets.TESTKUBE_CLI_GA_MEASUREMENT_ID}}"
           ANALYTICS_API_KEY: "${{secrets.TESTKUBE_CLI_GA_MEASUREMENT_SECRET}}"
+          KEYGEN_PUBLIC_KEY: "${{secrets.KEYGEN_PUBLIC_KEY}}"
           # Your GoReleaser Pro key, if you are using the 'goreleaser-pro' distribution
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
           DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}"
@@ -177,7 +178,7 @@ jobs:
         with:
           fetch-depth: 0
       - uses: sigstore/[email protected]
-      - uses: anchore/sbom-action/[email protected].6
+      - uses: anchore/sbom-action/[email protected].7
       - name: Download Artifacts for Linux
         uses: actions/download-artifact@master
         with:

diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
@@ -67,6 +67,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@b7cdb7fd39e52d1018799033ccd11eeb108aed85 # v2.25.3
+        uses: github/codeql-action/upload-sarif@f8e782af5670be7b12c0acb5b25c1c5703b931b9 # v2.25.3
         with:
           sarif_file: results.sarif
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -53,7 +53,7 @@ jobs:
 
       - name: Send coverage report to Codecov
         if: always()
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           file: ./coverage.out
           flags: unittests
@@ -181,7 +181,7 @@ jobs:
 
       - name: Send coverage report to Codecov
         if: always()
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           file: ./coverage.out
           flags: integrationtests

diff --git a/cmd/api-server/commons/commons.go b/cmd/api-server/commons/commons.go
@@ -282,18 +282,19 @@ func ReadDefaultExecutors(cfg *config.Config) (executors []testkube.ExecutorDeta
 
 func ReadProContext(ctx context.Context, cfg *config.Config, grpcClient cloud.TestKubeCloudAPIClient) config.ProContext {
 	proContext := config.ProContext{
-		APIKey:                           cfg.TestkubeProAPIKey,
-		URL:                              cfg.TestkubeProURL,
-		TLSInsecure:                      cfg.TestkubeProTLSInsecure,
-		WorkerCount:                      cfg.TestkubeProWorkerCount,
-		LogStreamWorkerCount:             cfg.TestkubeProLogStreamWorkerCount,
-		WorkflowNotificationsWorkerCount: cfg.TestkubeProWorkflowNotificationsWorkerCount,
-		SkipVerify:                       cfg.TestkubeProSkipVerify,
-		EnvID:                            cfg.TestkubeProEnvID,
-		OrgID:                            cfg.TestkubeProOrgID,
-		Migrate:                          cfg.TestkubeProMigrate,
-		ConnectionTimeout:                cfg.TestkubeProConnectionTimeout,
-		DashboardURI:                     cfg.TestkubeDashboardURI,
+		APIKey:                                  cfg.TestkubeProAPIKey,
+		URL:                                     cfg.TestkubeProURL,
+		TLSInsecure:                             cfg.TestkubeProTLSInsecure,
+		WorkerCount:                             cfg.TestkubeProWorkerCount,
+		LogStreamWorkerCount:                    cfg.TestkubeProLogStreamWorkerCount,
+		WorkflowNotificationsWorkerCount:        cfg.TestkubeProWorkflowNotificationsWorkerCount,
+		WorkflowServiceNotificationsWorkerCount: cfg.TestkubeProWorkflowServiceNotificationsWorkerCount,
+		SkipVerify:                              cfg.TestkubeProSkipVerify,
+		EnvID:                                   cfg.TestkubeProEnvID,
+		OrgID:                                   cfg.TestkubeProOrgID,
+		Migrate:                                 cfg.TestkubeProMigrate,
+		ConnectionTimeout:                       cfg.TestkubeProConnectionTimeout,
+		DashboardURI:                            cfg.TestkubeDashboardURI,
 	}
 
 	if cfg.TestkubeProAPIKey == "" || grpcClient == nil {

diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go
@@ -15,6 +15,7 @@ import (
 	"github.com/kubeshop/testkube/cmd/api-server/commons"
 	"github.com/kubeshop/testkube/cmd/api-server/services"
 	"github.com/kubeshop/testkube/internal/app/api/debug"
+	agentclient "github.com/kubeshop/testkube/pkg/agent/client"
 	cloudartifacts "github.com/kubeshop/testkube/pkg/cloud/data/artifact"
 	cloudtestworkflow "github.com/kubeshop/testkube/pkg/cloud/data/testworkflow"
 	"github.com/kubeshop/testkube/pkg/event/kind/cdevent"
@@ -134,7 +135,7 @@ func main() {
 	if strings.HasPrefix(controlPlaneUrl, fmt.Sprintf("%s:%d", cfg.APIServerFullname, cfg.GRPCServerPort)) {
 		controlPlaneUrl = fmt.Sprintf("127.0.0.1:%d", cfg.GRPCServerPort)
 	}
-	grpcConn, err = agent.NewGRPCConnection(
+	grpcConn, err = agentclient.NewGRPCConnection(
 		ctx,
 		cfg.TestkubeProTLSInsecure,
 		cfg.TestkubeProSkipVerify,
@@ -324,6 +325,22 @@ func main() {
 		}
 		return notifications.Channel(), nil
 	}
+	getTestWorkflowServiceNotificationsStream := func(ctx context.Context, executionID, serviceName string, serviceIndex int) (<-chan testkube.TestWorkflowExecutionNotification, error) {
+		execution, err := testWorkflowResultsRepository.Get(ctx, executionID)
+		if err != nil {
+			return nil, err
+		}
+		notifications := executionWorker.Notifications(ctx, fmt.Sprintf("%s-%s-%d", execution.Id, serviceName, serviceIndex), executionworkertypes.NotificationsOptions{
+			Hints: executionworkertypes.Hints{
+				Namespace:   execution.Namespace,
+				ScheduledAt: common.Ptr(execution.ScheduledAt),
+			},
+		})
+		if notifications.Err() != nil {
+			return nil, notifications.Err()
+		}
+		return notifications.Channel(), nil
+	}
 	getDeprecatedLogStream := func(ctx context.Context, executionID string) (chan output.Output, error) {
 		return nil, errors.New("deprecated features have been disabled")
 	}
@@ -336,6 +353,7 @@ func main() {
 		grpcClient,
 		getDeprecatedLogStream,
 		getTestWorkflowNotificationsStream,
+		getTestWorkflowServiceNotificationsStream,
 		clusterId,
 		cfg.TestkubeClusterName,
 		features,

diff --git a/cmd/kubectl-testkube/commands/common/cloudcontext.go b/cmd/kubectl-testkube/commands/common/cloudcontext.go
@@ -26,7 +26,7 @@ func UiPrintContext(cfg config.Data) {
 		}
 
 		// add agent information only when need to change agent data, it's usually not needed in usual workflow
-		if ui.Verbose {
+		if ui.IsVerbose() {
 			contextData["Agent Key"] = text.Obfuscate(cfg.CloudContext.AgentKey)
 			contextData["Agent URI"] = cfg.CloudContext.AgentUri
 		}

diff --git a/cmd/kubectl-testkube/commands/common/errors.go b/cmd/kubectl-testkube/commands/common/errors.go
@@ -3,6 +3,7 @@ package common
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -52,13 +53,14 @@ const (
 const helpUrl = "https://testkubeworkspace.slack.com"
 
 type CLIError struct {
-	Code        ErrorCode
-	Title       string
-	Description string
-	ActualError error
-	StackTrace  string
-	MoreInfo    string
-	Telemetry   *ErrorTelemetry
+	Code            ErrorCode
+	Title           string
+	Description     string
+	ActualError     error
+	StackTrace      string
+	MoreInfo        string
+	ExecutedCommand string
+	Telemetry       *ErrorTelemetry
 }
 
 type ErrorTelemetry struct {
@@ -86,6 +88,15 @@ func (e *CLIError) Print() {
 
 	pterm.DefaultSection.Println("Error Details")
 
+	cmd := ""
+	if e.ExecutedCommand != "" {
+		pterm.FgDarkGray.Printfln("Executed command: %s", e.ExecutedCommand)
+		params := strings.Split(e.ExecutedCommand, " ")
+		if len(params) > 0 {
+			cmd = params[0]
+		}
+	}
+
 	items := []pterm.BulletListItem{
 		{Level: 0, Text: pterm.Sprintf("[%s]: %s", e.Code, e.Title), TextStyle: pterm.NewStyle(pterm.FgRed)},
 		{Level: 0, Text: pterm.Sprintf("%s", e.Description), TextStyle: pterm.NewStyle(pterm.FgLightWhite)},
@@ -94,6 +105,9 @@ func (e *CLIError) Print() {
 		items = append(items, pterm.BulletListItem{Level: 0, Text: pterm.Sprintf("%s", e.MoreInfo), TextStyle: pterm.NewStyle(pterm.FgGray)})
 	}
 	pterm.DefaultBulletList.WithItems(items).Render()
+	if cmd != "" {
+		pterm.DefaultBox.Printfln("Error description is provided in context of binary execution %s", cmd)
+	}
 
 	pterm.Println()
 	pterm.Println("Let us help you!")
@@ -111,6 +125,11 @@ func NewCLIError(code ErrorCode, title, moreInfoURL string, err error) *CLIError
 	}
 }
 
+func (err *CLIError) WithExecutedCommand(executedCommand string) *CLIError {
+	err.ExecutedCommand = executedCommand
+	return err
+}
+
 // HandleCLIError checks does the error exist, and if it does, prints the error and exits the program.
 func HandleCLIError(err *CLIError) {
 	if err != nil {