Merge branch 'main' into dawid/feat/move-scheduling

api/v1/testkube.yaml

@@ -8690,6 +8690,10 @@ components:
             type: string
           readinessProbe:
             $ref: "#/components/schemas/Probe"
+          pvcs:
+            type: object
+            additionalProperties:
+              $ref: "#/components/schemas/TestWorkflowPvcConfig"
 
     TestWorkflowServiceSpec:
       type: object
@@ -8809,6 +8813,10 @@ components:
             $ref: "#/components/schemas/TestWorkflowEvent"
         execution:
           $ref: "#/components/schemas/TestWorkflowTagSchema"
+        pvcs:
+          type: object
+          additionalProperties:
+            $ref: "#/components/schemas/TestWorkflowPvcConfig"
 
     TestWorkflowTemplateSpec:
       type: object
@@ -8847,6 +8855,10 @@ components:
             $ref: "#/components/schemas/TestWorkflowEvent"
         execution:
           $ref: "#/components/schemas/TestWorkflowTagSchema"
+        pvcs:
+          type: object
+          additionalProperties:
+            $ref: "#/components/schemas/TestWorkflowPvcConfig"
 
     TestWorkflowStepControl:
       type: object
@@ -9305,6 +9317,9 @@ components:
         mountPath:
           type: string
           description: where to mount the fetched repository contents (defaults to "repo" directory in the data volume)
+        cone:
+          type: boolean
+          description: enable cone mode for sparse checkout with paths
         paths:
           type: array
           description: paths to fetch for the sparse checkout
@@ -10621,6 +10636,67 @@ components:
                 must be defined
               type: boolean
 
+    TestWorkflowPvcConfig:
+      type: object
+      properties:
+        accessModes:
+          description: 'Access mode for claim storage. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes'
+          type: array
+          items:
+            type: string
+        volumeMode:
+          description: 'Volume mode indicates the consumption of the volume as either a filesystem or block device.
+            More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#volume-mode'
+          $ref: "#/components/schemas/BoxedString"
+        resources:
+          description: 'Resources required for pvc'
+          $ref: "#/components/schemas/TestWorkflowResources"
+        storageClassName:
+          description: 'Storage class name specifies the name of a StorageClass. More info: https://kubernetes.io/docs/concepts/storage/storage-classes/'
+          $ref: "#/components/schemas/BoxedString"
+        volumeName:
+          description: 'Volume name is used to identify the volume'
+          type: string
+        selector:
+          description: Only the volumes whose labels match the selector can be bound to the claim
+          $ref: "#/components/schemas/LabelSelector"
+        dataSource:
+          description: 'Data source field can be used to specify either:
+            * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot)
+            * An existing PVC (PersistentVolumeClaim)'
+          $ref: "#/components/schemas/TypedLocalObjectReference"
+        dataSourceRef:
+          description: 'Data source reference specifies the object from which to populate the volume with data, if a non-empty volume is desired'
+          $ref: "#/components/schemas/TypedObjectReference"
+        volumeAttributesClassName:
+          description: 'Volume attributes class name may be used to set the VolumeAttributesClass used by this claim.
+            More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass'
+          $ref: "#/components/schemas/BoxedString"
+
+    TypedLocalObjectReference:
+      description: TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace
+      type: object
+      properties:
+        apiGroup:
+          description: api group is the group for the resource being referenced
+          $ref: "#/components/schemas/BoxedString"
+        kind:
+          description: kind is the type of resource being referenced
+          type: string
+        name:
+          description: name is the name of resource being referenced
+          type: string
+
+    TypedObjectReference:
+      description: TypedObjectReference contains enough information to let you locate the typed referenced object inside the specified namespace
+      type: object
+      allOf:
+      - $ref: "#/components/schemas/TypedLocalObjectReference"     
+      properties:
+        namespace:
+          description: Namespace is the namespace of resource being referenced
+          $ref: "#/components/schemas/BoxedString"
+
     #
     # Errors
     #

cmd/tcl/testworkflow-toolkit/commands/parallel.go

@@ -205,6 +205,7 @@ func NewParallelCmd() *cobra.Command {
 					testworkflowconfig.CreateResourceMachine(&cfg.Resource),
 					testworkflowconfig.CreateWorkerMachine(&cfg.Worker),
 					baseMachine,
+					testworkflowconfig.CreatePvcMachine(cfg.Execution.PvcNames),
 					params.MachineAt(index),
 				)
 

cmd/tcl/testworkflow-toolkit/commands/services.go

@@ -156,6 +156,7 @@ func NewServicesCmd() *cobra.Command {
 						Steps: []testworkflowsv1.Step{
 							{StepOperations: testworkflowsv1.StepOperations{Run: common.Ptr(svcSpec.StepRun)}},
 						},
+						Pvcs: svcSpec.Pvcs,
 					}
 					spec.Steps[0].Run.ContainerConfig = testworkflowsv1.ContainerConfig{}
 
@@ -260,6 +261,7 @@ func NewServicesCmd() *cobra.Command {
 					testworkflowconfig.CreateResourceMachine(&cfg.Resource),
 					testworkflowconfig.CreateWorkerMachine(&cfg.Worker),
 					baseMachine,
+					testworkflowconfig.CreatePvcMachine(cfg.Execution.PvcNames),
 					params.MachineAt(index),
 				)
 

cmd/testworkflow-toolkit/commands/clone.go

@@ -35,6 +35,7 @@ func NewCloneCmd() *cobra.Command {
 		sshKey   string
 		authType string
 		revision string
+		cone     bool
 	)
 
 	cmd := &cobra.Command{
@@ -59,6 +60,10 @@ func NewCloneCmd() *cobra.Command {
 			paths := make([]string, 0)
 			for _, p := range rawPaths {
 				p = filepath.Clean(p)
+				if cone && p != "/" && strings.HasPrefix(p, "/") {
+					// Delete leading '/' for cone
+					p = p[1:]
+				}
 				if p != "" && p != "." {
 					paths = append(paths, p)
 				}
@@ -114,7 +119,11 @@ func NewCloneCmd() *cobra.Command {
 				ui.Debug("sparse checkout")
 				err = RunWithRetry(CloneRetryOnFailureMaxAttempts, CloneRetryOnFailureBaseDelay, "git", "clone", configArgs, authArgs, "--filter=blob:none", "--no-checkout", "--sparse", "--depth", 1, "--verbose", uri.String(), outputPath)
 				ui.ExitOnError("cloning repository", err)
-				err = RunWithRetry(CloneRetryOnFailureMaxAttempts, CloneRetryOnFailureBaseDelay, "git", "-C", outputPath, configArgs, "sparse-checkout", "set", "--no-cone", paths)
+				coneArgs := []string{"--no-cone"}
+				if cone {
+					coneArgs = nil
+				}
+				err = RunWithRetry(CloneRetryOnFailureMaxAttempts, CloneRetryOnFailureBaseDelay, "git", "-C", outputPath, configArgs, "sparse-checkout", "set", coneArgs, paths)
 				ui.ExitOnError("sparse checkout repository", err)
 				if revision != "" {
 					err = RunWithRetry(CloneRetryOnFailureMaxAttempts, CloneRetryOnFailureBaseDelay, "git", "-C", outputPath, configArgs, "fetch", authArgs, "--depth", 1, "origin", revision)
@@ -168,6 +177,7 @@ func NewCloneCmd() *cobra.Command {
 	cmd.Flags().StringVarP(&sshKey, "sshKey", "s", "", "")
 	cmd.Flags().StringVarP(&authType, "authType", "a", "basic", "allowed: basic, header")
 	cmd.Flags().StringVarP(&revision, "revision", "r", "", "commit hash, branch name or tag")
+	cmd.Flags().BoolVar(&cone, "cone", false, "should enable cone mode for sparse checkout")
 
 	return cmd
 }

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/keygen-sh/jsonapi-go v1.2.1
 	github.com/keygen-sh/keygen-go/v3 v3.2.0
 	github.com/kubepug/kubepug v1.7.1
-	github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a
+	github.com/kubeshop/testkube-operator v1.17.55-0.20241219143413-6a33aef7130f
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/montanaflynn/stats v0.7.1
 	github.com/moogar0880/problems v0.1.1

go.sum

@@ -336,8 +336,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubepug/kubepug v1.7.1 h1:LKhfSxS8Y5mXs50v+3Lpyec+cogErDLcV7CMUuiaisw=
 github.com/kubepug/kubepug v1.7.1/go.mod h1:lv+HxD0oTFL7ZWjj0u6HKhMbbTIId3eG7aWIW0gyF8g=
-github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a h1:xget2cwwqOL+K2Op9FPbMgfzj9lSVJAzZ9p48yxuFrE=
-github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
+github.com/kubeshop/testkube-operator v1.17.55-0.20241219143413-6a33aef7130f h1:7wDuGy4MohoFcgz2QPH/02JWCsgBqkWdGgefIH2YSEE=
+github.com/kubeshop/testkube-operator v1.17.55-0.20241219143413-6a33aef7130f/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=

pkg/api/v1/testkube/model_test_workflow_content_git.go

@@ -26,6 +26,8 @@ type TestWorkflowContentGit struct {
 	AuthType   *ContentGitAuthType `json:"authType,omitempty"`
 	// where to mount the fetched repository contents (defaults to \"repo\" directory in the data volume)
 	MountPath string `json:"mountPath,omitempty"`
+	// enable cone mode for sparse checkout with paths
+	Cone bool `json:"cone,omitempty"`
 	// paths to fetch for the sparse checkout
 	Paths []string `json:"paths,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go

@@ -36,6 +36,7 @@ type TestWorkflowIndependentServiceSpec struct {
 	Logs           *BoxedString                       `json:"logs,omitempty"`
 	RestartPolicy  string                             `json:"restartPolicy,omitempty"`
 	ReadinessProbe *Probe                             `json:"readinessProbe,omitempty"`
+	Pvcs           map[string]TestWorkflowPvcConfig   `json:"pvcs,omitempty"`
 	Count          *BoxedString                       `json:"count,omitempty"`
 	MaxCount       *BoxedString                       `json:"maxCount,omitempty"`
 	// matrix of parameters to spawn instances

pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go

@@ -53,4 +53,5 @@ type TestWorkflowIndependentStepParallel struct {
 	After     []TestWorkflowIndependentStep                 `json:"after,omitempty"`
 	Events    []TestWorkflowEvent                           `json:"events,omitempty"`
 	Execution *TestWorkflowTagSchema                        `json:"execution,omitempty"`
+	Pvcs      map[string]TestWorkflowPvcConfig              `json:"pvcs,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_pvc_config.go

@@ -0,0 +1,24 @@
+/*
+ * Testkube API
+ *
+ * Testkube provides a Kubernetes-native framework for test definition, execution and results
+ *
+ * API version: 1.0.0
+ * Contact: [email protected]
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+package testkube
+
+type TestWorkflowPvcConfig struct {
+	// Access mode for claim storage. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes
+	AccessModes      []string               `json:"accessModes,omitempty"`
+	VolumeMode       *BoxedString           `json:"volumeMode,omitempty"`
+	Resources        *TestWorkflowResources `json:"resources,omitempty"`
+	StorageClassName *BoxedString           `json:"storageClassName,omitempty"`
+	// Volume name is used to identify the volume
+	VolumeName                string                     `json:"volumeName,omitempty"`
+	Selector                  *LabelSelector             `json:"selector,omitempty"`
+	DataSource                *TypedLocalObjectReference `json:"dataSource,omitempty"`
+	DataSourceRef             *TypedObjectReference      `json:"dataSourceRef,omitempty"`
+	VolumeAttributesClassName *BoxedString               `json:"volumeAttributesClassName,omitempty"`
+}

pkg/api/v1/testkube/model_test_workflow_service_spec.go

@@ -36,6 +36,7 @@ type TestWorkflowServiceSpec struct {
 	Logs           *BoxedString                       `json:"logs,omitempty"`
 	RestartPolicy  string                             `json:"restartPolicy,omitempty"`
 	ReadinessProbe *Probe                             `json:"readinessProbe,omitempty"`
+	Pvcs           map[string]TestWorkflowPvcConfig   `json:"pvcs,omitempty"`
 	Use            []TestWorkflowTemplateRef          `json:"use,omitempty"`
 	Count          *BoxedString                       `json:"count,omitempty"`
 	MaxCount       *BoxedString                       `json:"maxCount,omitempty"`

pkg/api/v1/testkube/model_test_workflow_spec.go

@@ -23,4 +23,5 @@ type TestWorkflowSpec struct {
 	After     []TestWorkflowStep                     `json:"after,omitempty"`
 	Events    []TestWorkflowEvent                    `json:"events,omitempty"`
 	Execution *TestWorkflowTagSchema                 `json:"execution,omitempty"`
+	Pvcs      map[string]TestWorkflowPvcConfig       `json:"pvcs,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_step_parallel.go

@@ -55,4 +55,5 @@ type TestWorkflowStepParallel struct {
 	After     []TestWorkflowStep                     `json:"after,omitempty"`
 	Events    []TestWorkflowEvent                    `json:"events,omitempty"`
 	Execution *TestWorkflowTagSchema                 `json:"execution,omitempty"`
+	Pvcs      map[string]TestWorkflowPvcConfig       `json:"pvcs,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_template_spec.go

@@ -22,4 +22,5 @@ type TestWorkflowTemplateSpec struct {
 	After     []TestWorkflowIndependentStep                 `json:"after,omitempty"`
 	Events    []TestWorkflowEvent                           `json:"events,omitempty"`
 	Execution *TestWorkflowTagSchema                        `json:"execution,omitempty"`
+	Pvcs      map[string]TestWorkflowPvcConfig              `json:"pvcs,omitempty"`
 }

pkg/api/v1/testkube/model_typed_local_object_reference.go

@@ -0,0 +1,19 @@
+/*
+ * Testkube API
+ *
+ * Testkube provides a Kubernetes-native framework for test definition, execution and results
+ *
+ * API version: 1.0.0
+ * Contact: [email protected]
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+package testkube
+
+// TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace
+type TypedLocalObjectReference struct {
+	ApiGroup *BoxedString `json:"apiGroup,omitempty"`
+	// kind is the type of resource being referenced
+	Kind string `json:"kind,omitempty"`
+	// name is the name of resource being referenced
+	Name string `json:"name,omitempty"`
+}

pkg/api/v1/testkube/model_typed_object_reference.go

@@ -0,0 +1,20 @@
+/*
+ * Testkube API
+ *
+ * Testkube provides a Kubernetes-native framework for test definition, execution and results
+ *
+ * API version: 1.0.0
+ * Contact: [email protected]
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+package testkube
+
+// TypedObjectReference contains enough information to let you locate the typed referenced object inside the specified namespace
+type TypedObjectReference struct {
+	ApiGroup *BoxedString `json:"apiGroup,omitempty"`
+	// kind is the type of resource being referenced
+	Kind string `json:"kind,omitempty"`
+	// name is the name of resource being referenced
+	Name      string       `json:"name,omitempty"`
+	Namespace *BoxedString `json:"namespace,omitempty"`
+}