WIP: UPSTREAM: <carry>: feat(backend): Upgrade argo to v3.4.16

.cloudbuild.yaml

@@ -170,10 +170,10 @@ steps:
   args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.25.0']
   id:   'pullCloudsqlProxy'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.3.10-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance']
   id:   'pullArgoExecutor'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.3.10-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance']
   id:   'pullArgoWorkflowController'
 
 # V2 related images

.release.cloudbuild.yaml

@@ -478,14 +478,14 @@ steps:
     docker push gcr.io/ml-pipeline/google/pipelines-test/cloudsqlproxy:$(cat /workspace/mm.ver)
 
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.3.10-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance']
   id:   'pullArgoExecutor'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.3.10-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME']
   id:   'tagArgoExecutorForMarketplace'
   waitFor: ['pullArgoExecutor']
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.3.10-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME']
   id:   'tagArgoExecutorForMarketplaceTest'
   waitFor: ['pullArgoExecutor']
 - id:   'tagArgoExecutorForMarketplaceMajorMinor'
@@ -495,20 +495,20 @@ steps:
   args:
   - -ceux
   - |
-    docker tag gcr.io/ml-pipeline/argoexec:v3.3.10-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
-    docker tag gcr.io/ml-pipeline/argoexec:v3.3.10-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
 
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.3.10-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance']
   id:   'pullArgoWorkflowController'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.3.10-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME']
   id:   'tagArgoWorkflowControllerForMarketplace'
   waitFor: ['pullArgoWorkflowController']
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.3.10-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME']
   id:   'tagArgoWorkflowControllerForMarketplaceTest'
   waitFor: ['pullArgoWorkflowController']
 - id:   'tagArgoWorkflowControllerForMarketplaceMajorMinor'
@@ -518,8 +518,8 @@ steps:
   args:
   - -ceux
   - |
-    docker tag gcr.io/ml-pipeline/workflow-controller:v3.3.10-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
-    docker tag gcr.io/ml-pipeline/workflow-controller:v3.3.10-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
 

backend/Dockerfile

@@ -20,6 +20,11 @@ FROM registry.access.redhat.com/ubi8/go-toolset:1.20 as builder
 
 ## Build args to be used at this step
 ARG SOURCE_CODE
+ENV ARGO_VERSION v3.4.16
+RUN curl -sLO https://github.com/argoproj/argo-workflows/releases/download/${ARGO_VERSION}/argo-linux-amd64.gz && \
+    gunzip argo-linux-amd64.gz && \
+    chmod +x argo-linux-amd64 && \
+    mv ./argo-linux-amd64 /usr/local/bin/argo
 
 USER root
 

backend/src/agent/persistence/worker/metrics_reporter_test.go

@@ -18,10 +18,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"testing"
+
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/protobuf/testing/protocmp"
-	"testing"
 
 	workflowapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	api "github.com/kubeflow/pipelines/backend/api/v1beta1/go_client"
@@ -46,8 +47,9 @@ func TestReportMetrics_NoCompletedNode_NoOP(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeRunning,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeRunning,
 				},
 			},
 		},
@@ -98,8 +100,9 @@ func TestReportMetrics_NoArtifact_NoOP(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 				},
 			},
 		},
@@ -125,8 +128,9 @@ func TestReportMetrics_NoMetricsArtifact_NoOP(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-ui-metadata"}},
 					},
@@ -153,8 +157,9 @@ func TestReportMetrics_Succeed(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-metrics"}},
 					},
@@ -167,7 +172,7 @@ func TestReportMetrics_Succeed(t *testing.T) {
 	pipelineFake.StubArtifact(
 		&api.ReadArtifactRequest{
 			RunId:        "run-1",
-			NodeId:       "node-1",
+			NodeId:       "MY_NAME-template-1-1",
 			ArtifactName: "mlpipeline-metrics",
 		},
 		&api.ReadArtifactResponse{
@@ -185,12 +190,12 @@ func TestReportMetrics_Succeed(t *testing.T) {
 		Metrics: []*api.RunMetric{
 			{
 				Name:   "accuracy",
-				NodeId: "node-1",
+				NodeId: "MY_NAME-template-1-1",
 				Value:  &api.RunMetric_NumberValue{NumberValue: 0.77},
 			},
 			{
 				Name:   "logloss",
-				NodeId: "node-1",
+				NodeId: "MY_NAME-template-1-1",
 				Value:  &api.RunMetric_NumberValue{NumberValue: 1.2},
 			},
 		},
@@ -216,8 +221,9 @@ func TestReportMetrics_EmptyArchive_Fail(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-metrics"}},
 					},
@@ -229,7 +235,7 @@ func TestReportMetrics_EmptyArchive_Fail(t *testing.T) {
 	pipelineFake.StubArtifact(
 		&api.ReadArtifactRequest{
 			RunId:        "run-1",
-			NodeId:       "node-1",
+			NodeId:       "MY_NAME-template-1-1",
 			ArtifactName: "mlpipeline-metrics",
 		},
 		&api.ReadArtifactResponse{
@@ -257,8 +263,9 @@ func TestReportMetrics_MultipleFilesInArchive_Fail(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "MY_NAME-template-1-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-metrics"}},
 					},
@@ -272,7 +279,7 @@ func TestReportMetrics_MultipleFilesInArchive_Fail(t *testing.T) {
 	pipelineFake.StubArtifact(
 		&api.ReadArtifactRequest{
 			RunId:        "run-1",
-			NodeId:       "node-1",
+			NodeId:       "MY_NAME-template-1-1",
 			ArtifactName: "mlpipeline-metrics",
 		},
 		&api.ReadArtifactResponse{
@@ -300,8 +307,9 @@ func TestReportMetrics_InvalidMetricsJSON_Fail(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-metrics"}},
 					},
@@ -314,7 +322,7 @@ func TestReportMetrics_InvalidMetricsJSON_Fail(t *testing.T) {
 	pipelineFake.StubArtifact(
 		&api.ReadArtifactRequest{
 			RunId:        "run-1",
-			NodeId:       "node-1",
+			NodeId:       "MY_NAME-template-1-1",
 			ArtifactName: "mlpipeline-metrics",
 		},
 		&api.ReadArtifactResponse{
@@ -342,15 +350,17 @@ func TestReportMetrics_InvalidMetricsJSON_PartialFail(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-metrics"}},
 					},
 				},
 				"node-2": workflowapi.NodeStatus{
-					ID:    "node-2",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-2",
+					TemplateName: "template-2",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-metrics"}},
 					},
@@ -366,7 +376,7 @@ func TestReportMetrics_InvalidMetricsJSON_PartialFail(t *testing.T) {
 	pipelineFake.StubArtifact(
 		&api.ReadArtifactRequest{
 			RunId:        "run-1",
-			NodeId:       "node-1",
+			NodeId:       "MY_NAME-template-1-1",
 			ArtifactName: "mlpipeline-metrics",
 		},
 		&api.ReadArtifactResponse{
@@ -375,7 +385,7 @@ func TestReportMetrics_InvalidMetricsJSON_PartialFail(t *testing.T) {
 	pipelineFake.StubArtifact(
 		&api.ReadArtifactRequest{
 			RunId:        "run-1",
-			NodeId:       "node-2",
+			NodeId:       "MY_NAME-template-2-2",
 			ArtifactName: "mlpipeline-metrics",
 		},
 		&api.ReadArtifactResponse{
@@ -392,12 +402,12 @@ func TestReportMetrics_InvalidMetricsJSON_PartialFail(t *testing.T) {
 		Metrics: []*api.RunMetric{
 			&api.RunMetric{
 				Name:   "accuracy",
-				NodeId: "node-2",
+				NodeId: "MY_NAME-template-2-2",
 				Value:  &api.RunMetric_NumberValue{NumberValue: 0.77},
 			},
 			&api.RunMetric{
 				Name:   "logloss",
-				NodeId: "node-2",
+				NodeId: "MY_NAME-template-2-2",
 				Value:  &api.RunMetric_NumberValue{NumberValue: 1.2},
 			},
 		},
@@ -423,8 +433,9 @@ func TestReportMetrics_CorruptedArchiveFile_Fail(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-metrics"}},
 					},
@@ -435,7 +446,7 @@ func TestReportMetrics_CorruptedArchiveFile_Fail(t *testing.T) {
 	pipelineFake.StubArtifact(
 		&api.ReadArtifactRequest{
 			RunId:        "run-1",
-			NodeId:       "node-1",
+			NodeId:       "MY_NAME-template-1-1",
 			ArtifactName: "mlpipeline-metrics",
 		},
 		&api.ReadArtifactResponse{
@@ -463,8 +474,9 @@ func TestReportMetrics_MultiplMetricErrors_TransientErrowWin(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-metrics"}},
 					},
@@ -478,7 +490,7 @@ func TestReportMetrics_MultiplMetricErrors_TransientErrowWin(t *testing.T) {
 	pipelineFake.StubArtifact(
 		&api.ReadArtifactRequest{
 			RunId:        "run-1",
-			NodeId:       "node-1",
+			NodeId:       "MY_NAME-template-1-1",
 			ArtifactName: "mlpipeline-metrics",
 		},
 		&api.ReadArtifactResponse{
@@ -526,8 +538,9 @@ func TestReportMetrics_Unauthorized(t *testing.T) {
 		Status: workflowapi.WorkflowStatus{
 			Nodes: map[string]workflowapi.NodeStatus{
 				"node-1": workflowapi.NodeStatus{
-					ID:    "node-1",
-					Phase: workflowapi.NodeSucceeded,
+					ID:           "node-1",
+					TemplateName: "template-1",
+					Phase:        workflowapi.NodeSucceeded,
 					Outputs: &workflowapi.Outputs{
 						Artifacts: []workflowapi.Artifact{{Name: "mlpipeline-metrics"}},
 					},
@@ -540,7 +553,7 @@ func TestReportMetrics_Unauthorized(t *testing.T) {
 	pipelineFake.StubArtifact(
 		&api.ReadArtifactRequest{
 			RunId:        "run-1",
-			NodeId:       "node-1",
+			NodeId:       "MY_NAME-template-1-1",
 			ArtifactName: "mlpipeline-metrics",
 		},
 		&api.ReadArtifactResponse{

backend/src/apiserver/resource/resource_manager_util_test.go

@@ -136,7 +136,7 @@ status:
 
 	newWfString, err := yaml.Marshal(newWf)
 	assert.Nil(t, err)
-	assert.Equal(t, []string{"resubmit-hl9ft-3879090716"}, nodes)
+	assert.Equal(t, []string{"resubmit-hl9ft-random-fail-3879090716"}, nodes)
 
 	expectedNewWfString := `apiVersion: argoproj.io/v1alpha1
 kind: Workflow
@@ -202,7 +202,7 @@ status:
       startedAt: "2021-05-26T09:14:07Z"
       templateName: rand-fail-dag
       type: DAG
-    resubmit-hl9ft-3929423573:
+    resubmit-hl9ft-random-fail-3929423573:
       boundaryID: resubmit-hl9ft
       children:
       - resubmit-hl9ft-3879090716

backend/src/apiserver/server/api_converter_test.go

@@ -3112,7 +3112,7 @@ func Test_toModelTasks_v2(t *testing.T) {
 func Test_toModelTasks_wf(t *testing.T) {
 	expectedWf := []*model.Task{
 		{
-			PodName:           "boudary_exec_id-node0",
+			PodName:           "run1-file-passing-pipelines-node0",
 			Namespace:         "kubeflow",
 			RunId:             "run1_uid_true",
 			CreatedTimestamp:  -62135596800,
@@ -3123,7 +3123,7 @@ func Test_toModelTasks_wf(t *testing.T) {
 			ChildrenPods:      []string{"boudary_exec_id-node1"},
 		},
 		{
-			PodName:           "boudary_exec_id-node1",
+			PodName:           "run1-print-text-node1",
 			Namespace:         "kubeflow",
 			RunId:             "run1_uid_true",
 			CreatedTimestamp:  -62135596800,

backend/src/apiserver/template/argo_template.go

@@ -227,7 +227,7 @@ func ValidateWorkflow(template []byte) (*util.Workflow, error) {
 	if wf.Kind != argoK8sResource {
 		return nil, util.NewInvalidInputError("Unexpected resource type. Expected: %v. Received: %v", argoK8sResource, wf.Kind)
 	}
-	_, err = validate.ValidateWorkflow(nil, nil, &wf, validate.ValidateOpts{
+	err = validate.ValidateWorkflow(nil, nil, &wf, validate.ValidateOpts{
 		Lint:                       true,
 		IgnoreEntrypoint:           true,
 		WorkflowTemplateValidation: false, // not used by kubeflow

backend/src/common/types.go

@@ -21,7 +21,7 @@ package common
 type ExecutionPhase string
 
 // borrow from Workflow.Status.Phase:
-// https://pkg.go.dev/github.com/argoproj/argo-workflows/v3@v3.3.10/pkg/apis/workflow/v1alpha1#WorkflowPhase
+// https://pkg.go.dev/github.com/argoproj/argo-workflows/v3@v3.4.16/pkg/apis/workflow/v1alpha1#WorkflowPhase
 const (
 	ExecutionUnknown   ExecutionPhase = ""
 	ExecutionPending   ExecutionPhase = "Pending" // pending some set-up - rarely used