diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 03d85f18019..fda16075ca1 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -23,11 +23,21 @@ import ( ) const ( - volumeNameKFPLauncher = "kfp-launcher" - DefaultLauncherImage = "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4" - LauncherImageEnvVar = "V2_LAUNCHER_IMAGE" - DefaultDriverImage = "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707" - DriverImageEnvVar = "V2_DRIVER_IMAGE" + volumeNameKFPLauncher = "kfp-launcher" + DefaultLauncherImage = "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4" + LauncherImageEnvVar = "V2_LAUNCHER_IMAGE" + DefaultDriverImage = "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707" + DriverImageEnvVar = "V2_DRIVER_IMAGE" + gcsScratchLocation = "/gcs" + gcsScratchName = "gcs-scratch" + s3ScratchLocation = "/s3" + s3ScratchName = "s3-scratch" + minioScratchLocation = "/minio" + minioScratchName = "minio-scratch" + dotLocalScratchLocation = "/.local" + dotLocalScratchName = "dot-local-scratch" + dotCacheScratchLocation = "/.cache" + dotCacheScratchName = "dot-cache-scratch" ) func (c *workflowCompiler) Container(name string, component *pipelinespec.ComponentSpec, container *pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec) error { @@ -238,21 +248,55 @@ func (c *workflowCompiler) addContainerExecutorTemplate() string { // args come from. It is treated as a strategic merge patch on // top of the Pod spec. PodSpecPatch: inputValue(paramPodSpecPatch), - Volumes: []k8score.Volume{{ - Name: volumeNameKFPLauncher, - VolumeSource: k8score.VolumeSource{ - EmptyDir: &k8score.EmptyDirVolumeSource{}, + Volumes: []k8score.Volume{ + { + Name: volumeNameKFPLauncher, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, }, - }}, + { + Name: gcsScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: s3ScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: minioScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: dotLocalScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: dotCacheScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + }, InitContainers: []wfapi.UserContainer{{ Container: k8score.Container{ Name: "kfp-launcher", Image: GetLauncherImage(), Command: []string{"launcher-v2", "--copy", component.KFPLauncherPath}, - VolumeMounts: []k8score.VolumeMount{{ - Name: volumeNameKFPLauncher, - MountPath: component.VolumePathKFPLauncher, - }}, + VolumeMounts: []k8score.VolumeMount{ + { + Name: volumeNameKFPLauncher, + MountPath: component.VolumePathKFPLauncher, + }, + }, Resources: launcherResources, }, }}, @@ -265,10 +309,32 @@ func (c *workflowCompiler) addContainerExecutorTemplate() string { // These are added to pass argo workflows linting. Image: "gcr.io/ml-pipeline/should-be-overridden-during-runtime", Command: []string{"should-be-overridden-during-runtime"}, - VolumeMounts: []k8score.VolumeMount{{ - Name: volumeNameKFPLauncher, - MountPath: component.VolumePathKFPLauncher, - }}, + VolumeMounts: []k8score.VolumeMount{ + { + Name: volumeNameKFPLauncher, + MountPath: component.VolumePathKFPLauncher, + }, + { + Name: gcsScratchName, + MountPath: gcsScratchLocation, + }, + { + Name: s3ScratchName, + MountPath: s3ScratchLocation, + }, + { + Name: minioScratchName, + MountPath: minioScratchLocation, + }, + { + Name: dotLocalScratchName, + MountPath: dotLocalScratchLocation, + }, + { + Name: dotCacheScratchName, + MountPath: dotCacheScratchLocation, + }, + }, EnvFrom: []k8score.EnvFromSource{metadataEnvFrom}, Env: commonEnvs, },