diff --git a/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml index 31b905d295f..579a5738cdb 100644 --- a/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml +++ b/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml @@ -22,4 +22,4 @@ jobs: resources: instance_type: standard_e8s_v3 - runtime_version: "3.3" + runtime_version: "3.4" diff --git a/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml index 6822ea9f9fa..945263cca43 100644 --- a/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml +++ b/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml @@ -25,4 +25,4 @@ jobs: resources: instance_type: standard_e8s_v3 - runtime_version: "3.3" + runtime_version: "3.4" diff --git a/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml index b5ec9d05ae6..ef8507a10a7 100644 --- a/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml +++ b/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml @@ -25,4 +25,4 @@ jobs: resources: instance_type: standard_e8s_v3 - runtime_version: "3.3" + runtime_version: "3.4" diff --git a/cli/jobs/spark/serverless-spark-standalone-default-identity.yml b/cli/jobs/spark/serverless-spark-standalone-default-identity.yml index 49a21c4ed4d..31d39196c48 100644 --- a/cli/jobs/spark/serverless-spark-standalone-default-identity.yml +++ b/cli/jobs/spark/serverless-spark-standalone-default-identity.yml @@ -31,5 +31,5 @@ args: >- resources: instance_type: standard_e4s_v3 - runtime_version: "3.3" + runtime_version: "3.4" \ No newline at end of file diff --git a/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml b/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml index 1f4af1781fd..04da3bd18e9 100644 --- a/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml +++ b/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml @@ -34,5 +34,5 @@ identity: resources: instance_type: standard_e4s_v3 - runtime_version: "3.3" + runtime_version: "3.4" \ No newline at end of file diff --git a/cli/jobs/spark/serverless-spark-standalone-user-identity.yml b/cli/jobs/spark/serverless-spark-standalone-user-identity.yml index 86ce8695749..93482bb89a1 100644 --- a/cli/jobs/spark/serverless-spark-standalone-user-identity.yml +++ b/cli/jobs/spark/serverless-spark-standalone-user-identity.yml @@ -34,5 +34,5 @@ identity: resources: instance_type: standard_e4s_v3 - runtime_version: "3.3" + runtime_version: "3.4" \ No newline at end of file diff --git a/cli/monitoring/advanced-model-monitoring.yaml b/cli/monitoring/advanced-model-monitoring.yaml index 5f3c3031029..3b183eb3dbf 100644 --- a/cli/monitoring/advanced-model-monitoring.yaml +++ b/cli/monitoring/advanced-model-monitoring.yaml @@ -17,7 +17,7 @@ create_monitor: compute: instance_type: standard_e4s_v3 - runtime_version: "3.3" + runtime_version: "3.4" monitoring_target: ml_task: classification diff --git a/cli/monitoring/custom-monitoring.yaml b/cli/monitoring/custom-monitoring.yaml index ecb17e91a7a..2a4ac50cf07 100644 --- a/cli/monitoring/custom-monitoring.yaml +++ b/cli/monitoring/custom-monitoring.yaml @@ -10,7 +10,7 @@ trigger: create_monitor: compute: instance_type: "standard_e8s_v3" - runtime_version: "3.3" + runtime_version: "3.4" monitoring_signals: customSignal: diff --git a/cli/monitoring/generation-safety-quality-monitoring.yaml b/cli/monitoring/generation-safety-quality-monitoring.yaml index b39c0bde809..3871f7dcbb9 100644 --- a/cli/monitoring/generation-safety-quality-monitoring.yaml +++ b/cli/monitoring/generation-safety-quality-monitoring.yaml @@ -22,7 +22,7 @@ trigger: create_monitor: compute: instance_type: "standard_e8s_v3" - runtime_version: "3.3" + runtime_version: "3.4" monitoring_target: ml_task: classification endpoint_deployment_id: fake_endpoint_id diff --git a/cli/monitoring/model-monitoring-with-collected-data.yaml b/cli/monitoring/model-monitoring-with-collected-data.yaml index b39dbc477ad..b73d7728e3e 100644 --- a/cli/monitoring/model-monitoring-with-collected-data.yaml +++ b/cli/monitoring/model-monitoring-with-collected-data.yaml @@ -16,7 +16,7 @@ trigger: create_monitor: compute: instance_type: standard_e4s_v3 - runtime_version: "3.3" + runtime_version: "3.4" monitoring_target: ml_task: classification endpoint_deployment_id: azureml:fraud-detection-endpoint:fraud-detection-deployment diff --git a/cli/monitoring/out-of-box-monitoring.yaml b/cli/monitoring/out-of-box-monitoring.yaml index 4745ad7f1ed..a30491ae8fd 100644 --- a/cli/monitoring/out-of-box-monitoring.yaml +++ b/cli/monitoring/out-of-box-monitoring.yaml @@ -17,7 +17,7 @@ create_monitor: compute: # specify a spark compute for monitoring job instance_type: standard_e4s_v3 - runtime_version: "3.3" + runtime_version: "3.4" monitoring_target: ml_task: classification # model task type: [classification, regression, question_answering] diff --git a/sdk/python/jobs/spark/setup_spark.sh b/sdk/python/jobs/spark/setup_spark.sh index 9b6b5d4505a..46e1c982a6b 100644 --- a/sdk/python/jobs/spark/setup_spark.sh +++ b/sdk/python/jobs/spark/setup_spark.sh @@ -156,7 +156,7 @@ else az storage fs create -n $GEN2_FILE_SYSTEM --account-name $GEN2_STORAGE_NAME az synapse workspace create --name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --storage-account $GEN2_STORAGE_NAME --file-system $GEN2_FILE_SYSTEM --sql-admin-login-user $SQL_ADMIN_LOGIN_USER --sql-admin-login-password $RANDOM_STRING --location $LOCATION az role assignment create --role "Storage Blob Data Owner" --assignee $AML_USER_MANAGED_ID_OID --scope /subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/$GEN2_STORAGE_NAME/blobServices/default/containers/$GEN2_FILE_SYSTEM - az synapse spark pool create --name $SPARK_POOL_NAME --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --spark-version 3.3 --node-count 3 --node-size Medium --min-node-count 3 --max-node-count 10 --enable-auto-scale true + az synapse spark pool create --name $SPARK_POOL_NAME --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --spark-version 3.4 --node-count 3 --node-size Medium --min-node-count 3 --max-node-count 10 --enable-auto-scale true az synapse workspace firewall-rule create --name allowAll --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --start-ip-address 0.0.0.0 --end-ip-address 255.255.255.255 # diff --git a/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb b/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb index 1a336efc40d..e637706d08d 100644 --- a/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb +++ b/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb @@ -342,7 +342,7 @@ " spark_step.identity = ManagedIdentityConfiguration()\n", " spark_step.resources = {\n", " \"instance_type\": \"Standard_E8S_V3\",\n", - " \"runtime_version\": \"3.3.0\",\n", + " \"runtime_version\": \"3.4.0\",\n", " }\n", "\n", "\n", @@ -422,7 +422,7 @@ " spark_step.identity = UserIdentityConfiguration()\n", " spark_step.resources = {\n", " \"instance_type\": \"Standard_E8S_V3\",\n", - " \"runtime_version\": \"3.3.0\",\n", + " \"runtime_version\": \"3.4.0\",\n", " }\n", "\n", "\n", @@ -501,7 +501,7 @@ " spark_step.outputs.wrangled_data.mode = InputOutputModes.DIRECT\n", " spark_step.resources = {\n", " \"instance_type\": \"Standard_E8S_V3\",\n", - " \"runtime_version\": \"3.3.0\",\n", + " \"runtime_version\": \"3.4.0\",\n", " }\n", "\n", "\n", diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb index 245c2671222..88c2cc2dade 100644 --- a/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb +++ b/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb @@ -267,7 +267,7 @@ " executor_instances=2,\n", " resources={\n", " \"instance_type\": \"Standard_E8S_V3\",\n", - " \"runtime_version\": \"3.3.0\",\n", + " \"runtime_version\": \"3.4.0\",\n", " },\n", " inputs={\n", " \"titanic_data\": Input(\n", @@ -329,7 +329,7 @@ " executor_instances=2,\n", " resources={\n", " \"instance_type\": \"Standard_E8S_V3\",\n", - " \"runtime_version\": \"3.3.0\",\n", + " \"runtime_version\": \"3.4.0\",\n", " },\n", " inputs={\n", " \"titanic_data\": Input(\n", @@ -391,7 +391,7 @@ " executor_instances=2,\n", " resources={\n", " \"instance_type\": \"Standard_E8S_V3\",\n", - " \"runtime_version\": \"3.3.0\",\n", + " \"runtime_version\": \"3.4.0\",\n", " },\n", " inputs={\n", " \"titanic_data\": Input(\n", diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb index eace1dce406..96a756b5da2 100644 --- a/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb +++ b/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb @@ -535,7 +535,7 @@ " executor_instances=2,\n", " resources={\n", " \"instance_type\": \"Standard_E8S_V3\",\n", - " \"runtime_version\": \"3.3.0\",\n", + " \"runtime_version\": \"3.4.0\",\n", " },\n", " inputs={\n", " \"titanic_data\": Input(\n", @@ -769,7 +769,7 @@ " executor_instances=2,\n", " resources={\n", " \"instance_type\": \"Standard_E8S_V3\",\n", - " \"runtime_version\": \"3.3.0\",\n", + " \"runtime_version\": \"3.4.0\",\n", " },\n", " inputs={\n", " \"titanic_data\": Input(\n",