Merge branch 'main' into santiagxf/batch-ga

.github/CODEOWNERS

@@ -1,3 +1,4 @@
 # See for instructions on this file https://help.github.com/articles/about-codeowners/
 
 /python-sdk/tutorials/automl-with-azureml @rtanase @cesardelatorre @anupsms @lauralj103 @jeff-shepherd @raduk @swatig007 @martinezgjuan @MercyPrasanna @PhaniShekhar @yoyolin @sjinthehouse @vadthyavath @mispa-ms
+/tutorials/ @sdgilley

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,7 +4,10 @@
 # Checklist
 
 
-- [ ] I have read the [contribution guidelines](https://github.com/Azure/azureml-examples/blob/main/CONTRIBUTING.md)
+- [ ] I have read the contribution guidelines.
+  - [General](https://github.com/Azure/azureml-examples/blob/main/CONTRIBUTING.md)
+  - [SDK](https://github.com/Azure/azureml-examples/blob/main/sdk/CONTRIBUTING.md)
+  - [CLI](https://github.com/Azure/azureml-examples/blob/main/cli/CONTRIBUTING.md)
 - [ ] I have coordinated with the docs team ([email protected]) if this PR deletes files or changes any file names or file extensions.
 - [ ] Pull request includes test coverage for the included changes.
 - [ ] This notebook or file is added to the [CODEOWNERS](https://github.com/Azure/azureml-examples/blob/main/.github/CODEOWNERS) file, pointing to the author or the author's team.

.github/workflows/cli-jobs-pipelines-with-components-pipeline_job_with_flow_as_component-pipeline.yml

@@ -0,0 +1,51 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: cli-jobs-pipelines-with-components-pipeline_job_with_flow_as_component-pipeline
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "4 6/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - cli/jobs/pipelines-with-components/pipeline_job_with_flow_as_component/**
+      - infra/bootstrapping/**
+      - .github/workflows/cli-jobs-pipelines-with-components-pipeline_job_with_flow_as_component-pipeline.yml
+      - cli/run-pipeline-jobs.sh
+      - cli/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out repo
+      uses: actions/checkout@v2
+    - name: azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
+    - name: bootstrap resources
+      run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap.sh
+      working-directory: infra/bootstrapping
+      continue-on-error: false
+    - name: setup-cli
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash setup.sh
+      working-directory: cli
+      continue-on-error: true
+    - name: run job
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash -x ../../../run-job.sh pipeline.yml
+      working-directory: cli/jobs/pipelines-with-components/pipeline_job_with_flow_as_component

.github/workflows/sdk-featurestore_sample-test_featurestore_cli_samples.yml → .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml

@@ -3,7 +3,7 @@
 # Any manual changes to this file may cause incorrect behavior.
 # Any manual changes will be overwritten if the code is regenerated.
 
-name: sdk-featurestore_sample-test_featurestore_cli_samples
+name: sdk-featurestore_sample-automation-test-test_featurestore_cli_samples
 # This file is created by sdk/python/readme.py.
 # Please do not edit directly.
 on:
@@ -14,8 +14,8 @@ on:
     branches:
       - main
     paths:
-      - sdk/python/featurestore_sample/**
-      - .github/workflows/sdk-featurestore_sample-test_featurestore_cli_samples.yml
+      - sdk/python/featurestore_sample/automation-test/**
+      - .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_cli_samples.yml
       - sdk/python/dev-requirements.txt
       - infra/bootstrapping/**
       - sdk/python/setup.sh
@@ -60,21 +60,21 @@ jobs:
       continue-on-error: true
     - name: setup feature-store resources
       run: |
-          bash -x setup-resources-cli.sh test_featurestore_cli_samples.ipynb
+          bash -x automation-test/setup-resources-cli.sh automation-test/test_featurestore_cli_samples.ipynb
       working-directory: sdk/python/featurestore_sample
       continue-on-error: true
-    - name: run featurestore_sample/test_featurestore_cli_samples.ipynb
+    - name: run featurestore_sample/automation-test/test_featurestore_cli_samples.ipynb
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
           source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
           bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
           bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "test_featurestore_cli_samples.ipynb";
           [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
           papermill -k python test_featurestore_cli_samples.ipynb test_featurestore_cli_samples.output.ipynb
-      working-directory: sdk/python/featurestore_sample
+      working-directory: sdk/python/featurestore_sample/automation-test
     - name: upload notebook's working folder as an artifact
       if: ${{ always() }}
       uses: actions/upload-artifact@v2
       with:
         name: test_featurestore_cli_samples
-        path: sdk/python/featurestore_sample
+        path: sdk/python/featurestore_sample/automation-test

.github/workflows/sdk-featurestore_sample-test_featurestore_sdk_samples.yml → .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml

@@ -3,7 +3,7 @@
 # Any manual changes to this file may cause incorrect behavior.
 # Any manual changes will be overwritten if the code is regenerated.
 
-name: sdk-featurestore_sample-test_featurestore_sdk_samples
+name: sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples
 # This file is created by sdk/python/readme.py.
 # Please do not edit directly.
 on:
@@ -14,8 +14,8 @@ on:
     branches:
       - main
     paths:
-      - sdk/python/featurestore_sample/**
-      - .github/workflows/sdk-featurestore_sample-test_featurestore_sdk_samples.yml
+      - sdk/python/featurestore_sample/automation-test/**
+      - .github/workflows/sdk-featurestore_sample-automation-test-test_featurestore_sdk_samples.yml
       - sdk/python/dev-requirements.txt
       - infra/bootstrapping/**
       - sdk/python/setup.sh
@@ -60,21 +60,21 @@ jobs:
       continue-on-error: true
     - name: setup feature-store resources
       run: |
-          bash -x setup-resources.sh test_featurestore_sdk_samples.ipynb
+          bash -x automation-test/setup-resources.sh automation-test/test_featurestore_sdk_samples.ipynb
       working-directory: sdk/python/featurestore_sample
       continue-on-error: true
-    - name: run featurestore_sample/test_featurestore_sdk_samples.ipynb
+    - name: run featurestore_sample/automation-test/test_featurestore_sdk_samples.ipynb
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
           source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
           bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
           bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "test_featurestore_sdk_samples.ipynb";
           [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
           papermill -k python test_featurestore_sdk_samples.ipynb test_featurestore_sdk_samples.output.ipynb
-      working-directory: sdk/python/featurestore_sample
+      working-directory: sdk/python/featurestore_sample/automation-test
     - name: upload notebook's working folder as an artifact
       if: ${{ always() }}
       uses: actions/upload-artifact@v2
       with:
         name: test_featurestore_sdk_samples
-        path: sdk/python/featurestore_sample
+        path: sdk/python/featurestore_sample/automation-test

.github/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml

@@ -0,0 +1,75 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline
+# This file is created by sdk/python/readme.py.
+# Please do not edit directly.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "41 6/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - sdk/python/jobs/pipelines/1l_flow_in_pipeline/**
+      - .github/workflows/sdk-jobs-pipelines-1l_flow_in_pipeline-flow_in_pipeline.yml
+      - sdk/python/dev-requirements.txt
+      - infra/bootstrapping/**
+      - sdk/python/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out repo
+      uses: actions/checkout@v2
+    - name: setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.8"
+    - name: pip install notebook reqs
+      run: pip install -r sdk/python/dev-requirements.txt
+    - name: azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
+    - name: bootstrap resources
+      run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap.sh
+      working-directory: infra/bootstrapping
+      continue-on-error: false
+    - name: setup SDK
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash setup.sh
+      working-directory: sdk/python
+      continue-on-error: true
+    - name: setup-cli
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash setup.sh
+      working-directory: cli
+      continue-on-error: true
+    - name: run jobs/pipelines/1l_flow_in_pipeline/flow_in_pipeline.ipynb
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "flow_in_pipeline.ipynb";
+          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
+          papermill -k python flow_in_pipeline.ipynb flow_in_pipeline.output.ipynb
+      working-directory: sdk/python/jobs/pipelines/1l_flow_in_pipeline
+    - name: upload notebook's working folder as an artifact
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: flow_in_pipeline
+        path: sdk/python/jobs/pipelines/1l_flow_in_pipeline

CONTRIBUTING.md

@@ -29,13 +29,13 @@ This repository contains notebooks and sample code that demonstrate how to devel
 
 ## Issues
 
-All forms of feedback are welcome through [issues](https://github.com/Azure/azureml-examples/issues/new/choose) - please follow the pre-defined templates where applicable.
+All forms of feedback are welcome through [issues](https://github.com/Azure/azureml-examples/issues/new/choose). Please follow the pre-defined templates where applicable.
 
 ## Repository structure
 
 Azure Machine Learning has multiple developer experiences. The subdirectories at the root of the repo correspond to a developer experience, with the slight exception of `notebooks`.
 
-The `notebooks` directory is intended for iterative, interactive code development examples such as exploratory data anlysis or querying logged metrics.
+The `notebooks` directory is intended for iterative, interactive code development examples such as exploratory data analysis or querying logged metrics.
 
 ## Pull Requests
 
@@ -67,30 +67,33 @@ Also if adding new examples or changing existing descriptions, run the `readme.p
 python readme.py
 ```
 
-This will also generate a GitHub Actions workflow file for any new examples in the `.github/workflows` directory (with exceptions) to test the examples on the PR and regularly after merging into the main branch. PRs which edit existing examples will generally trigger a workflow to test the example. See the specific contributing guidelines for the subdirectories for further details. If the new notebook uses compute cluster, please add it to the `sdk/python/notebooks_config.ini` file so the compute clusters will be properly deleted after notebook run was finished. Create a section with the notebook name and add the option `COMPUTE_NAMES` with the compute cluster name. 
+This will also generate a GitHub Actions workflow file for any new examples in the `.github/workflows` directory (with exceptions) to test the examples on the PR and regularly after merging into the main branch. PRs which edit existing examples will generally trigger a workflow to test the example. See the specific contributing guidelines for the subdirectories for further details. If the new notebook uses a compute cluster, please add it to the `sdk/python/notebooks_config.ini` file so the compute clusters will be properly deleted after notebook run was finished. Create a section with the notebook name and add the option `COMPUTE_NAMES` with the compute cluster name. 
 
 ### Discoverability
 
 Examples in this repository can be indexed in the [Microsoft code samples browser](https://docs.microsoft.com/samples), enabling organic discoverability. To accomplish this:
 
-- add an excellent `README.md` file in the example directory
+- add an excellent `README.md` file in the example directory noting the overview, objective, and estimated runtime. (Note than estimated runtimes should not exceed 30 minutes).
 - add required YAML frontmatter at the top of the `README.md`
 
-The YAML frontmatter is this:
+The YAML frontmatter format looks like this:
 
 ```YAML
 ---
 page_type: sample
 languages:
 - azurecli
-- python
+- language1
+- language2
 products:
 - azure-machine-learning
 description: Example description.
 ---
 ```
 
-**Edit the description** and update the languages as needed.
+Edit the description and update the languages as needed.
+
+The Code Samples browser content is updated twice a week, so it may take a few days for your changes to be reflected.
 
 ### Other resources
 * [CLI contributing guide.](cli/CONTRIBUTING.md)

README.md

@@ -1,34 +1,47 @@
----
-page_type: sample
-languages:
-- azurecli
-- python
-products:
-- azure-machine-learning
-description: Top-level directory for official Azure Machine Learning sample code and examples.
----
-
 # Azure Machine Learning examples
 
-[![Python code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![license: MIT](https://img.shields.io/badge/License-MIT-purple.svg)](LICENSE)
 
 Welcome to the Azure Machine Learning examples repository!
 
-## Contents
-
-|directory||description|
-|-|-|-|
-|[`.github`](.github)||GitHub files like issue templates and actions workflows.|
-|[`cli`](cli)||Azure Machine Learning CLI v2 examples.||
-|[`sdk`](sdk)||Azure Machine Learning SDK v2 examples.||
-||[`python`](sdk/python/)| Azure Machine Learning Python SDK v2 examples.|
-||[`dotnet`](sdk/dotnet)| Azure Machine Learning .Net SDK v2 examples.|
-|[`setup`](setup)||Folder with setup scripts|
-||[`setup-ci`](setup/setup-ci)|Setup scripts to customize and configure ||an Azure Machine Learning compute instance.|
-||[`setupdsvm`](setup/setup-dsvm/RStudio/)|Setup RStudio on Data Science Virtual Machine (DSVM)|
-||[`setup-repo`](setup/setup-repo)|Setup scripts for Azure/azureml-examples.|
-|[`tutorials`](tutorials/)||Azure Machine Learning end to end Python SDK v2 tutorials|
+## About This Repository
+
+The azureml-examples repository contains examples and tutorials to help you learn how to use Azure Machine Learning
+(Azure ML) services and features.
+
+
+### Getting Started
+
+If you're getting started with Azure ML, consider working through our [tutorials] for the v2 Python SDK. You may
+also want to read through our [documentation](#supplementary-documentation).
+
+### SDKs
+
+The `sdk/` folder houses the examples for the Azure ML SDKs across several languages.
+
+We have an extensive collection of examples for the [Azure ML Python SDK v2][azure cli ml extension v2 overview] in
+[`sdk/python`][azureml python sdk v2 examples].
+
+We also offer some examples for our SDKs in other languages:
+
+* .NET: [`sdk/dotnet`][azureml dotnet sdk v2 examples]
+* TypeScript: [`sdk/typescript`][azureml typescript sdk v2 examples]
+
+### Azure Machine Learning extension for Azure CLI
+
+The [`cli/` folder][azureml cli extension examples] hosts our examples to use the
+[Azure Machine Learning extension][azure cli ml extension v2 overview] for [Azure CLI][azure cli overview].
+
+_Note_: If you're looking for examples that submit Azure ML jobs that run non-Python code, see:
+
+* **R**: [`cli/jobs/single-step/r`](./cli/jobs/single-step/r)
+
+
+## Supplementary Documentation
+
+- [Azure Machine Learning Documentation](https://docs.microsoft.com/azure/machine-learning)
+- [AzureML Python SDK v2 Overview]
+- [Azure CLI ML extension v2 Overview]
 
 ## Contributing
 
@@ -38,7 +51,11 @@ We welcome contributions and suggestions! Please see the [contributing guideline
 
 This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). Please see the [code of conduct](CODE_OF_CONDUCT.md) for details.
 
-## Reference
-
-- [Documentation](https://docs.microsoft.com/azure/machine-learning)
-
+[tutorials]: ./tutorials
+[azure cli overview]: https://learn.microsoft.com/en-us/cli/azure/what-is-azure-cli
+[azureml cli extension examples]: ./cli
+[azureml python sdk v2 examples]: ./sdk/python
+[azureml dotnet sdk v2 examples]: ./sdk/dotnet
+[azureml typescript sdk v2 examples]: ./sdk/typescript
+[azure cli ml extension v2 overview]: https://learn.microsoft.com/en-us/azure/machine-learning/concept-v2?view=azureml-api-2#azure-machine-learning-cli-v2
+[azureml python sdk v2 overview]: https://learn.microsoft.com/en-us/azure/machine-learning/concept-v2?view=azureml-api-2#azure-machine-learning-python-sdk-v2