diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml new file mode 100644 index 0000000000..52542d15c1 --- /dev/null +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml @@ -0,0 +1,80 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn +# This file is created by sdk/python/readme.py. +# Please do not edit directly. +on: + workflow_dispatch: + schedule: + - cron: "8 2/12 * * *" + pull_request: + branches: + - main + paths: + - sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/** + - .github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml + - sdk/python/dev-requirements.txt + - infra/bootstrapping/** + - sdk/python/setup.sh +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: check out repo + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: pip install notebook reqs + run: pip install -r sdk/python/dev-requirements.txt + - name: pip install mlflow reqs + run: pip install -r sdk/python/mlflow-requirements.txt + - name: azure login + uses: azure/login@v1 + with: + creds: ${{secrets.AZUREML_CREDENTIALS}} + - name: bootstrap resources + run: | + echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; + bash bootstrap.sh + working-directory: infra/bootstrapping + continue-on-error: false + - name: setup SDK + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: sdk/python + continue-on-error: true + - name: setup-cli + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash setup.sh + working-directory: cli + continue-on-error: true + - name: run jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb + run: | + source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; + source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; + bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "automl-forecasting-distributed-tcn.ipynb"; + [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; + papermill -k python -p compute_name automl-cpu-cluster automl-forecasting-distributed-tcn.ipynb automl-forecasting-distributed-tcn.output.ipynb + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn + - name: upload notebook's working folder as an artifact + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: automl-forecasting-distributed-tcn + path: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn + + - name: Remove the compute if notebook did not done it properly. + run: bash "${{ github.workspace }}/infra/bootstrapping/remove_computes.sh" "distributed-tcn-cluster" diff --git a/sdk/python/README.md b/sdk/python/README.md index 003e5d8e70..8bd244a11e 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -74,6 +74,7 @@ Test Status is for branch - **_main_** |foundation-models|system|[summarization-batch-endpoint](foundation-models/system/inference/summarization/summarization-batch-endpoint.ipynb)|*no description*|[![summarization-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-summarization-summarization-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-summarization-summarization-batch-endpoint.yml)| |jobs|automl-standalone-jobs|[automl-classification-task-bankmarketing](jobs/automl-standalone-jobs/automl-classification-task-bankmarketing/automl-classification-task-bankmarketing.ipynb)|*no description*|[![automl-classification-task-bankmarketing](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-automl-classification-task-bankmarketing.yml)| |jobs|automl-standalone-jobs|[mlflow-model-local-inference-test](jobs/automl-standalone-jobs/automl-classification-task-bankmarketing/mlflow-model-local-inference-test.ipynb)|*no description* - _This sample is excluded from automated tests_|[![mlflow-model-local-inference-test](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-mlflow-model-local-inference-test.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-classification-task-bankmarketing-mlflow-model-local-inference-test.yml)| +|jobs|automl-standalone-jobs|[automl-forecasting-distributed-tcn](jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb)|*no description*|[![automl-forecasting-distributed-tcn](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-distributed-tcn-automl-forecasting-distributed-tcn.yml)| |jobs|automl-standalone-jobs|[auto-ml-forecasting-github-dau](jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau.ipynb)|*no description*|[![auto-ml-forecasting-github-dau](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml)| |jobs|automl-standalone-jobs|[automl-forecasting-orange-juice-sales-mlflow](jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow.ipynb)|*no description*|[![automl-forecasting-orange-juice-sales-mlflow](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml)| |jobs|automl-standalone-jobs|[automl-forecasting-recipe-univariate-experiment-settings](jobs/automl-standalone-jobs/automl-forecasting-recipes-univariate/automl-forecasting-recipe-univariate-experiment-settings.ipynb)|*no description*|[![automl-forecasting-recipe-univariate-experiment-settings](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-experiment-settings.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-recipes-univariate-automl-forecasting-recipe-univariate-experiment-settings.yml)| diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb new file mode 100644 index 0000000000..644f5e4545 --- /dev/null +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/automl-forecasting-distributed-tcn.ipynb @@ -0,0 +1,1351 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Automated Machine Learning\n", + "\n", + "## Demand Forecasting Using Many Models (preview)\n", + "\n", + "> [!IMPORTANT]\n", + "> Items marked (preview) in this article are currently in public preview.\n", + "> The preview version is provided without a service level agreement, and it's not recommended for production workloads. Certain features might not be supported or might have constrained capabilities.\n", + "> For more information, see [Supplemental Terms of Use for Microsoft Azure Previews](https://azure.microsoft.com/support/legal/preview-supplemental-terms/).\n", + "\n", + "## Contents\n", + "1. [Introduction](#Introduction)\n", + "1. [Setup](#Setup)\n", + "1. [Compute](#Compute)\n", + "1. [Data](#Data)\n", + "1. [Train AutoML Model](#Train)\n", + "1. [Import Components From Registry](#ImportComponents)\n", + "1. [Create a Pipeline](#Pipeline)\n", + "1. [Kick Off Pipeline Runs](#PipelineRun)\n", + "1. [Download Output](#DownloadOutput)\n", + "1. [Compare Evaluation Results](#CompareResults)\n", + "1. [Deployment](#Deployment)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Introduction \n", + "\n", + "The objective of this notebook is to illustrate how to use the component-based AutoML single model solution. It walks you through all stages of model evaluation and production process starting with data ingestion and concluding with batch endpoint deployment for production. In this tutorial we will illustrate how to leverage AutoML and train a destributed TCN model ([link](placeholder)). However, the same notebook can be used to train a non-distributed TCN as well as the conventional ML models.\n", + "\n", + "We use a subset of UCI electricity data ([link](https://archive.ics.uci.edu/ml/datasets/ElectricityLoadDiagrams20112014#)) with the objective of predicting electricity demand per consumer 24 hours ahead. The data was preprocessed using the [data prep notebook](https://github.com/Azure/azureml-examples/blob/main/v1/python-sdk/tutorials/automl-with-azureml/forecasting-data-preparation/auto-ml-forecasting-data-preparation.ipynb). Please refer to it for illustration on how to download the data from the source, aggregate to an hourly frequency, convert from wide to long format and upload to the Datastore. Here, we will work with the data that has been pre-processed and saved locally in the parquet format.\n", + "\n", + "There are a number of steps you need to take before you can put a model into production. A user needs to prepare the data, partition it into appropriate sets, select the best model, evaluate it against a baseline, and monitor the model in real life to collect enough observations on how it would perform had it been put in production. Some of these steps are time consuming, some require certain expertise in writing code. The steps shown in this notebook follow a typical thought process one follows before the model is put in production.\n", + "\n", + "Make sure you have executed the [configuration](https://github.com/Azure/MachineLearningNotebooks/blob/master/configuration.ipynb) before running this notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Setup " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1682992408484 + } + }, + "outputs": [], + "source": [ + "# Import required libraries\n", + "import os\n", + "import datetime\n", + "import json\n", + "import yaml\n", + "import azure.ai.ml\n", + "\n", + "import pandas as pd\n", + "\n", + "from time import sleep\n", + "\n", + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "\n", + "from azure.ai.ml import MLClient, Input, Output\n", + "from azure.ai.ml import load_component\n", + "from azure.ai.ml import automl\n", + "from azure.ai.ml.constants import AssetTypes\n", + "from azure.ai.ml.dsl import pipeline\n", + "from azure.ai.ml.entities import (\n", + " BatchEndpoint,\n", + " BatchDeployment,\n", + " AmlCompute,\n", + " PipelineComponentBatchDeployment,\n", + ")\n", + "from azure.ai.ml.entities._job.automl.tabular.forecasting_settings import (\n", + " ForecastingSettings,\n", + ")\n", + "\n", + "print(f\"SDK version: {azure.ai.ml.__version__}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2.1. Configure workspace details and get a handle to the workspace\n", + "\n", + "The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning. In this section we will connect to the workspace in which the job will be run.\n", + "\n", + "To connect to a workspace, we need identifier parameters - a subscription, resource group and workspace name. We will use these details in the `MLClient` from `azure.ai.ml` to get a handle to the required Azure Machine Learning workspace. We use the default [default azure authentication](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python) for this tutorial. Check the [configuration notebook](https://github.com/Azure/MachineLearningNotebooks/blob/master/configuration.ipynb) for more details on how to configure credentials and connect to a workspace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " credential = DefaultAzureCredential()\n", + " # Check if given credential can get token successfully.\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential does not work\n", + " credential = InteractiveBrowserCredential()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " ml_client = MLClient.from_config(credential)\n", + "except Exception as ex:\n", + " print(ex)\n", + " # Enter details of your AML workspace\n", + " subscription_id = \"\"\n", + " resource_group = \"\"\n", + " workspace = \"\"\n", + " ml_client = MLClient(credential, subscription_id, resource_group, workspace)\n", + " print(ml_client)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2. Show Azure ML Workspace information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ws = ml_client.workspaces.get(name=ml_client.workspace_name)\n", + "\n", + "output = {}\n", + "output[\"Workspace\"] = ml_client.workspace_name\n", + "output[\"Subscription ID\"] = ml_client.connections._subscription_id\n", + "output[\"Resource Group\"] = ws.resource_group\n", + "output[\"Location\"] = ws.location\n", + "pd.DataFrame(data=output, index=[\"\"]).T" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Compute \n", + "\n", + "#### Create or Attach existing AmlCompute\n", + "\n", + "You will need to create a compute target for your AutoML run. In this tutorial, you will create AmlCompute as your training compute resource.\n", + "\n", + "> Note that if you have an AzureML Data Scientist role, you will not have permission to create compute resources. Talk to your workspace or IT admin to create the compute targets described in this section, if they do not already exist.\n", + "\n", + "\n", + "Here, we use a 10 node cluster of the `STANDARD_NC6s_v3` series for illustration purposes. You will need to adjust the compute type and the number of nodes based on your needs which can be driven by the speed needed for model selection, data size, etc. \n", + "\n", + "#### Creation of AmlCompute takes approximately 5 minutes. \n", + "If the AmlCompute with that name is already in your workspace, this code will skip the creation process.\n", + "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read [this article](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-manage-quotas) on the default limits and how to request more quota." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.core.exceptions import ResourceNotFoundError\n", + "\n", + "amlcompute_cluster_name = \"distributed-tcn-cluster\"\n", + "\n", + "try:\n", + " # Retrieve an already attached Azure Machine Learning Compute.\n", + " compute_target = ml_client.compute.get(amlcompute_cluster_name)\n", + "except ResourceNotFoundError as e:\n", + " compute_target = AmlCompute(\n", + " name=amlcompute_cluster_name,\n", + " size=\"Standard_NC6s_v3\",\n", + " type=\"amlcompute\",\n", + " min_instances=0,\n", + " max_instances=10,\n", + " idle_time_before_scale_down=600,\n", + " # vm_priority=\"LowPriority\"\n", + " )\n", + " poller = ml_client.begin_create_or_update(compute_target)\n", + " poller.wait()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Data \n", + "\n", + "For illustration purposes we use the UCI electricity data ([link](https://archive.ics.uci.edu/ml/datasets/ElectricityLoadDiagrams20112014#)). The original dataset contains electricity consumption data for 370 consumers measured at 15 minute intervals. In the data set for this demonstrations, we have aggregated to an hourly frequency and convereted to the kilowatt hours (kWh) for 10 customers. Each customer is assigned to one of the two groups as denoted by the entries in the `group_id` column. The following cells read and print the first few rows of the training data as well as print the number of unique time series in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "time_column_name = \"datetime\"\n", + "target_column_name = \"usage\"\n", + "time_series_id_column_names = [\"group_id\", \"customer_id\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_type = \"train\"\n", + "df = pd.read_parquet(f\"./data/{dataset_type}/uci_electro_small_{dataset_type}.parquet\")\n", + "df.head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nseries = df.groupby(time_series_id_column_names).ngroups\n", + "print(f\"Data contains {nseries} individual time-series\\n---\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df[time_series_id_column_names].drop_duplicates()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Please note that the data used in this experiment is of the Azure Machine Learning Tables (`MLTable`) format. This allows for the creation of a *blueprint* that defines how to load data files into memory as a Pandas or Spark data frame. See the following [link](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-mltable?view=azureml-api-2&tabs=cli) on more inforamtion on this topic. For documentation on creating your own MLTable assets for jobs beyond this notebook:\n", + "\n", + "- https://learn.microsoft.com/en-us/azure/machine-learning/reference-yaml-mltable details how to write MLTable YAMLs (required for each MLTable asset).\n", + "- https://learn.microsoft.com/en-us/azure/machine-learning/how-to-create-data-assets?tabs=Python-SDK covers how to work with them in the v2 CLI/SDK.\n", + "\n", + "Next, we upload the directory with the train, validation and test set data which will be used in this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Training MLTable defined locally, with local data to be uploaded\n", + "train_dataset = Input(type=AssetTypes.MLTABLE, path=\"./data/train\")\n", + "valid_dataset = Input(type=AssetTypes.MLTABLE, path=\"./data/valid\")\n", + "test_dataset = Input(type=AssetTypes.URI_FOLDER, path=\"./data/test\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Train Distributed TCN \n", + " \n", + "In this section we will configure and run the AutoML job to train the model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.1 Configure the AutoML Job\n", + "\n", + "First, we create a set of parameters which will be used to define the `forecasting()` factory function to kick off the model training. Think of this as the bare minimum settings that are necessary to define an AutoML job, and it contains the following properties:\n", + "\n", + "|Property|Description|\n", + "|-|-|\n", + "| **task** | forecasting |\n", + "| **target_column_name** | The name of the column to target for predictions. It must always be specified. This parameter is applicable to `training_data`, `validation_data` and `test_data`. |\n", + "| **primary_metric** | This is the metric that you want to optimize. Forecasting supports the following primary metrics We recommend using either the normalized root mean squared error (default metric) or normalized mean absolute error as a primary metric because they measure forecast accuracy. See the [link](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-automl-forecasting-faq#how-do-i-choose-the-primary-metric) for a more detailed discussion on this topic. |\n", + "\n", + "Please note that the `forecasting()` function also requires a training and/or validation data. We will provide this is section 5.2." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# automl settings\n", + "task = \"forecasting\"\n", + "target_column_name = target_column_name\n", + "primary_metric = \"normalized_root_mean_squared_error\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we define the forecasting specific parameters for the experiment. Technically, there are only 2 parameters that are necessary for forecasting tasks (`forecast_horizon` and `time_column_name`). For a greater control over the experiment we also list optional parameters that users can set, they are marked with an asterisk $(*)$ in the table below. See the [forecast settings API doc](https://learn.microsoft.com/en-us/python/api/azure-ai-ml/azure.ai.ml.automl.forecastingjob#azure-ai-ml-automl-forecastingjob-set-forecast-settings) for a complete list of available parameters.\n", + "\n", + "|Property|Description|\n", + "|-|-|\n", + "| **time_column_name** | The name of the time column in the data. |\n", + "| **forecast_horizon** | The forecast horizon is how many periods forward you would like to forecast. This integer horizon is in units of the timeseries frequency (e.g. daily, weekly). |\n", + "| **time_series_id_column_names*** | The column names used to uniquely identify the time series in data that has multiple rows with the same timestamp. If the time series identifiers are not defined, AutoML will detect them for you. |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# forecast settings\n", + "forecast_horizon = 24\n", + "time_column_name = time_column_name\n", + "time_series_id_column_names = time_series_id_column_names" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we set parameters to configure limits such as timeouts, number of trails, etc.\n", + "\n", + "|Property|Description|\n", + "|-|-|\n", + "| **timeout_minutes** | Maximum amount of time in minutes that the whole AutoML job can take before the job terminates. This timeout includes setup, featurization and training runs but does not include the ensembling and model explainability runs at the end of the process since those actions need to happen once all the trials (children jobs) are done. If not specified, the default job's total timeout is 6 days (8,640 minutes). To specify a timeout less than or equal to 1 hour (60 minutes), make sure your dataset's size is not greater than 10,000,000 (rows times column) or an error results. |\n", + "| **max_trials** | Represents the maximum number of trials an Automated ML job can try to run a training algorithm with different combination of hyperparameters. Its default value is set to 1000. If `enable_early_termination` is defined, then the number of trials used to run training algorithms can be smaller. |\n", + "| **max_concurrent_trials** | The maximum number of trials (children jobs) that would be executed in parallel. It's highly recommended to set the number of concurrent runs to the number of nodes in the cluster (aml compute defined in `compute`). The default value is 1. |\n", + "| **max_nodes** | Maximum number of nodes to use in training. This value should be set only for the distirbuted TCN training. We encourage this value to be a multiple of max_concurrent_iterations. The multiple indicates the number of nodes that will be used by each concurrent iteration. Minimum acceptable value to kick off distributed training is 2. |\n", + "| **enable_early_termination** | Represents whether to enable of experiment termination if the loss score doesn't improve after 'x' number of iterations. In an Automated ML job, no early stopping is applied on first 20 iterations. The early stopping window starts only after first 20 iterations. The default value is `True`. |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# training limits\n", + "timeout_minutes = 60\n", + "max_concurrent_trials = 5\n", + "max_trials = 15\n", + "max_nodes = 10\n", + "enable_early_termination = True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we set parameters to configure training parameters such as enabling DNN and blocking/alowing specific models.\n", + "\n", + "|Property|Description|\n", + "|-|-|\n", + "| **enable_dnn_training** | A flag to turn on or off the inclusion of DNN based models to try out during model selection. The default value is `False`. |\n", + "| **training_mode** |The training mode to use. The possible values are `distributed` and `non_distributed` (default value). When this parameter is set to `distributed` and `enablle_dnn_training=True`, a disitributed TCN run will be kicked off. |\n", + "| **allowed_training_algorithms** | A list of Time Series Forecasting algorithms to try out as base model for model training in an experiment. If it is omitted or set to `None`, then all supported algorithms are used during experiment, except algorithms specified in `blocked_training_algorithms`. The default value is `None`. |\n", + "| **blocked_training_algorithms** | A list of Time Series Forecasting algorithms to not run as base model while model training in an experiment. If it is omitted or set to `None`, then all supported algorithms are used during model training. The default value is `None`.|\n", + "| **enable_model_explainability** | Represents a flag to turn on model explainability like feature importance, of best model evaluated by Automated ML system. The default value is `True`. |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# training settings\n", + "enable_dnn_training = True\n", + "training_mode = \"distributed\"\n", + "allowed_training_algorithms = [\"TCNForecaster\"]\n", + "blocked_training_algorithms = None\n", + "enable_model_explainability = True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.2. Create the AutoML Forecasting Job" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# -- 5.2.1. Create forecasting job\n", + "training_job = automl.forecasting(\n", + " compute=amlcompute_cluster_name,\n", + " training_data=Input(type=\"uri_folder\", path=\"./data/train\"),\n", + " validation_data=Input(type=\"uri_folder\", path=\"./data/valid\"),\n", + " target_column_name=target_column_name,\n", + " primary_metric=primary_metric,\n", + " enable_model_explainability=enable_model_explainability,\n", + " outputs={\"best_model\": Output(type=AssetTypes.CUSTOM_MODEL)},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# -- 5.2.2 Define forecasting settings\n", + "training_job.set_forecast_settings(\n", + " forecast_horizon=forecast_horizon,\n", + " time_column_name=time_column_name,\n", + " time_series_id_column_names=time_series_id_column_names,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# -- 5.2.3 Set training parameters\n", + "training_job.set_training(\n", + " enable_dnn_training=enable_dnn_training,\n", + " training_mode=training_mode,\n", + " enable_model_explainability=enable_model_explainability,\n", + " allowed_training_algorithms=allowed_training_algorithms,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# -- 5.2.4 Set training limits\n", + "training_job.set_limits(\n", + " timeout_minutes=timeout_minutes,\n", + " max_trials=max_trials,\n", + " max_concurrent_trials=max_concurrent_trials,\n", + " max_nodes=max_nodes,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.3. Train the AutoML model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Submit the AutoML job\n", + "returned_job = ml_client.jobs.create_or_update(training_job)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Wait until AutoML training runs are finished\n", + "ml_client.jobs.stream(returned_job.name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.4. Download Best Model's Artifacts\n", + "\n", + "Next, we download the best TCN model's artifacts which will be used in the evaluation pipeline and batch deployment.\n", + "\n", + "##### Obtain the tracking URI for MLFlow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import mlflow\n", + "\n", + "# Obtain the tracking URL from MLClient\n", + "MLFLOW_TRACKING_URI = ml_client.workspaces.get(\n", + " name=ml_client.workspace_name\n", + ").mlflow_tracking_uri\n", + "\n", + "print(MLFLOW_TRACKING_URI)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Set the MLFLOW TRACKING URI\n", + "\n", + "mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n", + "\n", + "print(\"\\nCurrent tracking uri: {}\".format(mlflow.get_tracking_uri()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "local_dir = \"./artifact_downloads\"\n", + "os.makedirs(local_dir, exist_ok=True)\n", + "\n", + "for child in ml_client.jobs.list(parent_job_name=returned_job.name):\n", + " print(f\"{child.name}\\n---\")\n", + " if \"_HD\" in child.name:\n", + " print(f'Best child run ID: {child.properties[\"best_child_run_id\"]}\\n---')\n", + " # Download best run's artifacts/outputs\n", + " local_path = mlflow.artifacts.download_artifacts(\n", + " run_id=child.properties[\"best_child_run_id\"],\n", + " artifact_path=\"outputs\",\n", + " dst_path=local_dir,\n", + " )\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Import Components From Registry " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An Azure Machine Learning component is a self-contained piece of code that does one step in a machine learning pipeline. A component is analogous to a function - it has a name, inputs, outputs, and a body. Components are the building blocks of the Azure Machine Learning pipelines. It's a good engineering practice to build a machine learning pipeline where each step has well-defined inputs and outputs. In Azure Machine Learning, a component represents one reusable step in a pipeline. Components are designed to help improve the productivity of pipeline building. Specifically, components offer:\n", + "\n", + "- Well-defined interface: Components require a well-defined interface (input and output). The interface allows the user to build steps and connect steps easily. The interface also hides the complex logic of a step and removes the burden of understanding how the step is implemented.\n", + "\n", + "- Share and reuse: As the building blocks of a pipeline, components can be easily shared and reused across pipelines, workspaces, and subscriptions. Components built by one team can be discovered and used by another team.\n", + "\n", + "- Version control: Components are versioned. The component producers can keep improving components and publish new versions. Consumers can use specific component versions in their pipelines. This gives them compatibility and reproducibility.\n", + "\n", + "For a more detailed information on this subject, refer to the this [link](https://learn.microsoft.com/en-us/azure/machine-learning/concept-component?view=azureml-api-2).\n", + "\n", + "To import components, we need to get the registry. The following command obtains the public regsitry from which we will import components for our experiment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get registry for the inference component\n", + "ml_client_registry = MLClient(credential=credential, registry_name=\"azureml\")\n", + "print(ml_client_registry)\n", + "print(\"---\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we pull specific components and use them to build a pipeline of steps. For the illustration of the product evaluation workflow we will use the following components:\n", + "\n", + "- Inference componnet: generates forecast for each partition. This can be done on the test and inference sets.\n", + "- Compute metrics component: calculates metrics per time series if the inference component was used on a test set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_component = ml_client_registry.components.get(\n", + " name=\"automl_forecasting_inference\", label=\"latest\"\n", + ")\n", + "print(f\"Inference component version: {inference_component.version}\\n---\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "compute_metrics_component = ml_client_registry.components.get(\n", + " name=\"compute_metrics\", label=\"latest\"\n", + ")\n", + "print(f\"Compute metrics component version: {compute_metrics_component.version}\\n---\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Build an evaluation pipeline \n", + "\n", + "Now that we imported the components we will build an evaluation pipeline. This pipeline will allow us to genererate rolling forecast on the test set, and calculate metrics on the test set output.\n", + "\n", + "### 7.1. Set Pipeline Parameters\n", + "\n", + "AzureML components can only receive specific object types such as strings, JSON/YML files, URI Folders and URI Files. Other object types are not accepted. Because of this, we will create the pipeline by utilizing the `pipeline_parameters` dictionary. Most of the parameters in this dictionary will define the settings for the model training step of the pipeline and the remaining ones will be used in inference and compute metrics components. To have a better understanding of what these settings represents, we will build this dictionary in sequential steps. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we set declare the parameters that will be used be the inference and compute metrics components.\n", + "\n", + "|Property|Description|\n", + "|-|-|\n", + "| **forecast_mode** | Type of forecat to perform on the test set. Can be `recursive` or `rolling`. Rolling forecast can be used for the evaluation purpose. The default value is `recursive`. |\n", + "| **forecast_step** | The forecast step used for rolling forecast. See the following [link](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-auto-train-forecast?view=azureml-api-2#evaluating-model-accuracy-with-a-rolling-forecast) for more details. |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipeline_parameters = dict(forecast_mode=\"rolling\", forecast_step=24)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@pipeline(\n", + " description=\"AutoML Forecasting TCN Evaluation Pipeline\",\n", + ")\n", + "def evaluation_pipeline(\n", + " inference_data: Input(type=AssetTypes.MLTABLE),\n", + " model_path: Input(type=AssetTypes.MLFLOW_MODEL),\n", + "):\n", + " # 0. Extract pipeline parameters from the dictionary\n", + " forecast_mode = pipeline_parameters.get(\"forecast_mode\", \"recursive\")\n", + " forecast_step = pipeline_parameters.get(\"forecast_step\", 1)\n", + "\n", + " # 1. Inferencing step\n", + " inference_node = inference_component(\n", + " test_data=inference_data,\n", + " model_path=model_path,\n", + " target_column_name=target_column_name,\n", + " forecast_mode=forecast_mode,\n", + " forecast_step=forecast_step,\n", + " )\n", + "\n", + " # 3. Metrics calculation step\n", + " compute_metrics_node = compute_metrics_component(\n", + " task=\"tabular-forecasting\",\n", + " prediction=inference_node.outputs.inference_output_file,\n", + " ground_truth=inference_node.outputs.inference_output_file,\n", + " evaluation_config=inference_node.outputs.evaluation_config_output_file,\n", + " )\n", + " compute_metrics_node.compute = (\n", + " amlcompute_cluster_name # compute_name # amlcompute_cluster_name\n", + " )\n", + "\n", + " # 4. Specify pipeline outputs\n", + " return {\n", + " \"output_files\": compute_metrics_node.outputs.evaluation_result,\n", + " \"forecast_output\": inference_node.outputs.inference_output_file,\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Kick Off the Evaluation Pipeline Run \n", + " \n", + "Now that the pipeline is defined, we will use it to kick off an experiment which will inference and evaluate the performance for the best AutoML model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "pipeline_job = evaluation_pipeline(\n", + " inference_data=Input(type=AssetTypes.URI_FOLDER, path=\"./data/test\"),\n", + " model_path=Input(\n", + " type=AssetTypes.MLFLOW_MODEL, path=\"./artifact_downloads/outputs/mlflow-model\"\n", + " ),\n", + ")\n", + "print(pipeline_job)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# set pipeline level compute\n", + "pipeline_job.settings.default_compute = amlcompute_cluster_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "experiment_name = \"tcn-evaluation-\" + datetime.datetime.now().strftime(\"%Y%m%d\")\n", + "\n", + "pipeline_submitted_job = ml_client.jobs.create_or_update(\n", + " pipeline_job,\n", + " experiment_name=experiment_name,\n", + ")\n", + "ml_client.jobs.stream(pipeline_submitted_job.name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# To rehydrate run\n", + "# RUN_ID = \"\"\n", + "# pipeline_submitted_job = ml_client.jobs.get(RUN_ID)\n", + "# pipeline_submitted_job" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 9. Download Pipeline Output \n", + " \n", + "Next, we will download the output files generated by the compute metrics components for each executed pipeline and save them in the corresponfing subfolder of the `output` folder. First, we create corresponding output directories. Then, we execute the `ml_client.jobs.download` command which downloads experiments' outputs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create output directories\n", + "automl_output_dir = os.path.join(os.getcwd(), \"output/automl\")\n", + "os.makedirs(automl_output_dir, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ml_client.jobs.download(\n", + " name=pipeline_submitted_job.name,\n", + " download_path=automl_output_dir,\n", + " output_name=\"output_files\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ml_client.jobs.download(\n", + " name=pipeline_submitted_job.name,\n", + " download_path=automl_output_dir,\n", + " output_name=\"forecast_output\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 10. Evaluation Results \n", + "\n", + "### 10.1. Examine Metrics\n", + "\n", + "In this section, we compare metrics for the 2 pipeline runs to quantify accuracy improvement of AutoML over the baseline model. First, we compare metrics that are calculated for the entire dataset. Since there are 10 unique time series in the test dataset, these individual metrics are aggregated into a single number. The non-normalized metrics can be misleading due to the difference in scales of each unique time series. The following [article (placeholder)](https://review.learn.microsoft.com/en-us/azure/machine-learning/how-to-understand-automated-ml?view=azureml-api-2&branch=pr-en-us-238443#forecasting-metrics-normalization-and-aggregation) explains this topic in a greater detail.\n", + "\n", + "The code in the next cell loads dataset metrics for each of the experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metrics_artifacts_path = os.path.join(\n", + " \"named-outputs\", \"output_files\", \"evaluationResult\"\n", + ")\n", + "\n", + "with open(os.path.join(automl_output_dir, metrics_artifacts_path, \"metrics.json\")) as f:\n", + " metrics_automl_series = json.load(f)\n", + " metrics_automl = (\n", + " pd.Series(metrics_automl_series).to_frame(name=\"score\").reset_index(drop=False)\n", + " )\n", + "metrics_automl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 10.1.1. Detailed Metrics\n", + "\n", + "Next, we will load and examine the detailed accuracy metrics since the aggregate metrics may not convey enough information to make a decision about product accuracy. It may be helpful to examine metrics at a more granular level. We will extract metrics per time series. To do this, we create a helper function `extract_specific_metric` which reads the JSON file and returns a specified metric for each time series. Even though the file contains the following metrics, we will we will focus on the normalized root mean squared error (NRMSE) accuracy metric for illustration purposes. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def extract_specific_metric(path, metric_name):\n", + " with open(path) as f:\n", + " artifact = json.load(f)\n", + " all_metrics = pd.DataFrame(artifact[\"data\"])\n", + " index_scores = [\"customer_id\"] + [metric_name]\n", + " return all_metrics[index_scores]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metrics_table_relative_path = os.path.join(\n", + " metrics_artifacts_path, \"artifacts\", \"forecast_time_series_id_distribution_table\"\n", + ")\n", + "automl_detailed_metrics = extract_specific_metric(\n", + " os.path.join(automl_output_dir, metrics_table_relative_path),\n", + " \"normalized_root_mean_squared_error\",\n", + ")\n", + "automl_detailed_metrics" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 10.2. Generate Time Series Plots\n", + "\n", + "Here, we generate forecast versus actuals plot for the test set for both the best model and the baseline. Since we use rolling evaluation with the step size of 24 hours, this mimics the behavior of putting both models in production and monitoring their behavior for the duration of the test set. This step helps you make informed decisions about model performance and saves numerous costs associated with productionalizing the model and monitoring its performance in real life. \n", + "\n", + "In the next block of code, we, load the test set output for each of the runs and merge the data. Then, we generate and save time series plots." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "forecast_table_relative_path = os.path.join(\n", + " \"named-outputs\", \"forecast_output\", \"inference_output_file\"\n", + ")\n", + "\n", + "forecast_column_name = \"automl_prediction\"\n", + "actual_column_name = \"automl_actual\"\n", + "forecast_origin_column_name = \"automl_forecast_origin\"\n", + "\n", + "backtest = pd.read_json(\n", + " os.path.join(automl_output_dir, forecast_table_relative_path), lines=True\n", + ")\n", + "print(f\"AutoML backtest table size: {backtest.shape}\\n---\")\n", + "backtest.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from scripts.helper_scripts import draw_one_plot\n", + "from matplotlib import pyplot as plt\n", + "from matplotlib.backends.backend_pdf import PdfPages\n", + "\n", + "plot_filename = \"forecast_vs_actual.pdf\"\n", + "\n", + "pdf = PdfPages(os.path.join(os.getcwd(), \"./output\", plot_filename))\n", + "for _, one_forecast in backtest.groupby(\"customer_id\"):\n", + " one_forecast[time_column_name] = pd.to_datetime(one_forecast[time_column_name])\n", + " one_forecast.sort_values(time_column_name, inplace=True)\n", + " draw_one_plot(\n", + " one_forecast,\n", + " time_column_name,\n", + " target_column_name,\n", + " [\"customer_id\"],\n", + " [actual_column_name, forecast_column_name],\n", + " pdf,\n", + " plot_predictions=True,\n", + " )\n", + "pdf.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import IFrame\n", + "\n", + "IFrame(os.path.join(\"./output/forecast_vs_actual.pdf\"), width=800, height=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 11. Deployment \n", + "\n", + "In this section, we will illustrate how to perfrom batch inference using the inference component. Batch endpoints are endpoints that are used to do batch inferencing on large volumes of data in asynchronous way. Batch endpoints receive pointers to data and run jobs asynchronously to process the data in parallel on compute clusters and store outputs to a datastore for further analysis. For more information on batch endpoints see this [link](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-use-batch-scoring-pipeline?view=azureml-api-2&tabs=python)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 11.1. Build the pipeline\n", + "\n", + "First, we create a pipeline consisting of one step that invokes the inference componnet. This pipeline takes the inference data set as a parameter, generates a forecast and returns the predictions. The only one named output of this pipeline will be `forecast`. It is a table with predictions, stored in JSONL format. In the current setup, users can generate distribution forecast. To do this, uncomment the `forecast_quantiles` line in the pipeline definition and specify desired quantiles as a string. In the code example below the valuess of 0.1 and 0.9 are entered as `\"0.1,0.9\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define pipeline\n", + "@pipeline(\n", + " description=\"AutoML Inferencing Pipeline\",\n", + ")\n", + "def demand_inference_single_model(\n", + " test_data: Input(type=AssetTypes.MLTABLE),\n", + " model_path: Input(type=AssetTypes.MLFLOW_MODEL),\n", + " target_column_name: Input(type=\"string\"),\n", + " forecast_mode: Input(type=\"string\"),\n", + "):\n", + " inference_node = inference_component(\n", + " test_data=test_data,\n", + " model_path=model_path,\n", + " target_column_name=target_column_name,\n", + " forecast_mode=forecast_mode,\n", + " # forecast_quantiles=\"0.1,0.9\"\n", + " )\n", + " return {\n", + " \"forecast\": inference_node.outputs.inference_output_file,\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 11.2. Create Batch Endpoint\n", + "\n", + "A batch endpoint's name needs to be unique in each region since the name is used to construct the invocation URI. To ensure uniqueness, append any trailing characters to the name specified in the following code." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import string\n", + "\n", + "# Creating a unique endpoint name by including a random suffix\n", + "allowed_chars = string.ascii_lowercase + string.digits\n", + "endpoint_suffix = \"\".join(random.choice(allowed_chars) for x in range(5))\n", + "endpoint_name = \"sdk-tcn-\" + endpoint_suffix\n", + "\n", + "print(f\"Endpoint name: {endpoint_name}\\n---\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "endpoint = BatchEndpoint(\n", + " name=endpoint_name,\n", + " description=\"An endpoint for component deployments\",\n", + " properties={\"ComponentDeployment.Enabled\": True},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following command creates the endpoint in the workspace usign the MLClient created earlier. This command will start the endpoint creation and return a confirmation response while the endpoint creation continues." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ml_client.batch_endpoints.begin_create_or_update(endpoint).result()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 11.2. Create the Deployment\n", + "\n", + "A deployment is a set of resources required for hosting the model that does the actual inferencing. Our pipeline is defined in a function. To transform it to a component, you'll use the `build()` method. Pipeline components are reusable compute graphs that can be included in batch deployments or used to compose more complex pipelines." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipeline_component = demand_inference_single_model._pipeline_builder.build()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can define the deployment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "deployment = PipelineComponentBatchDeployment(\n", + " name=\"sdk-tcn-deployment\",\n", + " description=\"A TCN deployment.\",\n", + " endpoint_name=endpoint.name,\n", + " component=pipeline_component,\n", + " settings={\"default_compute\": amlcompute_cluster_name},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following command creates the deployment in the workspace usign the MLClient created earlier. This command will start the deployment creation and return a confirmation response while the deployment creation continues." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ml_client.batch_deployments.begin_create_or_update(deployment).result()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 11.3. Invoke the Endpoint\n", + "\n", + "The next cell contains the command that invokes the endpoint for batch inference job. The `invoke` method contains the `inputs` parameter. This parameter contains the inputs necessary to execute the inference component on the endpoint. To convince yourself this is the case, compare the input parameters for the `inference_component_from_registry` in section 7.1 with the `inputs` we are providing in the next cell. They are identical.\n", + "\n", + "Notice, the the `forecast_mode` is set to `\"recursive\"`. In the evaluation pipeline this component was used to generate rolling forecast to evaluate model performance on the test set. For more details on rolling evaluation, see our [forecasting model evaluation article](placeholder). Here, we are using it to generate a forecast." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "batch_job = ml_client.batch_endpoints.invoke(\n", + " endpoint_name=endpoint.name,\n", + " deployment_name=deployment.name,\n", + " inputs={\n", + " \"test_data\": Input(path=os.path.join(os.getcwd(), \"data\", \"inference\")),\n", + " \"model_path\": Input(\n", + " path=os.path.join(\"./artifact_downloads\", \"outputs\", \"mlflow-model\")\n", + " ),\n", + " \"target_column_name\": Input(type=\"string\", default=target_column_name),\n", + " \"forecast_mode\": Input(type=\"string\", default=\"recursive\"),\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we will stream the job output to monitor the execution." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "job_name = batch_job.name\n", + "batch_job = ml_client.jobs.get(name=job_name)\n", + "print(f\"Batch job status: {batch_job.status}\\n---\")\n", + "ml_client.jobs.stream(name=job_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 11.4. Download Forecast Output\n", + "\n", + "Finally, we download the forecast output and print the first few rows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fcst_output_dir = os.path.join(os.getcwd(), \"forecast\")\n", + "\n", + "for child in ml_client.jobs.list(parent_job_name=job_name):\n", + " print(f\"{child.name}\\n---\\nDownloading data ...\\n---\")\n", + " for attempt in range(3):\n", + " print(f\"Attempt: {attempt}\")\n", + " try:\n", + " ml_client.jobs.download(\n", + " child.name,\n", + " download_path=fcst_output_dir,\n", + " output_name=\"inference_output_file\",\n", + " )\n", + " break\n", + " except BaseException:\n", + " sleep(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fcst_df = pd.read_json(\n", + " os.path.join(\n", + " fcst_output_dir,\n", + " \"named-outputs\",\n", + " \"inference_output_file\",\n", + " \"inference_output_file\",\n", + " ),\n", + " orient=\"records\",\n", + " lines=True,\n", + ")\n", + "fcst_df[time_column_name] = pd.to_datetime(fcst_df[time_column_name], unit=\"ms\")\n", + "fcst_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 11.5. [Optional] Delete the Endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ml_client.online_endpoints.begin_delete(name=endpoint.name).wait()" + ] + } + ], + "metadata": { + "kernel_info": { + "name": "python310-sdkv2" + }, + "kernelspec": { + "display_name": "Python 3.10 - SDK v2", + "language": "python", + "name": "python310-sdkv2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true + } + }, + "ms_spell_check": { + "ms_spell_check_language": "en" + } + }, + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/inference/MLTable b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/inference/MLTable new file mode 100644 index 0000000000..99d87d700a --- /dev/null +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/inference/MLTable @@ -0,0 +1,7 @@ +paths: +- file: ./uci_electro_small_inference.parquet +transformations: +- read_parquet: + include_path_column: false + path_column: Path +type: mltable \ No newline at end of file diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/inference/uci_electro_small_inference.parquet b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/inference/uci_electro_small_inference.parquet new file mode 100644 index 0000000000..8f888b77c4 Binary files /dev/null and b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/inference/uci_electro_small_inference.parquet differ diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/test/MLTable b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/test/MLTable new file mode 100644 index 0000000000..73dde98406 --- /dev/null +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/test/MLTable @@ -0,0 +1,7 @@ +paths: +- file: ./uci_electro_small_test.parquet +transformations: +- read_parquet: + include_path_column: false + path_column: Path +type: mltable \ No newline at end of file diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/test/uci_electro_small_test.parquet b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/test/uci_electro_small_test.parquet new file mode 100644 index 0000000000..a111d06d13 Binary files /dev/null and b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/test/uci_electro_small_test.parquet differ diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/train/MLTable b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/train/MLTable new file mode 100644 index 0000000000..1f63f5e271 --- /dev/null +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/train/MLTable @@ -0,0 +1,7 @@ +paths: +- file: ./uci_electro_small_train.parquet +transformations: +- read_parquet: + include_path_column: false + path_column: Path +type: mltable \ No newline at end of file diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/train/uci_electro_small_train.parquet b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/train/uci_electro_small_train.parquet new file mode 100644 index 0000000000..7901b15381 Binary files /dev/null and b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/train/uci_electro_small_train.parquet differ diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/valid/MLTable b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/valid/MLTable new file mode 100644 index 0000000000..2c71f640bd --- /dev/null +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/valid/MLTable @@ -0,0 +1,7 @@ +paths: +- file: ./uci_electro_small_valid.parquet +transformations: +- read_parquet: + include_path_column: false + path_column: Path +type: mltable \ No newline at end of file diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/valid/uci_electro_small_valid.parquet b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/valid/uci_electro_small_valid.parquet new file mode 100644 index 0000000000..2d20a65116 Binary files /dev/null and b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/data/valid/uci_electro_small_valid.parquet differ diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/scripts/helper_scripts.py b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/scripts/helper_scripts.py new file mode 100644 index 0000000000..e906be85ca --- /dev/null +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-distributed-tcn/scripts/helper_scripts.py @@ -0,0 +1,61 @@ +from typing import Any, Dict, Optional, List + +import argparse +import json +import os +import re +import shutil + +import pandas as pd + +from typing import List, Union, Tuple +from matplotlib import pyplot as plt +from matplotlib.backends.backend_pdf import PdfPages + + +def _format_grain_name(grain: Union[str, Tuple[str], List[str]]) -> str: + """ + Convert grain name to string. + + :param grain: the grain name. + :return: the string representation of the given grain. + """ + if not isinstance(grain, tuple) and not isinstance(grain, list): + return str(grain) + grain = list(map(str, grain)) + return "|".join(grain) + + +def draw_one_plot( + df: pd.DataFrame, + time_column_name: str, + target_column_name: str, + grain_column_names: List[str], + columns_to_plot: List[str], + pdf: PdfPages, + plot_predictions=False, +) -> None: + """ + Draw the single plot. + + :param df: The data frame with the data to build plot. + :param time_column_name: The name of a time column. + :param grain_column_names: The name of grain columns. + :param pdf: The pdf backend used to render the plot. + """ + if isinstance(grain_column_names, str): + grain_column_names = [grain_column_names] + fig, _ = plt.subplots(figsize=(20, 10)) + df = df.set_index(time_column_name) + plt.plot(df[columns_to_plot]) + plt.xticks(rotation=45) + if grain_column_names: + grain_name = [df[grain].iloc[0] for grain in grain_column_names] + plt.title(f"Time series ID: {_format_grain_name(grain_name)}") + plt.legend(columns_to_plot) + plt.close(fig) + pdf.savefig(fig) + + +if __name__ == "__main__": + pass diff --git a/sdk/python/notebooks_config.ini b/sdk/python/notebooks_config.ini index 7dbbd4e00f..b59062b3c2 100644 --- a/sdk/python/notebooks_config.ini +++ b/sdk/python/notebooks_config.ini @@ -37,3 +37,7 @@ COMPUTE_NAMES = demand-fcst-hts-cluster [automl-forecasting-demand-many-models-in-pipeline] USE_FORECAST_REQUIREMENTS = 0 COMPUTE_NAMES = demand-fcst-mm-cluster + +[automl-forecasting-distributed-tcn] +USE_FORECAST_REQUIREMENTS = 0 +COMPUTE_NAMES = "distributed-tcn-cluster" diff --git a/sdk/python/readme.py b/sdk/python/readme.py index a96db90d07..b98b7f4b41 100644 --- a/sdk/python/readme.py +++ b/sdk/python/readme.py @@ -105,7 +105,12 @@ def get_additional_requirements(req_name, req_path): def get_mlflow_import(notebook, validation_yml): with open(notebook, "r", encoding="utf-8") as f: - if validation_yml or "import mlflow" in f.read(): + string_file = f.read() + if ( + validation_yml + or "import mlflow" in string_file + or "from mlflow" in string_file + ): return get_additional_requirements( "mlflow", "sdk/python/mlflow-requirements.txt" )