From 47ddab5f9514b6c7b907a6f0e3ff8c2c698c675a Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Mon, 6 Nov 2023 09:47:50 -0800 Subject: [PATCH] Initial commit --- .github/CODE_OF_CONDUCT.md | 9 + .github/SECURITY.md | 41 +++++ .github/SUPPORT.md | 25 +++ .github/workflows/build_arm.yml | 28 +++ .gitignore | 142 ++++++++++++++ Environments/FunctionApp/azuredeploy.json | 159 ++++++++++++++++ Environments/FunctionApp/main.bicep | 119 ++++++++++++ Environments/FunctionApp/manifest.yaml | 32 ++++ Environments/README.md | 21 +++ Environments/Sandbox/azuredeploy.json | 12 ++ Environments/Sandbox/main.bicep | 2 + Environments/Sandbox/manifest.yaml | 7 + Environments/WebApp/azuredeploy.json | 65 +++++++ Environments/WebApp/main.bicep | 35 ++++ Environments/WebApp/manifest.yaml | 14 ++ LICENSE | 21 +++ README.md | 35 ++++ documentation/README.md | 9 + .../terraform-private-preview/README.md | 47 +++++ .../sample-catalog/function-app/README.md | 8 + .../function-app/function_app.tf | 96 ++++++++++ .../sample-catalog/function-app/manifest.yaml | 22 +++ .../sample-catalog/machine-learning/README.md | 31 ++++ .../machine-learning/application_insights.tf | 12 ++ .../machine-learning/config-lab.yml | 25 +++ .../machine-learning/container_registry.tf | 18 ++ .../machine-learning/jumphost.tf | 31 ++++ .../machine-learning/key_vault.tf | 20 ++ .../sample-catalog/machine-learning/locals.tf | 47 +++++ .../log_analytics_workspace.tf | 11 ++ .../machine-learning/machine_learning.tf | 63 +++++++ .../sample-catalog/machine-learning/main.tf | 22 +++ .../machine-learning/manifest.yml | 38 ++++ .../machine-learning/network.tf | 174 ++++++++++++++++++ .../machine-learning/private_dns_zone.tf | 14 ++ .../machine-learning/resource_group.tf | 23 +++ .../machine-learning/storage_account.tf | 49 +++++ .../machine-learning/synapse.tf | 60 ++++++ .../machine-learning/terraform.tfvars.example | 30 +++ .../machine-learning/variables.tf | 91 +++++++++ .../sample-catalog/sandbox/README.md | 4 + .../sample-catalog/sandbox/empty.tf | 1 + .../sample-catalog/sandbox/manifest.yaml | 6 + .../sample-catalog/synapse/README.md | 32 ++++ .../synapse/analysis_services_server.tf | 13 ++ .../sample-catalog/synapse/config-lab.yml | 27 +++ .../sample-catalog/synapse/data_factory.tf | 19 ++ .../sample-catalog/synapse/event_hub.tf | 16 ++ .../sample-catalog/synapse/jumphost.tf | 31 ++++ .../sample-catalog/synapse/key_vault.tf | 18 ++ .../sample-catalog/synapse/locals.tf | 48 +++++ .../synapse/log_analytics_workspace.tf | 11 ++ .../sample-catalog/synapse/main.tf | 24 +++ .../sample-catalog/synapse/manifest.yml | 46 +++++ .../sample-catalog/synapse/network.tf | 39 ++++ .../synapse/private_dns_zone.tf | 14 ++ .../sample-catalog/synapse/resource_group.tf | 23 +++ .../sample-catalog/synapse/storage_account.tf | 24 +++ .../sample-catalog/synapse/synapse.tf | 70 +++++++ .../synapse/terraform.tfvars.example | 6 + .../sample-catalog/synapse/variables.tf | 101 ++++++++++ .../sample-catalog/web-app/README.md | 7 + .../sample-catalog/web-app/manifest.yaml | 14 ++ .../sample-catalog/web-app/web_app.tf | 43 +++++ tools/build-arm.py | 44 +++++ 65 files changed, 2389 insertions(+) create mode 100644 .github/CODE_OF_CONDUCT.md create mode 100644 .github/SECURITY.md create mode 100644 .github/SUPPORT.md create mode 100644 .github/workflows/build_arm.yml create mode 100644 .gitignore create mode 100644 Environments/FunctionApp/azuredeploy.json create mode 100644 Environments/FunctionApp/main.bicep create mode 100644 Environments/FunctionApp/manifest.yaml create mode 100644 Environments/README.md create mode 100644 Environments/Sandbox/azuredeploy.json create mode 100644 Environments/Sandbox/main.bicep create mode 100644 Environments/Sandbox/manifest.yaml create mode 100644 Environments/WebApp/azuredeploy.json create mode 100644 Environments/WebApp/main.bicep create mode 100644 Environments/WebApp/manifest.yaml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 documentation/README.md create mode 100644 documentation/terraform-private-preview/README.md create mode 100644 documentation/terraform-private-preview/sample-catalog/function-app/README.md create mode 100644 documentation/terraform-private-preview/sample-catalog/function-app/function_app.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/function-app/manifest.yaml create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/README.md create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/application_insights.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/config-lab.yml create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/container_registry.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/jumphost.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/key_vault.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/locals.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/log_analytics_workspace.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/machine_learning.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/main.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/manifest.yml create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/network.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/private_dns_zone.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/resource_group.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/storage_account.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/synapse.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/terraform.tfvars.example create mode 100644 documentation/terraform-private-preview/sample-catalog/machine-learning/variables.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/sandbox/README.md create mode 100644 documentation/terraform-private-preview/sample-catalog/sandbox/empty.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/sandbox/manifest.yaml create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/README.md create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/analysis_services_server.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/config-lab.yml create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/data_factory.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/event_hub.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/jumphost.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/key_vault.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/locals.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/log_analytics_workspace.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/main.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/manifest.yml create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/network.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/private_dns_zone.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/resource_group.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/storage_account.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/synapse.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/terraform.tfvars.example create mode 100644 documentation/terraform-private-preview/sample-catalog/synapse/variables.tf create mode 100644 documentation/terraform-private-preview/sample-catalog/web-app/README.md create mode 100644 documentation/terraform-private-preview/sample-catalog/web-app/manifest.yaml create mode 100644 documentation/terraform-private-preview/sample-catalog/web-app/web_app.tf create mode 100644 tools/build-arm.py diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f9ba8cf --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,9 @@ +# Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000..e138ec5 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). + + diff --git a/.github/SUPPORT.md b/.github/SUPPORT.md new file mode 100644 index 0000000..291d4d4 --- /dev/null +++ b/.github/SUPPORT.md @@ -0,0 +1,25 @@ +# TODO: The maintainer of this repo has not yet edited this file + +**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? + +- **No CSS support:** Fill out this template with information about how to file issues and get help. +- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. +- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. + +*Then remove this first heading from this SUPPORT.MD file before publishing your repo.* + +# Support + +## How to file issues and get help + +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE +FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER +CHANNEL. WHERE WILL YOU HELP PEOPLE?**. + +## Microsoft Support Policy + +Support for this **PROJECT or PRODUCT** is limited to the resources listed above. diff --git a/.github/workflows/build_arm.yml b/.github/workflows/build_arm.yml new file mode 100644 index 0000000..01fe7dd --- /dev/null +++ b/.github/workflows/build_arm.yml @@ -0,0 +1,28 @@ +name: Bicep -> ARM + +on: + push: + branches: [main] + paths: + # only run if bicep files changed + - 'Environments/**/*.bicep' + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + contents: write + + steps: + - uses: actions/checkout@v3 + + - name: Build Bicep -> ARM + run: python ./tools/build-arm.py + + - name: Commit changes + run: | + git config --global user.name "${{ github.actor }}" + git config --global user.email "${{ github.actor }}@users.noreply.github.com" + git commit -am "Rebuild ARM templates" + git push diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485a155 --- /dev/null +++ b/.gitignore @@ -0,0 +1,142 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +.DS_Store + +[lL]ocal +.[lL]ocal + +.venv +.env + +local.settings.json +local.parameters.json + +schemas \ No newline at end of file diff --git a/Environments/FunctionApp/azuredeploy.json b/Environments/FunctionApp/azuredeploy.json new file mode 100644 index 0000000..0e7a533 --- /dev/null +++ b/Environments/FunctionApp/azuredeploy.json @@ -0,0 +1,159 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.15.31.15270", + "templateHash": "135257884104161714" + } + }, + "parameters": { + "name": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Name of the Function App" + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location to deploy the environment resources" + } + }, + "supportsHttpsTrafficOnly": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Allows https traffic only to Storage Account and Functions App if set to true." + } + }, + "runtime": { + "type": "string", + "defaultValue": "dotnet-isolated", + "allowedValues": [ + "dotnet", + "dotnet-isolated", + "java", + "node", + "powershell", + "python" + ], + "metadata": { + "description": "The language worker runtime to load in the function app" + } + }, + "tags": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Tags to apply to environment resources" + } + } + }, + "variables": { + "linexFxVersions": { + "dotnet": "DOTNET|6.0", + "dotnet-isolated": "DOTNET-ISOLATED|7.0", + "java": "JAVA|17", + "node": "NODE|18", + "powershell": "POWERSHELL|7.2", + "python": "PYTHON|3.10" + }, + "resourceName": "[if(not(empty(parameters('name'))), replace(parameters('name'), ' ', '-'), format('a{0}', uniqueString(resourceGroup().id)))]", + "storageAcctName": "[take(toLower(replace(replace(variables('resourceName'), '-', ''), '_', '')), 24)]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('resourceName')]", + "kind": "web", + "location": "[parameters('location')]", + "properties": { + "Application_Type": "web" + }, + "tags": "[parameters('tags')]" + }, + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2022-03-01", + "name": "[variables('resourceName')]", + "location": "[parameters('location')]", + "kind": "functionapp,linux", + "sku": { + "tier": "Dynamic", + "name": "Y1" + }, + "properties": { + "reserved": true + }, + "tags": "[parameters('tags')]" + }, + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2022-05-01", + "name": "[variables('storageAcctName')]", + "location": "[parameters('location')]", + "sku": { + "name": "Standard_RAGRS" + }, + "kind": "StorageV2", + "properties": { + "supportsHttpsTrafficOnly": "[parameters('supportsHttpsTrafficOnly')]" + }, + "tags": "[parameters('tags')]" + }, + { + "type": "Microsoft.Web/sites", + "apiVersion": "2022-03-01", + "name": "[variables('resourceName')]", + "kind": "functionapp,linux", + "location": "[parameters('location')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('resourceName'))]", + "httpsOnly": "[parameters('supportsHttpsTrafficOnly')]", + "siteConfig": { + "linuxFxVersion": "[variables('linexFxVersions')[parameters('runtime')]]", + "appSettings": [ + { + "name": "AzureWebJobsStorage", + "value": "[format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', variables('storageAcctName'), listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAcctName')), '2022-05-01').keys[0].value)]" + }, + { + "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", + "value": "[format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', variables('storageAcctName'), listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAcctName')), '2022-05-01').keys[0].value)]" + }, + { + "name": "APPINSIGHTS_INSTRUMENTATIONKEY", + "value": "[reference(resourceId('Microsoft.Insights/components', variables('resourceName')), '2020-02-02').InstrumentationKey]" + }, + { + "name": "AZURE_FUNCTIONS_ENVIRONMENT", + "value": "Production" + }, + { + "name": "FUNCTIONS_EXTENSION_VERSION", + "value": "~4" + }, + { + "name": "FUNCTIONS_WORKER_RUNTIME", + "value": "[parameters('runtime')]" + } + ] + } + }, + "tags": "[parameters('tags')]", + "dependsOn": [ + "[resourceId('Microsoft.Insights/components', variables('resourceName'))]", + "[resourceId('Microsoft.Web/serverfarms', variables('resourceName'))]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAcctName'))]" + ] + } + ] +} \ No newline at end of file diff --git a/Environments/FunctionApp/main.bicep b/Environments/FunctionApp/main.bicep new file mode 100644 index 0000000..607a75d --- /dev/null +++ b/Environments/FunctionApp/main.bicep @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +@description('Name of the Function App') +param name string = '' + +@description('Location to deploy the environment resources') +param location string = resourceGroup().location + +@description('Allows https traffic only to Storage Account and Functions App if set to true.') +param supportsHttpsTrafficOnly bool = true + +@description('The language worker runtime to load in the function app') +@allowed([ + 'dotnet' + 'dotnet-isolated' + 'java' + 'node' + 'powershell' + 'python' +]) +param runtime string = 'dotnet-isolated' + +@description('Tags to apply to environment resources') +param tags object = {} + +var linexFxVersions = { + dotnet: 'DOTNET|6.0' + 'dotnet-isolated': 'DOTNET-ISOLATED|7.0' + java: 'JAVA|17' + node: 'NODE|18' + powershell: 'POWERSHELL|7.2' + python: 'PYTHON|3.10' +} + +var resourceName = !empty(name) ? replace(name, ' ', '-') : 'a${uniqueString(resourceGroup().id)}' + +// storage account names can be no longer than 24 chars +var storageAcctName = take(toLower(replace(replace(resourceName, '-', ''), '_', '')), 24) + +resource appInsights 'Microsoft.Insights/components@2020-02-02' = { + kind: 'web' + name: resourceName + location: location + properties: { + Application_Type: 'web' + } + tags: tags +} + +resource hostingPlan 'Microsoft.Web/serverfarms@2022-03-01' = { + name: resourceName + location: location + kind: 'functionapp,linux' + sku: { + tier: 'Dynamic' + name: 'Y1' + } + properties: { + reserved: true + } + tags: tags +} + +resource storageAccount 'Microsoft.Storage/storageAccounts@2022-05-01' = { + name: storageAcctName + location: location + sku: { + name: 'Standard_RAGRS' + } + kind: 'StorageV2' + properties: { + supportsHttpsTrafficOnly: supportsHttpsTrafficOnly + } + tags: tags +} + +resource functionApp 'Microsoft.Web/sites@2022-03-01' = { + kind: 'functionapp,linux' + name: resourceName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + serverFarmId: hostingPlan.id + httpsOnly: supportsHttpsTrafficOnly + siteConfig: { + linuxFxVersion: linexFxVersions[runtime] + appSettings: [ + { + name: 'AzureWebJobsStorage' + value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' + } + { + name: 'WEBSITE_CONTENTAZUREFILECONNECTIONSTRING' + value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' + } + { + name: 'APPINSIGHTS_INSTRUMENTATIONKEY' + value: appInsights.properties.InstrumentationKey + } + { + name: 'AZURE_FUNCTIONS_ENVIRONMENT' + value: 'Production' + } + { + name: 'FUNCTIONS_EXTENSION_VERSION' + value: '~4' + } + { + name: 'FUNCTIONS_WORKER_RUNTIME' + value: runtime + } + ] + } + } + tags: tags +} diff --git a/Environments/FunctionApp/manifest.yaml b/Environments/FunctionApp/manifest.yaml new file mode 100644 index 0000000..280a456 --- /dev/null +++ b/Environments/FunctionApp/manifest.yaml @@ -0,0 +1,32 @@ +# yaml-language-server: $schema=https://github.com/Azure/deployment-environments/releases/download/2022-11-11-preview/manifest.schema.json +name: FunctionApp +version: 1.0.0 +summary: Azure Function App Environment +description: Deploys an Azure Function App, Storage Account, and Application Insights +runner: ARM +templatePath: azuredeploy.json + +parameters: + - id: name + name: Name + description: 'Name of the Function App.' + type: string + required: true + + - id: supportsHttpsTrafficOnly + name: 'Supports Https Traffic Only' + description: 'Allows https traffic only to Storage Account and Functions App if set to true.' + type: boolean + + - id: runtime + name: Runtime + description: 'The language worker runtime to load in the function app.' + type: string + allowed: + - 'dotnet' + - 'dotnet-isolated' + - 'java' + - 'node' + - 'powershell' + - 'python' + default: 'dotnet-isolated' diff --git a/Environments/README.md b/Environments/README.md new file mode 100644 index 0000000..ae20f97 --- /dev/null +++ b/Environments/README.md @@ -0,0 +1,21 @@ +# Sample Catalog + +The sample Catalog consists of a few catalog items (ARM Template + associated manifest) to help you get started with the service. You can fork the sample catalog and create your catalog to use with DevCenter. + +## Catalog Items + +- [Function App](FunctionApp): Deploys an Azure Function App, Storage Account, and Application Insights +- [Sandbox](Sandbox): Deploys an empty "sandbox" environment +- [Web App](WebApp): Deploys an Azure Web App without a data store + +## ARM and Bicep + +Each catalog item has _main.bicep_ file in addition to the ARM template (_azuredeploy.json_). This is because the ARM templates in this repository are written in [bicep](https://github.com/Azure/bicep) and transpiled to ARM using the [build-arm.py](/tools/build-arm.py) script in the [tools](/tools/) folder. The script simply walks the Environments folder and runs the [`az bicep build`](https://learn.microsoft.com/en-us/cli/azure/bicep?view=azure-cli-latest#az-bicep-build) command on each folder's main.bicep file, and is automatically run via the [build_arm.yml](/.github/workflows/build_arm.yml) workflow any time a bicep file changes. + +**Please note: This is not a requirement for creating catalog item templates. It is done in this repo to make it easier to understand what is being deployed in each template.** + +### What is Bicep? + +Bicep is a Domain Specific Language (DSL) for deploying Azure resources declaratively. It aims to drastically simplify the authoring experience with a cleaner syntax, improved type safety, and better support for modularity and code re-use. Bicep is a transparent abstraction over ARM and ARM templates, which means anything that can be done in an ARM Template can be done in Bicep (outside of temporary [known limitations](https://github.com/Azure/bicep#known-limitations)). All resource types, apiVersions, and properties that are valid in an ARM template are equally valid in Bicep on day one (Note: even if Bicep warns that type information is not available for a resource, it can still be deployed). + +Bicep code is transpiled to standard ARM Template JSON files, which effectively treats the ARM Template as an Intermediate Language (IL). diff --git a/Environments/Sandbox/azuredeploy.json b/Environments/Sandbox/azuredeploy.json new file mode 100644 index 0000000..6d19ce0 --- /dev/null +++ b/Environments/Sandbox/azuredeploy.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.15.31.15270", + "templateHash": "17750672647523699609" + } + }, + "resources": [] +} \ No newline at end of file diff --git a/Environments/Sandbox/main.bicep b/Environments/Sandbox/main.bicep new file mode 100644 index 0000000..fc36ab2 --- /dev/null +++ b/Environments/Sandbox/main.bicep @@ -0,0 +1,2 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. diff --git a/Environments/Sandbox/manifest.yaml b/Environments/Sandbox/manifest.yaml new file mode 100644 index 0000000..43ed242 --- /dev/null +++ b/Environments/Sandbox/manifest.yaml @@ -0,0 +1,7 @@ +# yaml-language-server: $schema=https://github.com/Azure/deployment-environments/releases/download/2022-11-11-preview/manifest.schema.json +name: Sandbox +version: 1.0.0 +summary: Empty sandbox environment +description: Deploys an empty sandbox environment +runner: ARM +templatePath: azuredeploy.json diff --git a/Environments/WebApp/azuredeploy.json b/Environments/WebApp/azuredeploy.json new file mode 100644 index 0000000..6e87396 --- /dev/null +++ b/Environments/WebApp/azuredeploy.json @@ -0,0 +1,65 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.15.31.15270", + "templateHash": "6640258515963112941" + } + }, + "parameters": { + "name": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Name of the Function App" + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location to deploy the environment resources" + } + }, + "tags": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Tags to apply to environment resources" + } + } + }, + "variables": { + "resourceName": "[if(not(empty(parameters('name'))), replace(parameters('name'), ' ', '-'), format('a{0}', uniqueString(resourceGroup().id)))]", + "hostingPlanName": "[format('{0}-hp', variables('resourceName'))]", + "webAppName": "[format('{0}-web', variables('resourceName'))]" + }, + "resources": [ + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2022-03-01", + "name": "[variables('hostingPlanName')]", + "location": "[parameters('location')]", + "sku": { + "tier": "Standard", + "name": "S1" + }, + "tags": "[parameters('tags')]" + }, + { + "type": "Microsoft.Web/sites", + "apiVersion": "2022-03-01", + "name": "[variables('webAppName')]", + "location": "[parameters('location')]", + "properties": { + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]" + }, + "tags": "[parameters('tags')]", + "dependsOn": [ + "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]" + ] + } + ] +} \ No newline at end of file diff --git a/Environments/WebApp/main.bicep b/Environments/WebApp/main.bicep new file mode 100644 index 0000000..6470155 --- /dev/null +++ b/Environments/WebApp/main.bicep @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +@description('Name of the Function App') +param name string = '' + +@description('Location to deploy the environment resources') +param location string = resourceGroup().location + +var resourceName = !empty(name) ? replace(name, ' ', '-') : 'a${uniqueString(resourceGroup().id)}' + +@description('Tags to apply to environment resources') +param tags object = {} + +var hostingPlanName = '${resourceName}-hp' +var webAppName = '${resourceName}-web' + +resource hostingPlan 'Microsoft.Web/serverfarms@2022-03-01' = { + name: hostingPlanName + location: location + sku: { + tier: 'Standard' + name: 'S1' + } + tags: tags +} + +resource webApp 'Microsoft.Web/sites@2022-03-01' = { + name: webAppName + location: location + properties: { + serverFarmId: hostingPlan.id + } + tags: tags +} diff --git a/Environments/WebApp/manifest.yaml b/Environments/WebApp/manifest.yaml new file mode 100644 index 0000000..48b963d --- /dev/null +++ b/Environments/WebApp/manifest.yaml @@ -0,0 +1,14 @@ +# yaml-language-server: $schema=https://github.com/Azure/deployment-environments/releases/download/2022-11-11-preview/manifest.schema.json +name: WebApp +version: 1.0.0 +summary: Azure Web App Environment +description: Deploys an Azure Web App without a data store +runner: ARM +templatePath: azuredeploy.json + +parameters: + - id: name + name: Name + description: 'Name of the Web App.' + type: string + required: true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9e841e7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d77c4f --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +# Azure Deployment Environments Community + +The goal of this repository is to provide sample infrastructure-as-code(IaC) templates that could be used to get started with the [Azure Deployment Environments](https://aka.ms/deploymentenvironments) service. This repository will also be used to provide documentation on capabilities that are in private preview(gated) and engage with customers who are experimenting with the capabilities in private preview. + +Azure Deployment Environments(ADE) empowers development teams to quickly and easily spin-up app infrastructure with project-based templates that establish consistency and best practices while maximizing security, compliance, and cost efficiency. This on-demand access to secure environments accelerates the different stages of the software development lifecycle in a compliant and cost-efficient manner. + +An Environment is a collection of Azure resources on which your application is deployed. For example, to deploy a web application, you might create an environment consisting of an App Service, Key Vault, Cosmos DB and a Storage account. An environment could consist of both Azure PaaS and IaaS resources such as AKS Cluster, App Service, VMs, databases, etc. + +[Environments](https://github.com/Azure/deployment-environments/tree/main/Environments) folder consists of sample templates that you can use to quickly get started with the service. + +[Documentation](https://github.com/Azure/deployment-environments/tree/main/documentation) folder details out capabilities that are currently in private preview and instructions on how to try them out. + +> Note - ADE currently supports [ARM templates](https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/overview) publicly and Terraform support is in private preview. In the near future, will support other IaC tools such as Bicep, Pulumi, etc. + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. + +When you submit a pull request, a CLA bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +## Trademarks + +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft +trademarks or logos is subject to and must follow +[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). +Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. +Any use of third-party trademarks or logos are subject to those third-party's policies. diff --git a/documentation/README.md b/documentation/README.md new file mode 100644 index 0000000..5be336a --- /dev/null +++ b/documentation/README.md @@ -0,0 +1,9 @@ +# Documentation on features in Private Preview + +Documentation folder is used to provide documentation on features/capabilities that are currently in private preview(gated) for [Azure Deployment Environments](https://aka.ms/deploymentenvironments) service. + +Below is the current list of capabilities that are in private preview - + +- Terraform Support: Ability to configure infrastructure using Terraform IaC templates and create environments. Documentation and sample templates to experiment with the Terraform support can be accessed [here](https://github.com/Azure/deployment-environments/tree/main/documentation/terraform-private-preview) + +> Note - If you are interested in learning more about the private preview capabilities or would like to sign-up to gain early-access to new capabilities, please [email us](mailto:adesupport@microsoft.com) diff --git a/documentation/terraform-private-preview/README.md b/documentation/terraform-private-preview/README.md new file mode 100644 index 0000000..c7097a3 --- /dev/null +++ b/documentation/terraform-private-preview/README.md @@ -0,0 +1,47 @@ +# Onboarding Process + +If you didn't sign-up for Terraform early access yet and want to sign-up now, please [sign-up by filling out the form](https://aka.ms/ade-terraform-signup) or [email us](mailto:adesupport@microsoft.com). + +We will need following information to onboard you onto the private preview of the Terraform support: + - Subscription(s) in which Dev Center(s) are or would be created + - Region(s) in which the Dev Center(s) are created + +Once we receive the information, we will onboard your subscriptions and will directly share instructions on how you can try out the private preview. + +# Instructions to leverage Terraform support in Azure Deployment Environments(ADE) + +ADE's support for Terraform templates enables enterprises to configure the app infrastructure using Terraform IaC, securely provide them to dev teams through Catalog and empower dev teams to leverage those Terraform templates when spinning up deployment environments. + +The end-to-end workflows largely remain the same and are detailed out in [official ADE documentation](https://learn.microsoft.com/en-us/azure/deployment-environments/). + +The only difference is how an Environment Definition needs to be configured when using Terraform IaC and is detailed out in the below section - + +We are also providing a [sample catalog](https://github.com/Azure/deployment-environments/tree/main/documentation/terraform-private-preview/sample-catalog) that contains 5 sample environment definitions that were built using Terraform IaC. You may directly leverage these templates to quickly experiment with the end to end workflows. Official documentation details out the step-by-step process to [attach a repo as a catalog](https://learn.microsoft.com/en-us/azure/deployment-environments/how-to-configure-catalog). + + +## How to Configure Terraform Environment Definitions + +Official documentation details out the step by step process that needs to be followed when [configuring an environment definition using ARM](https://learn.microsoft.com/en-us/azure/deployment-environments/configure-environment-definition) and below are a few instructions that needs to be followed when configuring an environment definition using Terraform IaC. + +* Create a Terraform Environment Definition + * Similar to ARM files, you need to point to an entrypoint file with the "templatePath" property. In [this](https://github.com/Azure/deployment-environments/blob/main/documentation/terraform-private-preview/sample-catalog/web-app/manifest.yaml#L5) example, we use web_app.tf. + + * Ensure the "runner" property is set to "Terraform". + + * When configuring your "AzureRM" Terraform provider block, ensure you set the "skip_provider_registration" to true. If you don't do this, Terraform will attempt to register tons of providers, which our PET identity usually doesn't have permissions to do. + +``` +provider "azurerm" { + features {} + + skip_provider_registration = true +} +``` + +* If you are adding Terraform based environment definitons to an existing Catalog, sync the Catalog before attempting to create an environment using the specific environment definition. You will now be able to create an environment directly in the [developer portal](https://learn.microsoft.com/en-us/azure/deployment-environments/quickstart-create-access-environments#create-an-environment) or [through CLI](https://learn.microsoft.com/en-us/azure/deployment-environments/how-to-create-access-environments#create-an-environment) using the Terraform based environment definitions + +# Getting help or providing feedback + +If you are facing any issues or have feedback to share on Terraform support, please create a new issues in [GitHub Issues](https://github.com/Azure/deployment-environments/issues). + +If you have general feedback about the product, please submit the feedback on the [Developer Community](https://developercommunity.visualstudio.com/deploymentenvironments) or by [emailing us directly](mailto:adesupport@microsoft.com). diff --git a/documentation/terraform-private-preview/sample-catalog/function-app/README.md b/documentation/terraform-private-preview/sample-catalog/function-app/README.md new file mode 100644 index 0000000..55b4eec --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/function-app/README.md @@ -0,0 +1,8 @@ +# Simple Function App Deployment using Terraform +This template allows a user to deploy a simple Function App to Azure using Terraform configuration, utilizing the following resources:

+[Application Insights](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/application_insights)

+[Storage Account](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account)

+[App Service Plan](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/app_service)

+[Function App](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/function_app)

+ +For insight on how to utilize this template and deploy an environment, please see [Create an Environment](https://learn.microsoft.com/en-us/azure/deployment-environments/quickstart-create-access-environments) \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/function-app/function_app.tf b/documentation/terraform-private-preview/sample-catalog/function-app/function_app.tf new file mode 100644 index 0000000..f1f253c --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/function-app/function_app.tf @@ -0,0 +1,96 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# This file is hosted @ https://fidalgocli.blob.core.windows.net/cli/function_app.tf + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 2.75" + } + } + + required_version = ">= 1.0.0" +} + +provider "azurerm" { + features {} + + skip_provider_registration = true +} + + +variable "resource_group_name" {} + +variable "resource_name" {} + +variable "location" {} + +variable "os_type" { + default = "linux" +} + +variable "runtime" { + default = "python" +} + +data "azurerm_resource_group" "rg" { + name = var.resource_group_name +} + +resource "azurerm_application_insights" "main" { + name = replace(var.resource_name, " ", "-") + resource_group_name = data.azurerm_resource_group.rg.name + location = var.location + application_type = "web" + tags = {} +} + +resource "azurerm_storage_account" "main" { + name = lower(replace(replace(var.resource_name, " ", ""), "-", "")) + resource_group_name = data.azurerm_resource_group.rg.name + location = var.location + account_tier = "Standard" + account_replication_type = "RAGRS" + account_kind = "StorageV2" + enable_https_traffic_only = true + tags = {} +} + +resource "azurerm_app_service_plan" "main" { + name = replace(var.resource_name, " ", "-") + resource_group_name = data.azurerm_resource_group.rg.name + location = var.location + kind = "FunctionApp" + tags = {} + + reserved = var.os_type == "linux" ? true : false + + sku { + tier = "Dynamic" + size = "Y1" + } +} + +resource "azurerm_function_app" "main" { + name = replace(var.resource_name, " ", "-") + resource_group_name = data.azurerm_resource_group.rg.name + location = var.location + app_service_plan_id = azurerm_app_service_plan.main.id + storage_account_name = azurerm_storage_account.main.name + storage_account_access_key = azurerm_storage_account.main.primary_access_key + https_only = true + version = "~3" + tags = {} + + os_type = var.os_type + app_settings = { + APPINSIGHTS_INSTRUMENTATIONKEY = azurerm_application_insights.main.instrumentation_key + FUNCTIONS_WORKER_RUNTIME = var.runtime + } + + identity { + type = "SystemAssigned" + } +} diff --git a/documentation/terraform-private-preview/sample-catalog/function-app/manifest.yaml b/documentation/terraform-private-preview/sample-catalog/function-app/manifest.yaml new file mode 100644 index 0000000..2c18e67 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/function-app/manifest.yaml @@ -0,0 +1,22 @@ +name: FunctionAppName +summary: Creates a function app +description: Deploys an environment using Terraform containing a Function App +runner: Terraform +templatePath: function_app.tf +parameters: +- id: "resource_name" + name: "Resource Name" + required: true + type: "string" +- id: "location" + name: "Location" + required: true + type: "string" +- id: "os_type" + name: "Operating System" + default: "linux" + type: "string" +- id: "runtime" + name: "Function Runtime" + default: "python" + type: "string" diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/README.md b/documentation/terraform-private-preview/sample-catalog/machine-learning/README.md new file mode 100644 index 0000000..34faa88 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/README.md @@ -0,0 +1,31 @@ +# Azure Data Labs - Azure Machine Learning + +This template repository contains all templates to deploy a (secure) Azure Machine Learning environment. This template offers some optional modules which can be enabled/disabled to support some of architectures below. This template was provided by Azure Data Labs and utilized their modules, and was tweaked to provide ADE support. + +### What will be deployed? + +By navigating through the deployment steps, you will deploy the following resources in an Azure subscription: + +| Module | Default? | Comment | +| - | - | - | +| [Storage Account](./infra/storage_account.tf) | Yes | ADLS Gen2. Includes `blob` and `file` private endpoints (PEs) if `enable_private_endpoints` +| [Key Vault](./infra/key_vault.tf) | Yes | Includes `vault` PE if `enable_private_endpoints` +| [Container Registry](./infra/container_registry.tf) | Yes | Includes `registry` PE if `enable_private_endpoints` +| [Application Insights](./infra/application_insights.tf) | Yes | n/a +| [Virtual Network](./infra/network.tf) | Yes | 10.0.0.0/16 by default +| [Subnet](./infra/network.tf) | Yes | Includes three subnets `default` (10.0.1.0/24), `compute` (10.0.2.0/24) and `bastion` (10.0.10.0/27) +| [Synapse Workspace](./infra/synapse.tf) | No | Enable by `enable_synapse_workspace` +| [Synapse Spark Pool](./infra/synapse.tf) | No | Enable by `enable_synapse_spark_pool` +| [Jumphost (Windows)](./infra/jumphost.tf) | No | Includes Bastion, enable by `enable_jumphost` + +### Deployment + +- **Enabling / disabling secure deployment**: to enable/disable secure deployment, change `enable_private_endpoints` in config-lab.yml. +- **Enabling / disabling resources**: to enable/disable optional modules, change `enable_{optional-module}` flag in config-lab.yml. +- **Deploying the template**: to deploy this template, see [Create an Environment](https://learn.microsoft.com/en-us/azure/deployment-environments/quickstart-create-access-environments). + +### Related Architectures + +- 📘 [Secure Azure Machine Learning Service (AzureML) Environment](https://techcommunity.microsoft.com/t5/fasttrack-for-azure/secure-azure-machine-learning-service-azureml-environment/ba-p/3162297) +- 📘 [Azure Machine Learning Architecture](https://docs.microsoft.com/en-us/azure/architecture/solution-ideas/articles/azure-machine-learning-solution-architecture) +- 📘 [Enterprise Security and Compliance for Azure Machine Learning](https://techcommunity.microsoft.com/t5/ai-machine-learning-blog/enterprise-security-and-compliance-for-azure-machine-learning/ba-p/3484858) diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/application_insights.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/application_insights.tf new file mode 100644 index 0000000..6f6aece --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/application_insights.tf @@ -0,0 +1,12 @@ +# Appplication insights + +module "application_insights" { + source = "github.com/Azure/azure-data-labs-modules/terraform/application-insights" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + application_type = "web" + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/config-lab.yml b/documentation/terraform-private-preview/sample-catalog/machine-learning/config-lab.yml new file mode 100644 index 0000000..5e66635 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/config-lab.yml @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +variables: + #global - make null to use vars + prefix: null + postfix: null + location: null + # Feature flags + enable_private_endpoints: "false" + enable_jumphost: "false" + enable_synapse_workspace: "false" + enable_machine_learning_compute_cluster: "false" + enable_machine_learning_synapse_spark: "false" + + # Enable ADE based deployment + enable_ade_deployment: "true" + + # Terraform remote storage + terraform_st_resource_group: rg-adl-terraform-state + terraform_st_storage_account: stadlterraformstate + + # Internals - do not touch + az_cli_version: 2.45.0 + terraform_st_container_name: default \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/container_registry.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/container_registry.tf new file mode 100644 index 0000000..15bbc9a --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/container_registry.tf @@ -0,0 +1,18 @@ +# Container registry + +module "container_registry" { + source = "github.com/Azure/azure-data-labs-modules/terraform/container-registry" + + basename = local.safe_basename + resource_group_name = local.resource_group_name + location = local.location + sku = "Premium" + admin_enabled = true + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.azurecr.io"].id] : null + + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/jumphost.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/jumphost.tf new file mode 100644 index 0000000..e0dbb6c --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/jumphost.tf @@ -0,0 +1,31 @@ +# Bastion + +module "bastion" { + source = "github.com/Azure/azure-data-labs-modules/terraform/bastion-host" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + subnet_id = local.enable_jumphost ? module.subnet_bastion[0].id : null + + module_enabled = local.enable_jumphost + + tags = local.tags +} + +# Virtual machine + +module "virtual_machine_jumphost" { + source = "github.com/Azure/azure-data-labs-modules/terraform/virtual-machine" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + subnet_id = local.enable_jumphost ? module.subnet_default[0].id : null + jumphost_username = var.jumphost_username + jumphost_password = var.jumphost_password + + module_enabled = local.enable_jumphost + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/key_vault.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/key_vault.tf new file mode 100644 index 0000000..5833231 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/key_vault.tf @@ -0,0 +1,20 @@ +# Key Vault +#ADE only gives ownership over RG, so we don't have permission to purge the KV from soft delete +#If you'd like to purge your KV, give your Project Environment Type Identity Owner permissions over the subscription, +# and then set purge_protection_enabled to false +module "key_vault" { + source = "github.com/Azure/azure-data-labs-modules/terraform/key-vault" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + sku_name = "premium" + purge_protection_enabled = false + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.vaultcore.azure.net"].id] : null + + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/locals.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/locals.tf new file mode 100644 index 0000000..4b9cf7d --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/locals.tf @@ -0,0 +1,47 @@ +locals { + tags = { + Owner = "Azure Data Labs" + Project = "Azure Data Labs" + Environment = "dev" + Toolkit = "Terraform" + Template = "data-science-aml" + Name = "${var.prefix}" + } + + dns_zones = [ + "privatelink.sql.azuresynapse.net", + "privatelink.dev.azuresynapse.net", + "privatelink.azuresynapse.net", + "privatelink.blob.core.windows.net", + "privatelink.file.core.windows.net", + "privatelink.dfs.core.windows.net", + "privatelink.vaultcore.azure.net", + "privatelink.azurecr.io", + "privatelink.servicebus.windows.net", + "privatelink.api.azureml.ms", + "privatelink.notebooks.azure.net" + ] + + safe_prefix = replace(local.prefix, "-", "") + safe_postfix = replace(local.postfix, "-", "") + + basename = "${local.prefix}-${local.postfix}" + safe_basename = "${local.safe_prefix}${local.safe_postfix}" + + # Configuration + + config = yamldecode(file("config-lab.yml")) + + resource_group_name = local.config.variables.enable_ade_deployment == "true" ? var.resource_group_name : length(module.resource_group) > 0 ? module.resource_group[0].name : "" + location = local.config.variables.location != null ? local.config.variables.location : var.location + prefix = local.config.variables.prefix != null ? local.config.variables.prefix : var.prefix + postfix = local.config.variables.postfix != null ? local.config.variables.postfix : var.postfix + + enable_private_endpoints = local.config.variables.enable_private_endpoints != null ? local.config.variables.enable_private_endpoints : var.enable_private_endpoints + enable_jumphost = local.config.variables.enable_jumphost != null ? local.config.variables.enable_jumphost : var.enable_jumphost + enable_synapse_workspace = local.config.variables.enable_synapse_workspace != null ? local.config.variables.enable_synapse_workspace : var.enable_synapse_workspace + enable_machine_learning_compute_cluster = local.config.variables.enable_machine_learning_compute_cluster != null ? local.config.variables.enable_machine_learning_compute_cluster : var.enable_machine_learning_compute_cluster + enable_machine_learning_synapse_spark = local.config.variables.enable_machine_learning_synapse_spark != null ? local.config.variables.enable_machine_learning_synapse_spark : var.enable_machine_learning_synapse_spark + + enable_ade_deployment = local.config.variables.enable_ade_deployment +} diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/log_analytics_workspace.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/log_analytics_workspace.tf new file mode 100644 index 0000000..a50be22 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/log_analytics_workspace.tf @@ -0,0 +1,11 @@ +# Log Analytics workspace + +module "log_analytics_workspace" { + source = "github.com/Azure/azure-data-labs-modules/terraform/log-analytics/log-analytics-workspace" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/machine_learning.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/machine_learning.tf new file mode 100644 index 0000000..2a1255f --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/machine_learning.tf @@ -0,0 +1,63 @@ +# Machine learning workspace + +module "machine_learning_workspace" { + source = "github.com/Azure/azure-data-labs-modules/terraform/machine-learning/machine-learning-workspace" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + storage_account_id = module.storage_account_mlw.id + key_vault_id = module.key_vault.id + application_insights_id = module.application_insights.id + container_registry_id = module.container_registry.id + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids = local.enable_private_endpoints ? [ + module.private_dns_zones[0].list["privatelink.api.azureml.ms"].id, + module.private_dns_zones[0].list["privatelink.notebooks.azure.net"].id + ] : null + + public_network_access_enabled = true + image_build_compute_name = "image-builder" + + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} + +# Machine learning Synapse Spark + +module "machine_learning_synapse_spark" { + source = "github.com/Azure/azure-data-labs-modules/terraform/machine-learning/machine-learning-synapse-spark" + + basename = local.postfix + location = local.location + machine_learning_workspace_id = module.machine_learning_workspace.id + synapse_spark_pool_id = module.synapse_spark_pool.id + + module_enabled = local.enable_machine_learning_synapse_spark +} + +# Machine learning compute clusters + +module "machine_learning_compute_cluster_image-builder" { + source = "github.com/Azure/azure-data-labs-modules/terraform/machine-learning/machine-learning-compute-cluster" + + basename = "image-builder" + location = local.location + subnet_id = local.enable_private_endpoints ? module.subnet_compute[0].id : null + machine_learning_workspace_id = module.machine_learning_workspace.id + + module_enabled = false +} + +module "machine_learning_compute_cluster_cpu_cluster" { + source = "github.com/Azure/azure-data-labs-modules/terraform/machine-learning/machine-learning-compute-cluster" + + basename = "cpu-cluster" + location = local.location + subnet_id = local.enable_private_endpoints ? module.subnet_compute[0].id : null + machine_learning_workspace_id = module.machine_learning_workspace.id + + module_enabled = local.enable_machine_learning_compute_cluster +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/main.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/main.tf new file mode 100644 index 0000000..3d3bcea --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/main.tf @@ -0,0 +1,22 @@ +terraform { + # backend "azurerm" {} + required_providers { + azurerm = { + version = "= 3.53.0" + } + } +} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = false + } + } +} + +data "azurerm_client_config" "current" {} + +data "http" "ip" { + url = "https://ifconfig.me" +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/manifest.yml b/documentation/terraform-private-preview/sample-catalog/machine-learning/manifest.yml new file mode 100644 index 0000000..42beea9 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/manifest.yml @@ -0,0 +1,38 @@ +name: AzureMachineLearning +summary: Terraform example for Azure Data Labs - Data Science with Azure Machine Learning +description: Deploys an environment using Terraform containing Azure ML resources. +runner: Terraform +templatePath: main.tf +parameters: +- id: "prefix" + name: "Prefix" + type: "string" + required: true +- id: "postfix" + name: "Postfix" + type: "string" + required: true +- id: "location" + name: "Location" + default: "centralus" + type: "string" +- id: "enable_private_endpoints" + name: "enable_private_endpoints" + default: "false" + type: "string" +- id: "enable_jumphost" + name: "enable_jumphost" + default: "false" + type: "string" +- id: "enable_synapse_workspace" + name: "enable_synapse_workspace" + default: "false" + type: "string" +- id: "enable_machine_learning_compute_cluster" + name: "enable_machine_learning_compute_cluster" + default: "false" + type: "string" +- id: "enable_machine_learning_synapse_spark" + name: "enable_machine_learning_synapse_spark" + default: "false" + type: "string" diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/network.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/network.tf new file mode 100644 index 0000000..9005824 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/network.tf @@ -0,0 +1,174 @@ +# Virtual network + +module "virtual_network" { + source = "github.com/Azure/azure-data-labs-modules/terraform/virtual-network" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + address_space = ["10.0.0.0/16"] + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) + + tags = local.tags +} +# Subnets + +module "subnet_default" { + source = "github.com/Azure/azure-data-labs-modules/terraform/subnet" + + name = "snet-${local.prefix}-${local.postfix}-default" + resource_group_name = local.resource_group_name + vnet_name = module.virtual_network[0].name + address_prefixes = ["10.0.1.0/24"] + private_endpoint_network_policies_enabled = true + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +module "subnet_bastion" { + source = "github.com/Azure/azure-data-labs-modules/terraform/subnet" + + name = "AzureBastionSubnet" + resource_group_name = local.resource_group_name + vnet_name = module.virtual_network[0].name + address_prefixes = ["10.0.10.0/27"] + + count = local.enable_jumphost ? 1 : 0 +} + +module "subnet_compute" { + source = "github.com/Azure/azure-data-labs-modules/terraform/subnet" + + name = "snet-${local.prefix}-${local.postfix}-compute" + resource_group_name = local.resource_group_name + vnet_name = module.virtual_network[0].name + address_prefixes = ["10.0.2.0/24"] + private_endpoint_network_policies_enabled = true + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +# Network security groups + +module "network_security_group_training" { + source = "github.com/Azure/azure-data-labs-modules/terraform/network-security-group" + + basename = "nsg-${local.basename}-01" + resource_group_name = local.resource_group_name + location = local.location + + tags = local.tags + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +# Network security rules + +module "network_security_rule_training_batchnodemanagement" { + source = "github.com/Azure/azure-data-labs-modules/terraform/network-security-rule" + + name = "BatchNodeManagement" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "29876-29877" + source_address_prefix = "BatchNodeManagement" + destination_address_prefix = "*" + + resource_group_name = local.resource_group_name + network_security_group_name = module.network_security_group_training[0].name + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +module "network_security_rule_training_azuremachinelearning" { + source = "github.com/Azure/azure-data-labs-modules/terraform/network-security-rule" + + name = "AzureMachineLearning" + priority = 110 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "44224" + source_address_prefix = "AzureMachineLearning" + destination_address_prefix = "*" + + resource_group_name = local.resource_group_name + network_security_group_name = module.network_security_group_training[0].name + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +# NSG associations + +module "subnet_network_security_group_association_training" { + source = "github.com/Azure/azure-data-labs-modules/terraform/subnet-network-security-group-association" + + subnet_id = module.subnet_compute[0].id + network_security_group_id = module.network_security_group_training[0].id + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +# User Defined Routes + +module "route_table_training" { + source = "github.com/Azure/azure-data-labs-modules/terraform/route-table" + + basename = local.basename + location = local.location + resource_group_name = local.resource_group_name + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +module "route_training_internet" { + source = "github.com/Azure/azure-data-labs-modules/terraform/route" + + name = "Internet" + resource_group_name = local.resource_group_name + route_table_name = module.route_table_training[0].name + address_prefix = "0.0.0.0/0" + next_hop_type = "Internet" + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +module "route_training_azureml" { + source = "github.com/Azure/azure-data-labs-modules/terraform/route" + + name = "AzureMLRoute" + resource_group_name = local.resource_group_name + route_table_name = module.route_table_training[0].name + address_prefix = "AzureMachineLearning" + next_hop_type = "Internet" + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +module "route_training_batch" { + source = "github.com/Azure/azure-data-labs-modules/terraform/route" + + name = "BatchRoute" + resource_group_name = local.resource_group_name + route_table_name = module.route_table_training[0].name + address_prefix = "BatchNodeManagement" + next_hop_type = "Internet" + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +# UDR associations + +module "subnet_route_table_association_training" { + source = "github.com/Azure/azure-data-labs-modules/terraform/subnet-route-table-association" + + subnet_id = module.subnet_compute[0].id + route_table_id = module.route_table_training[0].id + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/private_dns_zone.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/private_dns_zone.tf new file mode 100644 index 0000000..d0fd8cd --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/private_dns_zone.tf @@ -0,0 +1,14 @@ +# Private DNS zones + +module "private_dns_zones" { + source = "github.com/Azure/azure-data-labs-modules/terraform/private-dns-zone" + + resource_group_name = local.enable_private_endpoints ? module.resource_group_global_dns[0].name : "none" + + vnet_id = local.enable_private_endpoints ? module.virtual_network[0].id : null + dns_zones = local.dns_zones + + count = local.enable_private_endpoints ? 1 : 0 + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/resource_group.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/resource_group.tf new file mode 100644 index 0000000..2651a41 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/resource_group.tf @@ -0,0 +1,23 @@ +# Resource group + +module "resource_group" { + source = "github.com/Azure/azure-data-labs-modules/terraform/resource-group" + + basename = local.basename + location = local.location + + count = local.enable_ade_deployment ? 0 : 1 + + tags = local.tags +} + +module "resource_group_global_dns" { + source = "github.com/Azure/azure-data-labs-modules/terraform/resource-group" + + basename = "${local.basename}-global-dns" + location = local.location + + count = local.enable_private_endpoints ? 1 : 0 + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/storage_account.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/storage_account.tf new file mode 100644 index 0000000..12f586f --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/storage_account.tf @@ -0,0 +1,49 @@ +# Storage Account + +module "storage_account_mlw" { + source = "github.com/Azure/azure-data-labs-modules/terraform/storage-account" + + basename = "${local.safe_basename}mlw" + resource_group_name = local.resource_group_name + location = local.location + account_tier = "Standard" + + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids_blob = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.blob.core.windows.net"].id] : [] + private_dns_zone_ids_file = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.file.core.windows.net"].id] : [] + + hns_enabled = false + firewall_default_action = "Allow" + firewall_ip_rules = [data.http.ip.body] + firewall_bypass = ["AzureServices"] + + module_enabled = true + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} + +module "storage_account_syn" { + #source = "github.com/Azure/azure-data-labs-modules/terraform/storage-account" + source = "git::https://github.com/Azure/azure-data-labs-modules.git//terraform/storage-account?ref=v1.5.0-beta" + + basename = "${local.safe_basename}syn" + resource_group_name = local.resource_group_name + location = local.location + account_tier = "Standard" + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids_blob = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.blob.core.windows.net"].id] : [] + private_dns_zone_ids_dfs = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.dfs.core.windows.net"].id] : [] + + hns_enabled = true + firewall_default_action = "Allow" + firewall_ip_rules = [data.http.ip.body] + firewall_bypass = ["AzureServices"] + + module_enabled = local.enable_synapse_workspace + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/synapse.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/synapse.tf new file mode 100644 index 0000000..453ae63 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/synapse.tf @@ -0,0 +1,60 @@ +# Synapse workspace + +module "synapse_workspace" { + #source = "github.com/Azure/azure-data-labs-modules/terraform/synapse/synapse-workspace" + source = "git::https://github.com/Azure/azure-data-labs-modules.git//terraform/synapse/synapse-workspace?ref=v1.5.0-beta" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + adls_id = local.enable_synapse_workspace ? module.storage_account_syn.adls_id : "null" + storage_account_id = local.enable_synapse_workspace ? module.storage_account_syn.id : "null" + storage_account_name = local.enable_synapse_workspace ? module.storage_account_syn.name : "null" + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids_sql = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.sql.azuresynapse.net"].id] : null + private_dns_zone_ids_dev = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.dev.azuresynapse.net"].id] : null + + synadmin_username = var.synadmin_username + synadmin_password = var.synadmin_password + + aad_login = { + name = var.aad_login.name + object_id = var.aad_login.object_id + tenant_id = var.aad_login.tenant_id + } + + module_enabled = local.enable_synapse_workspace + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} + +# Synapse Private Link Hub + +module "synapse_private_link_hub" { + #source = "github.com/Azure/azure-data-labs-modules/terraform/synapse/synapse-private-link-hub" + source = "git::https://github.com/Azure/azure-data-labs-modules.git//terraform/synapse/synapse-private-link-hub?ref=v1.5.0-beta" + + basename = local.safe_basename + resource_group_name = local.resource_group_name + location = local.location + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.azuresynapse.net"].id] : null + + module_enabled = local.enable_synapse_workspace && local.enable_private_endpoints + + tags = local.tags +} + +# Synapse Spark pool + +module "synapse_spark_pool" { + source = "github.com/Azure/azure-data-labs-modules.git//terraform/synapse/synapse-spark-pool" + + basename = local.safe_basename + synapse_workspace_id = module.synapse_workspace.id + + module_enabled = local.enable_machine_learning_synapse_spark +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/terraform.tfvars.example b/documentation/terraform-private-preview/sample-catalog/machine-learning/terraform.tfvars.example new file mode 100644 index 0000000..95b5ed4 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/terraform.tfvars.example @@ -0,0 +1,30 @@ +# Resource name prefix +prefix = "data-02" + +# Deployment location +location = "North Europe" + +# Postfix +postfix = "101" + +# Jumpbox login credentials +jumphost_username = "azureuser" +jumphost_password = "ThisIsNotVerySecure!" + +# Synapse SQL admin credentials +synadmin_username = "sqladminuser" +synadmin_password = "ThisIsNotVerySecure!" + +# Synapse AAD login credentials +aad_login = { + name = "azureuser@contoso.com" + object_id = "00000000-0000-0000-0000-000000000000" + tenant_id = "00000000-0000-0000-0000-000000000000" +} + +# Feature flags +enable_private_endpoints = "false" +enable_jumphost = "true" +enable_synapse_workspace = "false" +enable_machine_learning_compute_cluster = "true" +enable_machine_learning_synapse_spark = "false" \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/machine-learning/variables.tf b/documentation/terraform-private-preview/sample-catalog/machine-learning/variables.tf new file mode 100644 index 0000000..30b92bb --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/machine-learning/variables.tf @@ -0,0 +1,91 @@ +variable "location" { + type = string + description = "Location of the resource group and modules" + default = "Central US" +} + +variable "prefix" { + type = string + description = "Prefix for module names" +} + +variable "postfix" { + type = string + description = "Postfix for module names" +} + +variable "aad_login" { + description = "AAD login" + type = object({ + name = string + object_id = string + tenant_id = string + }) + default = { + name = "AzureAD Admin" + object_id = "00000000-0000-0000-0000-000000000000" + tenant_id = "00000000-0000-0000-0000-000000000000" + } +} + +variable "jumphost_username" { + type = string + description = "VM username" + default = "azureuser" +} + +variable "jumphost_password" { + type = string + description = "VM password" + default = "ThisIsNotVerySecure!" +} + +variable "synadmin_username" { + type = string + description = "The Login Name of the SQL administrator" + default = "sqladminuser" +} + +variable "synadmin_password" { + type = string + description = "The Password associated with the sql_administrator_login for the SQL administrator" + default = "ThisIsNotVerySecure!" +} + +# Feature flags + +variable "enable_private_endpoints" { + type = bool + description = "Is secure enabled?" + default = false +} + +variable "enable_jumphost" { + type = bool + description = "Variable to enable or disable the module" + default = false +} + +variable "enable_synapse_workspace" { + type = bool + description = "Variable to enable or disable the module" + default = false +} + +variable "enable_machine_learning_synapse_spark" { + type = bool + description = "Variable to enable or disable the module" + default = false +} + +variable "enable_machine_learning_compute_cluster" { + type = bool + description = "Variable to enable or disable the module" + default = true +} + +variable "resource_group_name" { + type = string + description = "Variable for ADE based deployment" + default = "rg-j6de-001342" +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/sandbox/README.md b/documentation/terraform-private-preview/sample-catalog/sandbox/README.md new file mode 100644 index 0000000..e2cae1c --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/sandbox/README.md @@ -0,0 +1,4 @@ +# Simple Sandbox Environment using Terraform +This blank Terraform template and manifest file will allow a user to deploy an empty resource group managed under ADE. A note to be made about this template, is that any resources created within an environment created under this template will get deleted if the environment is deleted, but their state would not be tracked in a Terraform state file, since they are not configured at time of deployment. + +For insight on how to utilize this template and deploy an environment, please see [Create an Environment](https://learn.microsoft.com/en-us/azure/deployment-environments/quickstart-create-access-environments) \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/sandbox/empty.tf b/documentation/terraform-private-preview/sample-catalog/sandbox/empty.tf new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/sandbox/empty.tf @@ -0,0 +1 @@ + diff --git a/documentation/terraform-private-preview/sample-catalog/sandbox/manifest.yaml b/documentation/terraform-private-preview/sample-catalog/sandbox/manifest.yaml new file mode 100644 index 0000000..dabe65f --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/sandbox/manifest.yaml @@ -0,0 +1,6 @@ +name: Sandbox +version: 1.0.0 +summary: Empty sandbox environment +description: Deploys an empty sandbox environment +runner: Terraform +templatePath: empty.tf diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/README.md b/documentation/terraform-private-preview/sample-catalog/synapse/README.md new file mode 100644 index 0000000..ab973db --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/README.md @@ -0,0 +1,32 @@ +# Azure Data Labs - Azure Synapse Analytics + +This template repository contains all templates to deploy a (secure) **Azure Synapse Analytics environment**. This template offers some optional modules which can be enabled/disabled to support some of architectures below. This template was developed by Azure Data Labs and utilized its' modules, and was tweaked to provide support for ADE. + +### What will be deployed? + +By navigating through the deployment steps, you will deploy the following resources in an Azure subscription: + +| Module | Default? | Comment | +| - | - | - | +| [Storage Account](./infra/storage_account.tf) | Yes | ADLS Gen2. Includes `blob` and `dfs` private endpoints (PEs) if `enable_private_endpoints` +| [Key Vault](./infra/key_vault.tf) | Yes | Includes `vault` PE if `enable_private_endpoints` +| [Virtual Network](./infra/network.tf) | Yes | 10.0.0.0/16 by default +| [Subnet](./infra/network.tf) | Yes | Includes two subnets `default` (10.0.1.0/24) and `bastion` (10.0.10.0/27) +| [Synapse Workspace](./infra//synapse.tf) | Yes | Includes `sql` and `dev` PEs + Private Link Hub if `enable_private_endpoints` +| [Synapse Spark Pool](./infra/synapse.tf) | No | Enable by `enable_synapse_spark_pool` +| [Synapse SQL Pool](./infra/synapse.tf) | No | Enable by `enable_synapse_sql_pool` +| [Data Factory](./infra/data_factory.tf) | No | Enable by `enable_data_factory`. Includes `df` and `portal` PEs if `enable_private_endpoints` +| [Event Hub](./infra/event_hub.tf) | No | Enable by `enable_event_hub`. Includes `namespace` PE if `enable_private_endpoints` +| [Analysis Services Server](./infra/analysis_services_server.tf) | No | Enable by `enable_analysis_services_server` +| [Jumphost (Windows)](./infra/jumphost.tf) | No | Includes Bastion, enable by `enable_jumphost` + +### Deployment + +- **Enabling / disabling secure deployment**: to enable/disable secure deployment, change `enable_private_endpoints` in config-lab.yml. +- **Enabling / disabling resources**: to enable/disable optional modules, change `enable_{optional-module}` flag in config-lab.yml. +- **Deploying the template**: to deploy this template, see [Deploy an Environment](https://learn.microsoft.com/en-us/azure/deployment-environments/quickstart-create-access-environments). + +### Related Architectures + +- 📘 [Enterprise business intelligence](https://learn.microsoft.com/en-us/azure/architecture/example-scenario/analytics/enterprise-bi-synapse) +- 📘 [Near real-time lakehouse data processing](https://learn.microsoft.com/en-us/azure/architecture/example-scenario/data/real-time-lakehouse-data-processing) diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/analysis_services_server.tf b/documentation/terraform-private-preview/sample-catalog/synapse/analysis_services_server.tf new file mode 100644 index 0000000..f046ee2 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/analysis_services_server.tf @@ -0,0 +1,13 @@ +# Analysis Services + +module "analysis_services" { + source = "github.com/Azure/azure-data-labs-modules/terraform/analysis-services-server" + + basename = local.safe_basename + resource_group_name = local.resource_group_name + location = local.location + + module_enabled = local.enable_analysis_services_server + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/config-lab.yml b/documentation/terraform-private-preview/sample-catalog/synapse/config-lab.yml new file mode 100644 index 0000000..41abef3 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/config-lab.yml @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +variables: + #global - make null to use vars + prefix: null + postfix: null + location: null + # Feature flags + enable_private_endpoints: "false" + enable_jumphost: "false" + enable_analysis_services_server: "false" + enable_data_factory: "false" + enable_synapse_sql_pool: "false" + enable_synapse_spark_pool: "false" + enable_event_hub: "false" + + # Enable ADE based deployment + enable_ade_deployment: "true" + + # Terraform remote storage + terraform_st_resource_group: rg-adl-terraform-state + terraform_st_storage_account: stadlterraformstate + + # Internals - do not touch + az_cli_version: 2.45.0 + terraform_st_container_name: default \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/data_factory.tf b/documentation/terraform-private-preview/sample-catalog/synapse/data_factory.tf new file mode 100644 index 0000000..04420f3 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/data_factory.tf @@ -0,0 +1,19 @@ +# Data Factory + +module "data_factory" { + #source = "github.com/Azure/azure-data-labs-modules/terraform/data-factory/data-factory" + source = "git::https://github.com/Azure/azure-data-labs-modules.git//terraform/data-factory/data-factory?ref=v1.5.0-beta" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids_df = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.datafactory.azure.net"].id] : null + private_dns_zone_ids_portal = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.adf.azure.com"].id] : null + + module_enabled = local.enable_data_factory + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/event_hub.tf b/documentation/terraform-private-preview/sample-catalog/synapse/event_hub.tf new file mode 100644 index 0000000..b49ffa1 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/event_hub.tf @@ -0,0 +1,16 @@ +module "event_hubs_namespace" { + #source = "github.com/Azure/azure-data-labs-modules/terraform/event-hubs/event-hubs-namespace" + source = "git::https://github.com/Azure/azure-data-labs-modules.git//terraform/event-hubs/event-hubs-namespace?ref=v1.5.0-beta" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.servicebus.windows.net"].id] : null + + module_enabled = local.enable_event_hub + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/jumphost.tf b/documentation/terraform-private-preview/sample-catalog/synapse/jumphost.tf new file mode 100644 index 0000000..e0dbb6c --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/jumphost.tf @@ -0,0 +1,31 @@ +# Bastion + +module "bastion" { + source = "github.com/Azure/azure-data-labs-modules/terraform/bastion-host" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + subnet_id = local.enable_jumphost ? module.subnet_bastion[0].id : null + + module_enabled = local.enable_jumphost + + tags = local.tags +} + +# Virtual machine + +module "virtual_machine_jumphost" { + source = "github.com/Azure/azure-data-labs-modules/terraform/virtual-machine" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + subnet_id = local.enable_jumphost ? module.subnet_default[0].id : null + jumphost_username = var.jumphost_username + jumphost_password = var.jumphost_password + + module_enabled = local.enable_jumphost + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/key_vault.tf b/documentation/terraform-private-preview/sample-catalog/synapse/key_vault.tf new file mode 100644 index 0000000..8a22243 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/key_vault.tf @@ -0,0 +1,18 @@ +# Key Vault + +module "key_vault" { + source = "github.com/Azure/azure-data-labs-modules/terraform/key-vault" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + sku_name = "premium" + purge_protection_enabled = false + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.vaultcore.azure.net"].id] : null + + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/locals.tf b/documentation/terraform-private-preview/sample-catalog/synapse/locals.tf new file mode 100644 index 0000000..e61c60d --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/locals.tf @@ -0,0 +1,48 @@ +locals { + tags = { + Owner = "Azure Data Labs" + Project = "Azure Data Labs" + Environment = "dev" + Toolkit = "Terraform" + Template = "data-insights-syn" + Name = "${var.prefix}" + } + + dns_zones = [ + "privatelink.sql.azuresynapse.net", + "privatelink.dev.azuresynapse.net", + "privatelink.azuresynapse.net", + "privatelink.blob.core.windows.net", + "privatelink.dfs.core.windows.net", + "privatelink.vaultcore.azure.net", + "privatelink.servicebus.windows.net", + "privatelink.datafactory.azure.net", + "privatelink.adf.azure.com", + "privatelink.database.windows.net" + ] + + safe_prefix = replace(local.prefix, "-", "") + safe_postfix = replace(local.postfix, "-", "") + + basename = "${local.prefix}-${local.postfix}" + safe_basename = "${local.safe_prefix}${local.safe_postfix}" + + # Configuration + + config = yamldecode(file("config-lab.yml")) + + resource_group_name = local.config.variables.enable_ade_deployment == "true" ? var.resource_group_name : length(module.resource_group) > 0 ? module.resource_group[0].name : "" + location = local.config.variables.location != null ? local.config.variables.location : var.location + prefix = local.config.variables.prefix != null ? local.config.variables.prefix : var.prefix + postfix = local.config.variables.postfix != null ? local.config.variables.postfix : var.postfix + + enable_private_endpoints = local.config.variables.enable_private_endpoints != null ? local.config.variables.enable_private_endpoints : var.enable_private_endpoints + enable_jumphost = local.config.variables.enable_jumphost != null ? local.config.variables.enable_jumphost : var.enable_jumphost + enable_synapse_sql_pool = local.config.variables.enable_synapse_sql_pool != null ? local.config.variables.enable_synapse_sql_pool : var.enable_synapse_sql_pool + enable_synapse_spark_pool = local.config.variables.enable_synapse_spark_pool != null ? local.config.variables.enable_synapse_spark_pool : var.enable_synapse_spark_pool + enable_data_factory = local.config.variables.enable_data_factory != null ? local.config.variables.enable_data_factory : var.enable_data_factory + enable_analysis_services_server = local.config.variables.enable_analysis_services_server != null ? local.config.variables.enable_analysis_services_server : var.enable_analysis_services_server + enable_event_hub = local.config.variables.enable_event_hub != null ? local.config.variables.enable_event_hub : var.enable_event_hub + + enable_ade_deployment = local.config.variables.enable_ade_deployment +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/log_analytics_workspace.tf b/documentation/terraform-private-preview/sample-catalog/synapse/log_analytics_workspace.tf new file mode 100644 index 0000000..a50be22 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/log_analytics_workspace.tf @@ -0,0 +1,11 @@ +# Log Analytics workspace + +module "log_analytics_workspace" { + source = "github.com/Azure/azure-data-labs-modules/terraform/log-analytics/log-analytics-workspace" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/main.tf b/documentation/terraform-private-preview/sample-catalog/synapse/main.tf new file mode 100644 index 0000000..428780e --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/main.tf @@ -0,0 +1,24 @@ +terraform { + # backend "azurerm" {} + required_providers { + azurerm = { + version = "= 3.53.0" + } + } +} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = false + } + } +} + +data "azurerm_client_config" "current" {} + +data "http" "ip" { + url = "https://ifconfig.me" +} + + diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/manifest.yml b/documentation/terraform-private-preview/sample-catalog/synapse/manifest.yml new file mode 100644 index 0000000..0fc9fc5 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/manifest.yml @@ -0,0 +1,46 @@ +name: AzureSynapse +summary: Terraform example for Azure Synapse. +description: Deploys an environment using Terraform containing Azure Synapse. +runner: Terraform +templatePath: main.tf +parameters: +- id: "prefix" + name: "Prefix" + type: "string" + required: true +- id: "postfix" + name: "Postfix" + type: "string" + required: true +- id: "location" + name: "Location" + default: "centralus" + type: "string" +- id: "enable_private_endpoints" + name: "enable_private_endpoints" + default: "false" + type: "string" +- id: "enable_jumphost" + name: "enable_jumphost" + default: "false" + type: "string" +- id: "enable_analysis_services_server" + name: "enable_analysis_services_server" + default: "false" + type: "string" +- id: "enable_data_factory" + name: "enable_data_factory" + default: "false" + type: "string" +- id: "enable_synapse_sql_pool" + name: "enable_synapse_sql_pool" + default: "false" + type: "string" +- id: "enable_synapse_spark_pool" + name: "enable_synapse_spark_pool" + default: "false" + type: "string" +- id: "enable_event_hub" + name: "enable_event_hub" + default: "false" + type: "string" diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/network.tf b/documentation/terraform-private-preview/sample-catalog/synapse/network.tf new file mode 100644 index 0000000..b692ee0 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/network.tf @@ -0,0 +1,39 @@ +# Virtual network + +module "virtual_network" { + source = "github.com/Azure/azure-data-labs-modules/terraform/virtual-network" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + address_space = ["10.0.0.0/16"] + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) + + tags = local.tags +} + +# Subnets + +module "subnet_default" { + source = "github.com/Azure/azure-data-labs-modules/terraform/subnet" + + name = "snet-${local.prefix}-${local.postfix}-default" + resource_group_name = local.resource_group_name + vnet_name = module.virtual_network[0].name + address_prefixes = ["10.0.1.0/24"] + private_endpoint_network_policies_enabled = true + + count = local.enable_private_endpoints ? 1 : (local.enable_jumphost ? 1 : 0) +} + +module "subnet_bastion" { + source = "github.com/Azure/azure-data-labs-modules/terraform/subnet" + + name = "AzureBastionSubnet" + resource_group_name = local.resource_group_name + vnet_name = module.virtual_network[0].name + address_prefixes = ["10.0.10.0/27"] + + count = local.enable_jumphost ? 1 : 0 +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/private_dns_zone.tf b/documentation/terraform-private-preview/sample-catalog/synapse/private_dns_zone.tf new file mode 100644 index 0000000..d0fd8cd --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/private_dns_zone.tf @@ -0,0 +1,14 @@ +# Private DNS zones + +module "private_dns_zones" { + source = "github.com/Azure/azure-data-labs-modules/terraform/private-dns-zone" + + resource_group_name = local.enable_private_endpoints ? module.resource_group_global_dns[0].name : "none" + + vnet_id = local.enable_private_endpoints ? module.virtual_network[0].id : null + dns_zones = local.dns_zones + + count = local.enable_private_endpoints ? 1 : 0 + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/resource_group.tf b/documentation/terraform-private-preview/sample-catalog/synapse/resource_group.tf new file mode 100644 index 0000000..2651a41 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/resource_group.tf @@ -0,0 +1,23 @@ +# Resource group + +module "resource_group" { + source = "github.com/Azure/azure-data-labs-modules/terraform/resource-group" + + basename = local.basename + location = local.location + + count = local.enable_ade_deployment ? 0 : 1 + + tags = local.tags +} + +module "resource_group_global_dns" { + source = "github.com/Azure/azure-data-labs-modules/terraform/resource-group" + + basename = "${local.basename}-global-dns" + location = local.location + + count = local.enable_private_endpoints ? 1 : 0 + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/storage_account.tf b/documentation/terraform-private-preview/sample-catalog/synapse/storage_account.tf new file mode 100644 index 0000000..28997d5 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/storage_account.tf @@ -0,0 +1,24 @@ +# Storage Account + +module "storage_account_syn" { + source = "github.com/Azure/azure-data-labs-modules/terraform/storage-account" + + basename = "${local.safe_basename}syn" + resource_group_name = local.resource_group_name + location = local.location + account_tier = "Standard" + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids_blob = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.blob.core.windows.net"].id] : [] + private_dns_zone_ids_dfs = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.dfs.core.windows.net"].id] : [] + + hns_enabled = true + firewall_default_action = "Allow" + firewall_ip_rules = [data.http.ip.body] + firewall_bypass = ["AzureServices"] + + module_enabled = true + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/synapse.tf b/documentation/terraform-private-preview/sample-catalog/synapse/synapse.tf new file mode 100644 index 0000000..17caa54 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/synapse.tf @@ -0,0 +1,70 @@ +# # Synapse workspace + +module "synapse_workspace" { + source = "github.com/Azure/azure-data-labs-modules/terraform/synapse/synapse-workspace" + + basename = local.basename + resource_group_name = local.resource_group_name + location = local.location + adls_id = module.storage_account_syn.adls_id + storage_account_id = module.storage_account_syn.id + storage_account_name = module.storage_account_syn.name + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids_sql = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.sql.azuresynapse.net"].id] : null + private_dns_zone_ids_dev = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.dev.azuresynapse.net"].id] : null + + synadmin_username = var.synadmin_username + synadmin_password = var.synadmin_password + + aad_login = { + name = var.aad_login.name + object_id = var.aad_login.object_id + tenant_id = var.aad_login.tenant_id + } + + module_enabled = true + is_private_endpoint = local.enable_private_endpoints + + tags = local.tags +} + +# Synapse Private Link Hub + +module "synapse_private_link_hub" { + #source = "github.com/Azure/azure-data-labs-modules/terraform/synapse/synapse-private-link-hub" + source = "git::https://github.com/Azure/azure-data-labs-modules.git//terraform/synapse/synapse-private-link-hub?ref=v1.5.0-beta" + + basename = local.safe_basename + resource_group_name = local.resource_group_name + location = local.location + + subnet_id = local.enable_private_endpoints ? module.subnet_default[0].id : null + private_dns_zone_ids = local.enable_private_endpoints ? [module.private_dns_zones[0].list["privatelink.azuresynapse.net"].id] : null + + module_enabled = local.enable_private_endpoints + + tags = local.tags +} + +# Synapse Spark pool + +module "synapse_spark_pool" { + source = "github.com/Azure/azure-data-labs-modules.git//terraform/synapse/synapse-spark-pool" + + basename = local.safe_basename + synapse_workspace_id = module.synapse_workspace.id + + module_enabled = local.enable_synapse_spark_pool +} + +# Synapse SQL pool + +module "synapse_sql_pool" { + source = "github.com/Azure/azure-data-labs-modules.git//terraform/synapse/synapse-sql-pool" + + basename = local.safe_basename + synapse_workspace_id = module.synapse_workspace.id + + module_enabled = local.enable_synapse_sql_pool +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/terraform.tfvars.example b/documentation/terraform-private-preview/sample-catalog/synapse/terraform.tfvars.example new file mode 100644 index 0000000..c626456 --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/terraform.tfvars.example @@ -0,0 +1,6 @@ +# Synapse AAD login credentials +aad_login = { + name = "azureuser@contoso.com" + object_id = "00000000-0000-0000-0000-000000000000" + tenant_id = "00000000-0000-0000-0000-000000000000" +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/synapse/variables.tf b/documentation/terraform-private-preview/sample-catalog/synapse/variables.tf new file mode 100644 index 0000000..878603a --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/synapse/variables.tf @@ -0,0 +1,101 @@ +variable "location" { + type = string + description = "Location of the resource group and modules" +} + +variable "prefix" { + type = string + description = "Prefix for module names" +} + +variable "postfix" { + type = string + description = "Postfix for module names" +} + +variable "aad_login" { + description = "AAD login" + type = object({ + name = string + object_id = string + tenant_id = string + }) + default = { + name = "AzureAD Admin" + object_id = "00000000-0000-0000-0000-000000000000" + tenant_id = "00000000-0000-0000-0000-000000000000" + } +} + +variable "jumphost_username" { + type = string + description = "VM username" + default = "azureuser" +} + +variable "jumphost_password" { + type = string + description = "VM password" + default = "ThisIsNotVerySecure!" +} + +variable "synadmin_username" { + type = string + description = "The Login Name of the SQL administrator" + default = "sqladminuser" +} + +variable "synadmin_password" { + type = string + description = "The Password associated with the sql_administrator_login for the SQL administrator" + default = "ThisIsNotVerySecure!" +} + +# Feature flags + +variable "enable_private_endpoints" { + type = bool + description = "Is secure enabled?" + default = false +} + +variable "enable_jumphost" { + type = bool + description = "Is jumphost required?" + default = false +} + +variable "enable_synapse_spark_pool" { + type = bool + description = "Variable to enable or disable the module" + default = false +} + +variable "enable_synapse_sql_pool" { + type = bool + description = "Variable to enable or disable the module" + default = false +} + +variable "enable_data_factory" { + type = bool + description = "Variable to enable or disable the module" + default = false +} + +variable "enable_analysis_services_server" { + type = bool + description = "Variable to enable or disable the module" + default = false +} + +variable "enable_event_hub" { + type = bool + description = "Variable to enable or disable the module" + default = false +} + +variable "resource_group_name" { + type = string + description = "Variable for ADE based deployment" +} \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/web-app/README.md b/documentation/terraform-private-preview/sample-catalog/web-app/README.md new file mode 100644 index 0000000..60e5faf --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/web-app/README.md @@ -0,0 +1,7 @@ +# Simple Web App Deployment using Terraform + +This template provides a simple Terraform configuration to deploy an Azure Web App via a Azure App Services Plan and a Windows Web App. Descriptions of the resources can be found here, from the 'azurerm' provider for Terraform:

+[App Service Plan](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/app_service)

+[Windows Web App](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/windows_web_app)

+ +For insight on how to utilize this template and deploy an environment, please see [Create an Environment](https://learn.microsoft.com/en-us/azure/deployment-environments/quickstart-create-access-environments) \ No newline at end of file diff --git a/documentation/terraform-private-preview/sample-catalog/web-app/manifest.yaml b/documentation/terraform-private-preview/sample-catalog/web-app/manifest.yaml new file mode 100644 index 0000000..b0c1c3a --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/web-app/manifest.yaml @@ -0,0 +1,14 @@ +name: WebApp +summary: Creates a web app +description: Deploys an environment using Terraform containing a Web App +runner: Terraform +templatePath: web_app.tf +parameters: + - id: "resource_name" + name: "Resource Name" + required: true + type: "string" + - id: "location" + name: "Location" + required: true + type: "string" diff --git a/documentation/terraform-private-preview/sample-catalog/web-app/web_app.tf b/documentation/terraform-private-preview/sample-catalog/web-app/web_app.tf new file mode 100644 index 0000000..814617e --- /dev/null +++ b/documentation/terraform-private-preview/sample-catalog/web-app/web_app.tf @@ -0,0 +1,43 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.0.0" + } + } + + required_version = ">= 1.0.0" +} + +provider "azurerm" { + features {} + + skip_provider_registration = true +} + +variable "resource_group_name" {} + +variable "resource_name" {} + +variable "location" {} + +data "azurerm_resource_group" "rg" { + name = var.resource_group_name +} + +resource "azurerm_service_plan" "service_plan" { + name = replace(var.resource_name, " ", "-") + resource_group_name = data.azurerm_resource_group.rg.name + location = var.location + sku_name = "P1v2" + os_type = "Windows" +} + +resource "azurerm_windows_web_app" "example" { + name = replace(var.resource_name, " ", "-") + resource_group_name = data.azurerm_resource_group.rg.name + location = var.location + service_plan_id = azurerm_service_plan.service_plan.id + + site_config {} +} diff --git a/tools/build-arm.py b/tools/build-arm.py new file mode 100644 index 0000000..0507a2a --- /dev/null +++ b/tools/build-arm.py @@ -0,0 +1,44 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +import os +import shutil +import subprocess + +from pathlib import Path + +repository_root = Path(__file__).resolve().parent.parent +environments_path = repository_root / 'Environments' + +environments = [] + +print('Building ARM templates from bicep files...') +# walk the Environments directory and find all the child directories +for dirpath, dirnames, files in os.walk(environments_path): + # os.walk includes the root directory (i.e. repo/Environments) so we need to skip it + if not environments_path.samefile(dirpath) and Path(dirpath).parent.samefile(environments_path): + environments.append(Path(dirpath)) + # image_names.append(Path(dirpath).name) + +# get the full path to the git executable +git = shutil.which('git') + +for environment in environments: + print(f' Ensuring: {environment}/azuredeploy.json') + if not (environment / 'azuredeploy.json').exists(): + # if the azuredeploy.json file doesn't exist, create it + (environment / 'azuredeploy.json').touch() + # run the git command to add the azuredeploy.json file + subprocess.run([git, 'add', environment / 'azuredeploy.json']) + +# get the full path to the azure cli executable +az = shutil.which('az') + +for environment in environments: + print(f' Compiling template: {environment}') + # run the azure cli command to compile the template + subprocess.run([az, 'bicep', 'build', '--file', environment / 'main.bicep', '--outfile', environment / 'azuredeploy.json']) + +print('Done')