azure-pipelines.yml

# Starter pipeline
# Start with a minimal pipeline that you can customize to build and deploy your code.
# Add steps that build, run tests, deploy, and more:
# https://aka.ms/yaml
pool:
  vmImage: 'ubuntu-latest'

variables:
- name: PIPELINE_ORG
  value: Azure
- name: PIPELINE_REPO
  value: depth-coverage-pipeline
- name: PULL_NUMBER
  value: $(System.PullRequest.PullRequestNumber)
- name: ResourceProvider
  value: empty
- name: Resource
  value: ""
- name: ResourceMapJson
  value: ""
- name: RESOURCE_MAP_FILE
  value: ToGenerate.json
- name: README_FILE
  value: ""
- name: TARGET
  value: clicore
- name: AlertInstruction
  value: ""
- name: NotifyUsers
  value: ""
- name: Artifcact
  value: ""

stages:
- stage: setup
  jobs:
  # - job: controlappserver
  #   steps:
  #   - script: |
  #       curl -X GET http://10.172.15.142:3000
  - job: config
    steps:
    - script: |
        sudo npm install -g depthcoverage --unsafe-perm=true --allow-root
        sudo apt-get install jq -y
      displayName: 'prepare'
    - script: |
        depthcoverage --token=$(PIPELINE_TOKEN) --org=$(PIPELINE_ORG) --repo=$(PIPELINE_REPO) --operation=readPR --PRNumber=$(System.PullRequest.PullRequestNumber)
        # rp=`jq .RPName $(RESOURCE_MAP_FILE)`
        # echo $rp
        # echo '##vso[task.setvariable variable=ResourceProvider]$rp'
      displayName: 'Get resource config file'
    - pwsh: |
        $jsonfile=$env:RESOURCE_MAP_FILE
        echo $jsonfile
        $data = Get-content $jsonfile | ConvertFrom-Json
        echo $data
        $name=$data.RPName
        
        $readmefile=$data.readmeFile
        $target=$data.target

        $ignore=$data.ignoreFailures
        $excludeStages=$data.excludeStages

        echo $target
        Write-Host "##vso[task.setvariable variable=ResourceProvider]$name"
        Write-Host "##vso[task.setvariable variable=README_FILE]$readmefile"
        Write-Host "##vso[task.setvariable variable=TARGET]$target"
        $af=""
        $packagename=""
        if($target -eq "terraform") {
          $af="generated/$name ./go/src/github.com/terraform-providers/terraform-provider-azurerm/vendor/github.com/Azure/azure-sdk-for-go/services/$name"
          $packagename=$name
        } else {
          if($name -eq "compute") {
            $af="./azure-cli/src/azure-cli/azure/cli/command_modules/vm ./azure-cli/src/azure-cli/requirements*.txt ./azure-cli/doc/sphinx/azhelpgen/doc_source_map.json ./azure-rest-api-specs/specification/$(ResourceProvider)/resource-manager/readme.az.md"
            $packagename="vm"
          } else {
            $af="./azure-cli/src/azure-cli/azure/cli/command_modules/$name ./azure-cli/src/azure-cli/requirements*.txt ./azure-cli/doc/sphinx/azhelpgen/doc_source_map.json ./azure-rest-api-specs/specification/$(ResourceProvider)/resource-manager/readme.az.md"
            $packagename=$name
          }
        }
        echo $af
        Write-Host "##vso[task.setvariable variable=Artifcact]$af"
        Write-Host "##vso[task.setvariable variable=Package]$packagename"

        Write-Host "##vso[task.setvariable variable=ignore;isOutput=true]$ignore TestFailure"
        Write-Host "##vso[task.setvariable variable=excludestages;isOutput=true]$excludedStages"
      name: configResource
      displayName: "Set variable"
    - script: echo $(ResourceProvider)
      displayName: "Get variable"
    - script: |
        mkdir -p $(Pipeline.Workspace)/variables
        echo "$(ResourceProvider)" > $(Pipeline.Workspace)/variables/ResourceProvider
        echo "$(README_FILE)" > $(Pipeline.Workspace)/variables/README_FILE
        echo "$(TARGET)" > $(Pipeline.Workspace)/variables/TARGET
        echo "$(Artifcact)" > $(Pipeline.Workspace)/variables/Artifact

        echo ""
        ls -l $(Pipeline.Workspace)/variables
      displayName: "publish variable"
    - task: PublishPipelineArtifact@1
      inputs:
        targetPath: '$(Pipeline.Workspace)'
        artifact: 'variables'
        publishLocation: 'pipeline'
- stage: GenerateCode
  dependsOn: setup
  condition: succeeded()
  jobs:
  - job: GenerateJob
    steps:
    - template: .azure-pipelines/templates/VariableSetTask.yml
    - template: .azure-pipelines/templates/GenerateCode.yml
    - bash: |
        sdktarget=$(TARGET)
        if [ "$sdktarget" == "terraform" ]; then
          echo "##vso[task.setvariable variable=tobuild;isOutput=true]Yes"
        else
          echo "##vso[task.setvariable variable=tobuild;isOutput=true]No"
        fi
      displayName: "set output variable"
      name: DetermineResult
    - template: .azure-pipelines/templates/AfterTask.yml
- stage: Build
  dependsOn: GenerateCode
  condition: and(succeeded(), eq(stageDependencies.GenerateCode.outputs['GenerateJob.DetermineResult.tobuild'], 'Yes'))
  jobs:
  - job: build
    steps:
    - template: .azure-pipelines/templates/BeforeTask.yml
    - script: echo 'build'
    - template: .azure-pipelines/templates/BuildTF.yml
  - job: alertIssue
    dependsOn: build
    condition: and(failed(),eq(variables['TARGET'], 'terraform'))
    steps:
    - template: .azure-pipelines/templates/IssueAlertTask.yml
- stage: MockTest
  dependsOn:
  - GenerateCode
  - Build
  condition: and(in(dependencies.Build.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), succeeded('GenerateCode'))
  jobs:
  - job: DetermineSDKJob
    steps:
    - template: .azure-pipelines/templates/VariableSetTask.yml
    - bash: |
        sdktarget=$(TARGET)
        if [ "$sdktarget" == "terraform" ]; then
          echo "##vso[task.setvariable variable=sdk;isOutput=true]terraform"
        else
          echo "##vso[task.setvariable variable=sdk;isOutput=true]cli"
        fi
      name: determineSDK
  - job: ClIMockTest
    steps:
    - template: .azure-pipelines/templates/BeforeTask.yml
    - script: echo 'CLI MockTest'
    - template: .azure-pipelines/templates/SetupCliTestEnv.yml
    - template: .azure-pipelines/templates/SetupCliMockServerEnv.yml
    # - template: .azure-pipelines/templates/SetupCliTestEnv.yml
    - template: .azure-pipelines/templates/RunCliMockTest.yml
    displayName: CLIMockTest
    dependsOn: DetermineSDKJob
    condition: eq(dependencies.DetermineSDKJob.outputs['determineSDK.sdk'], 'cli')
  - job: TFMockTest
    steps:
    - template: .azure-pipelines/templates/BeforeTask.yml
    - script: echo 'TF MockTest'
    - template: .azure-pipelines/templates/SetupTFTestEnv.yml
    - template: .azure-pipelines/templates/SetupTFMockServerEnv.yml
    - template: .azure-pipelines/templates/RunTFMockTest.yml
    displayName: TFIMockTest
    dependsOn: DetermineSDKJob
    condition: eq(dependencies.DetermineSDKJob.outputs['determineSDK.sdk'], 'terraform')
  - job: alertIssue
    dependsOn:
    - CLIMockTest
    - TFMockTest
    condition: or(and(eq(dependencies.DetermineSDKJob.outputs['determineSDK.sdk'], 'cli'), failed('ClIMockTest')), and(eq(dependencies.DetermineSDKJob.outputs['determineSDK.sdk'], 'terraform'), failed('TFMockTest')))
    steps:
    - template: .azure-pipelines/templates/VariableSetTask.yml
    - template: .azure-pipelines/templates/IssueAlertTask.yml
- stage: LiveTest
  dependsOn: MockTest
  condition: succeeded('MockTest')
  jobs:
  - job: DetermineSDKForLiveJob
    steps:
    - template: .azure-pipelines/templates/VariableSetTask.yml
    - bash: |
        sdktarget=$(TARGET)
        if [ "$sdktarget" == "terraform" ]; then
          echo "##vso[task.setvariable variable=sdk;isOutput=true]terraform"
        else
          echo "##vso[task.setvariable variable=sdk;isOutput=true]cli"
        fi
      name: liveSDK
  - job: CLILiveTest
    steps:
    - script: echo 'CLILiveTest'
    - template: .azure-pipelines/templates/BeforeTask.yml
    - template: .azure-pipelines/templates/SetupCliTestEnv.yml
    - template: .azure-pipelines/templates/RunCliTest.yml
    displayName: CLILiveTest
    dependsOn: DetermineSDKForLiveJob
    condition: eq(dependencies.DetermineSDKForLiveJob.outputs['liveSDK.sdk'], 'cli')
  - job : TFLiveTest
    steps:
    - bash: echo 'TF LiveTest'
    - template: .azure-pipelines/templates/BeforeTask.yml
    - template: .azure-pipelines/templates/SetupTFTestEnv.yml
    - template: .azure-pipelines/templates/RunTFLiveTest.yml
    displayName: TFLiveTest
    dependsOn: DetermineSDKForLiveJob
    condition: eq(dependencies.DetermineSDKForLiveJob.outputs['liveSDK.sdk'], 'terraform')
  - job: alertIssue
    dependsOn:
    - CLILiveTest
    - TFLiveTest
    condition: or(and(eq(dependencies.DetermineSDKForLiveJob.outputs['liveSDK.sdk'], 'cli'), failed('ClIMockTest')), and(eq(dependencies.DetermineSDKForLiveJob.outputs['liveSDK.sdk'], 'terraform'), failed('TFMockTest')))
    steps:
    - template: .azure-pipelines/templates/VariableSetTask.yml
    - template: .azure-pipelines/templates/IssueAlertTask.yml
- stage: SubmitStage
  dependsOn:
  - setup
  - LiveTest
  condition: or(succeeded('LiveTest'),  contains(stageDependencies.setup.outputs['config.configResource.ignore'], 'TestFailure'))
  jobs:
  - job: submit
    steps:
    - template: .azure-pipelines/templates/BeforeTask.yml
    - bash: |
        echo $(Pipeline.Workspace)
        pwd

        echo "https://$(PIPELINE_USER):${PIPELINE_TOKEN}@github.com" > ~/.git-credentials
        sdktarget=$(TARGET)
        if [ "$sdktarget" == "terraform" ]; then
          git clone --branch depth-$(TARGET)-$(ResourceProvider) https://${PIPELINE_USER}:${PIPELINE_TOKEN}@github.com/${TF_ORG}/terraform-provider-azurerm.git
          cd ./terraform-provider-azurerm
        fi

        if [ "$sdktarget" == "clicore" ] || [ "$sdktarget" == "cliextension" ]; then
          git clone --branch depth-$(TARGET)-$(ResourceProvider) https://${PIPELINE_USER}:${PIPELINE_TOKEN}@github.com/${CLI_ORG}/azure-cli
          cd ./azure-cli
        fi

        git config credential.helper store; git config --global user.email "chunyu@microsoft.com";git config --global user.name "chunyu3;"
        git branch; git add -A; git commit -m "autogenerated"; git push origin depth-$(TARGET)-$(ResourceProvider);
      displayName: 'Generate pull request'
  - job: confirm
    dependsOn: submit
    condition: succeeded()
    displayName: confirm submit or customize
    steps:
    - template: .azure-pipelines/templates/VariableSetTask.yml
    - bash: |
        echo "https://$(PIPELINE_USER):${PIPELINE_TOKEN}@github.com" > ~/.git-credentials
        git clone --branch depth-$(TARGET)-$(ResourceProvider) https://${PIPELINE_USER}:${PIPELINE_TOKEN}@github.com/${Swagger_ORG}/azure-rest-api-specs.git
        sdktarget=$(TARGET)
        if [ "$sdktarget" == "terraform" ]; then
          git clone --branch depth-$(TARGET)-$(ResourceProvider) https://${PIPELINE_USER}:${PIPELINE_TOKEN}@github.com/${TF_ORG}/terraform-provider-azurerm.git
          cp ./azure-rest-api-specs/specification/$(ResourceProvider)/resource-manager/readme.trenton.md ./terraform-providers/terraform-provider-azurerm/azurerm/internal/services/$(ResourceProvider)/
          cd ./terraform-provider-azurerm
          # git checkout -b depth-code-$(TARGET)-$(ResourceProvider)
          tf_existed_in_remote=$(git ls-remote --heads origin depth-code-$(TARGET)-$(ResourceProvider))
          if [[ -z ${tf_existed_in_remote} ]]; then
            git checkout -b depth-code-$(TARGET)-$(ResourceProvider)
          else
            git checkout depth-code-$(TARGET)-$(ResourceProvider)
          fi
        fi

        if [ "$sdktarget" == "clicore" ] || [ "$sdktarget" == "cliextension" ]; then
          git clone --branch depth-$(TARGET)-$(ResourceProvider) https://${PIPELINE_USER}:${PIPELINE_TOKEN}@github.com/${CLI_ORG}/azure-cli
          name=$(ResourceProvider)
          if [ "$name" == "compute" ]; then 
            name="vm"
          fi
            
          # cp ./azure-rest-api-specs/specification/$(ResourceProvider)/resource-manager/readme.az.md ./azure-cli/src/azure-cli/azure/cli/command_modules/$name/
          cp ./azure-rest-api-specs/specification/$(ResourceProvider)/resource-manager/readme.az.md ./azure-cli/
          cd ./azure-cli

          # git checkout -b depth-code-$(TARGET)-$(ResourceProvider)
          tf_existed_in_remote=$(git ls-remote --heads origin depth-code-$(TARGET)-$(ResourceProvider))
          if [[ -z ${tf_existed_in_remote} ]]; then
            git checkout -b depth-code-$(TARGET)-$(ResourceProvider)
          else
            git checkout depth-code-$(TARGET)-$(ResourceProvider)
          fi
          git branch; git add -A; git commit -m "autogenerated"; git push origin depth-code-$(TARGET)-$(ResourceProvider);

        fi
        
        git config credential.helper store; git config --global user.email "chunyu@microsoft.com";git config --global user.name "chunyu3;"
        git branch; git add -A; git commit -m "autogenerated"; git push origin depth-code-$(TARGET)-$(ResourceProvider);

        cd $(Pipeline.Workspace)/s
        
        sdktarget=$(TARGET)
        org=""
        repo=""
        base="master"
        sdk="";
        if [ "$sdktarget" == "terraform" ]; then
          org=$(TF_ORG)
          repo="terraform-provider-azurerm"
          sdk="terraform"
        else
          org=$(CLI_ORG)
          repo="azure-cli"
          base="dev"
          sdk="clicore"
        fi
        echo "$repo"
        echo "$org"
        echo "$base"
        # curl -X POST -H "Content-Type: application/json" http://$(CodegenApp_Server)/DepthCoverage/generateCodePR -d '{"token":"$(PIPELINE_TOKEN)", "org":"$org", "repo":"$repo", "title":"[]pull request of $(ResourceProvider)", "branch":"depth-$(TARGET)-$(ResourceProvider)", "base":"$base"}'
        # codePR=`curl -X POST -H "Content-Type: application/json" http://$(CodegenApp_Server)/DepthCoverage/generateCodePR -d '{"token":"$(PIPELINE_TOKEN)", "org":"$(org)", "repo":"$(repo)", "title":"{depth coverage, not merge} pull request of $(ResourceProvider)", "branch":"depth-$(TARGET)-$(ResourceProvider)", "base":"$(base)"}'`
        codePR=$(curl -X POST -H "Content-Type: application/json" http://$(CodegenApp_Server)/DepthCoverage/generateCodePR -d '{"token":"$(PIPELINE_TOKEN)", "org":"'"$org"'", "repo":"'"$repo"'", "title":"{depth coverage, not merge} pull request of $(ResourceProvider)", "branch":"depth-code-$(TARGET)-$(ResourceProvider)", "base":"'"$base"'"}')
        echo $codePR

        triggerPR="https://github.com/Azure/depth-coverage-pipeline/pull/$(System.PullRequest.PullRequestNumber)"
        submitURL="http://$(CodegenApp_Server)/DepthCoverage/RPs/$(ResourceProvider)/SDKs/$sdk/submit?token=$(PIPELINE_TOKEN)&swaggerorg=$(Swagger_ORG)&org=$org"
        customizeURL="http://$(CodegenApp_Server)/DepthCoverage/RPs/$(ResourceProvider)/SDKs/$sdk/Customize?triggerPR=$triggerPR&token=$(PIPELINE_TOKEN)&codePR=$codePR&org=$org"

        
        python scripts/onboard.py "Onboarding" "$(TARGET) Onboarding $(ResourceProvider) confirm" "$codePR" "$triggerPR" "$submitURL" "$customizeURL" > scripts/email.html
        echo "start of email"
        cat scripts/email.html
        echo "end of email"
        to_mail="chunyu@microsoft.com"
        
        sudo pip install wheel
        sudo pip install sendgrid
        python scripts/sendGridemail.py "scripts/email.html" $to_mail "Depth-coverage Onboarding Confirm" $(API_KEY)
        echo "end of send email"
      displayName: "send confirm email"