Fix ci (#2785)

.chronus/changes/fix-ci-2024-2024-7-20-16-32-44.md

@@ -0,0 +1,7 @@
+---
+changeKind: internal
+packages:
+  - "@azure-tools/typespec-python"
+---
+
+Enable lint check in CI

.chronus/changes/fix-ci-2024-2024-7-22-11-47-37.md

@@ -0,0 +1,7 @@
+---
+changeKind: internal
+packages:
+  - "@autorest/python"
+---
+
+update ci to use npm

.gitignore

@@ -126,3 +126,4 @@ node_modules/
 # Generated test folders
 test/services/*/_generated
 **/autorest.python/generator
+**/autorest.python/scripts/eng

eng/dev_requirements.txt

@@ -5,3 +5,4 @@ pytest==8.3.2
 tox==4.18.0
 coverage==7.6.1
 black==24.8.0
+setuptools==69.2.0

eng/pipelines/ci-template.yml

@@ -3,11 +3,10 @@ parameters:
     installTypeSpec: false
     installCadlRanch: false
     folderName: ""
-    pythonCodeChecks: false
-    pythonFolderName: ""
     regenerate: false
     checkChange: true
     updateToLatestTypespec: false
+    unitTest: false
 
 steps:
     - checkout: self
@@ -55,12 +54,6 @@ steps:
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/
       condition: and(succeeded(), eq(${{ parameters.updateToLatestTypespec }}, false))
 
-    # - script: pnpm change verify
-    #   displayName: Check changelog
-    #   workingDirectory: $(Build.SourcesDirectory)/autorest.python/
-    #   condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/heads/publish/')), not(startsWith(variables['Build.SourceBranch'], 'refs/heads/dependabot/')))
-    #   continueOnError: true
-
     - script: pnpm list
       displayName: Pnpm list
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/typespec-python
@@ -81,46 +74,39 @@ steps:
       displayName: List installed packages
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}
 
-    - script: pylint ${{parameters.pythonFolderName}}
+    - script: pnpm run lint --command pylint
       displayName: Pylint
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}
-      condition: and(succeeded(), ${{ parameters.pythonCodeChecks }})
 
-    - script: mypy ${{parameters.pythonFolderName}}
+    - script: pnpm run lint --command mypy
       displayName: Mypy
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}
-      condition: and(succeeded(), ${{ parameters.pythonCodeChecks }})
 
-    - script: pyright ${{parameters.pythonFolderName}}
+    - script: pnpm run lint --command pyright
       displayName: Pyright
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}
-      condition: and(succeeded(), ${{ parameters.pythonCodeChecks }})
 
-    - script: black $(Build.SourcesDirectory)/autorest.python
+    - script: pnpm run format
       displayName: Black
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}
-      condition: and(succeeded(), ${{ parameters.pythonCodeChecks }})
 
     - script: node ./eng/scripts/check-for-changed-files.js
       displayName: Fail on black autorest diff
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/
-      condition: and(succeeded(), ${{ parameters.pythonCodeChecks }})
 
     - script: |
         cd test/unittests
         tox run -e ci
       displayName: Unit tests
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/
-      condition: and(succeeded(), ${{ parameters.pythonCodeChecks }})
+      condition: and(succeeded(), ${{ parameters.unitTest }})
 
     - script: inv regenerate
       displayName: "Regenerate Code"
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/
       condition: and(succeeded(), ${{ parameters.regenerate }}, eq('${{parameters.folderName}}', 'autorest.python'))
 
-    - script: |
-        find test/azure/generated -type f ! -name '*apiview_mapping_python.json*' -delete
-        npm run regenerate
+    - script: npm run regenerate
       displayName: "Regenerate Code"
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/
       condition: and(succeeded(), ${{ parameters.regenerate }}, eq('${{parameters.folderName}}', 'typespec-python'))

eng/pipelines/ci.yml

@@ -78,8 +78,7 @@ jobs:
         parameters:
             installAutorest: true
             folderName: "autorest.python"
-            pythonCodeChecks: true
-            pythonFolderName: autorest
+            unitTest: true
             regenerate: true
 
       - script: |

eng/scripts/run_mypy.py

@@ -12,11 +12,17 @@
 import os
 import logging
 import sys
-from util import run_check, AUTOREST_PACKAGE_DIR
+from util import run_check
 
 logging.getLogger().setLevel(logging.INFO)
 
-config_file_location = os.path.join(AUTOREST_PACKAGE_DIR, "mypy.ini")
+
+def get_config_file_location():
+    mypy_ini_path = os.path.join(os.getcwd(), "../../scripts/eng/mypy.ini")
+    if os.path.exists(mypy_ini_path):
+        return mypy_ini_path
+    else:
+        return os.path.join(os.getcwd(), "../../../scripts/eng/mypy.ini")
 
 
 def _single_dir_mypy(mod):
@@ -28,7 +34,7 @@ def _single_dir_mypy(mod):
                 "-m",
                 "mypy",
                 "--config-file",
-                config_file_location,
+                get_config_file_location(),
                 "--ignore-missing",
                 str(inner_class.absolute()),
             ]

eng/scripts/run_pylint.py

@@ -12,11 +12,17 @@
 import os
 import logging
 import sys
-from util import run_check, AUTOREST_PACKAGE_DIR
+from util import run_check
 
 logging.getLogger().setLevel(logging.INFO)
 
-rfc_file_location = os.path.join(AUTOREST_PACKAGE_DIR, "pylintrc")
+
+def get_rfc_file_location():
+    rfc_file_location = os.path.join(os.getcwd(), "../../scripts/eng/pylintrc")
+    if os.path.exists(rfc_file_location):
+        return rfc_file_location
+    else:
+        return os.path.join(os.getcwd(), "../../../scripts/eng/pylintrc")
 
 
 def _single_dir_pylint(mod):
@@ -27,7 +33,7 @@ def _single_dir_pylint(mod):
                 sys.executable,
                 "-m",
                 "pylint",
-                "--rcfile={}".format(rfc_file_location),
+                "--rcfile={}".format(get_rfc_file_location()),
                 "--load-plugins=pylint_guidelines_checker",
                 "--output-format=parseable",
                 str(inner_class.absolute()),

eng/scripts/run_pyright.py

@@ -8,15 +8,24 @@
 # This script is used to execute pyright within a tox environment. Depending on which package is being executed against,
 # a failure may be suppressed.
 
+import os
 from subprocess import check_output, CalledProcessError
 import logging
 import sys
 import time
-from util import run_check, AUTOREST_PACKAGE_DIR
+from util import run_check
 
 logging.getLogger().setLevel(logging.INFO)
 
 
+def get_pyright_config_file_location():
+    pyright_config = os.path.join(os.getcwd(), "../../scripts/eng/pyrightconfig.json")
+    if os.path.exists(pyright_config):
+        return pyright_config
+    else:
+        return os.path.join(os.getcwd(), "../../../scripts/eng/pyrightconfig.json")
+
+
 def _single_dir_pyright(mod):
     inner_class = next(d for d in mod.iterdir() if d.is_dir() and not str(d).endswith("egg-info"))
     retries = 3
@@ -28,7 +37,7 @@ def _single_dir_pyright(mod):
                     "-m",
                     "pyright",
                     "-p",
-                    str(AUTOREST_PACKAGE_DIR),
+                    get_pyright_config_file_location(),
                     str(inner_class.absolute()),
                 ],
                 text=True,

packages/autorest.python/package.json

@@ -5,10 +5,12 @@
   "scripts": {
     "start": "node ./scripts/run-python3.js ./scripts/start.py",
     "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py",
-    "build": "node ./scripts/copy-generator.js --force",
-    "install": "node ./scripts/copy-generator.js && node ./scripts/run-python3.js ./scripts/install.py",
+    "build": "tsx ./scripts/copy-generator.ts --force",
+    "install": "tsx ./scripts/copy-generator.ts && node ./scripts/run-python3.js ./scripts/install.py",
     "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug",
-    "test": "tsx ./scripts/run-tests.ts"
+    "test": "tsx ./scripts/eng/run-tests.ts --validFolders azure/legacy azure/version-tolerant vanilla/legacy vanilla/version-tolerant dpg/version-tolerant",
+    "lint": "tsx ./scripts/eng/lint.ts --folderName autorest",
+    "format": "tsx ./scripts/eng/format.ts --folderName autorest"
   },
   "main": "index.js",
   "repository": {
@@ -34,7 +36,8 @@
   "devDependencies": {
     "@microsoft.azure/autorest.testserver": "^3.3.46",
     "typescript": "~5.1.3",
-    "@azure-tools/typespec-python": "workspace:^"
+    "@azure-tools/typespec-python": "workspace:^",
+    "chalk": "5.3.0"
   },
   "files": [
     "autorest/**/*.py",

packages/autorest.python/scripts/copy-generator.ts

@@ -0,0 +1,24 @@
+import { copyFileSync, readdirSync } from "fs";
+import { existsSync, removeSync, copySync } from "fs-extra";
+import { join } from "path";
+
+const force: boolean = process.argv[2] === "--force";
+
+function copyAndCreateDir(sourceDir: string, destDir: string) {
+    // Delete the destination directory if it exists
+    if (existsSync(destDir)) {
+        if (force) removeSync(destDir);
+        else process.exit(0);
+    }
+
+    // Copy the source directory to the destination directory
+    copySync(sourceDir, destDir);
+}
+
+const typespecModulePath: string = join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python");
+
+// Copy the generator directory
+copyAndCreateDir(join(typespecModulePath, "generator"), join(__dirname, "..", "generator"));
+
+// Copy the scripts directory
+copyAndCreateDir(join(typespecModulePath, "scripts", "eng"), join(__dirname, "..", "scripts", "eng"));

packages/autorest.python/mypy.ini → packages/autorest.python/scripts/mypy.ini

@@ -33,3 +33,6 @@ ignore_missing_imports = True
 
 [mypy-pygen.*]
 ignore_missing_imports = True
+
+[mypy-yaml.*]
+ignore_missing_imports = True

packages/autorest.python/scripts/run-python3.ts

@@ -0,0 +1,25 @@
+// This script wraps logic in @azure-tools/extension to resolve
+// the path to Python 3 so that a Python script file can be run
+// from an npm script in package.json.  It uses the same Python 3
+// path resolution algorithm as AutoRest so that the behavior
+// is fully consistent (and also supports AUTOREST_PYTHON_EXE).
+//
+// Invoke it like so: "tsx run-python3.ts script.py"
+
+import cp from "child_process";
+import { patchPythonPath } from "./system-requirements.js";
+
+async function runPython3(...args: string[]) {
+    const command = await patchPythonPath(["python", ...args], {
+        version: ">=3.8",
+        environmentVariable: "AUTOREST_PYTHON_EXE",
+    });
+    cp.execSync(command.join(" "), {
+        stdio: [0, 1, 2],
+    });
+}
+
+runPython3(...process.argv.slice(2)).catch((err) => {
+    console.error(err.toString()); // eslint-disable-line no-console
+    process.exit(1);
+});