From feb80b586ebb49cce104939b06d65de1c3bf45e7 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 11 Nov 2024 14:03:17 +0800 Subject: [PATCH 01/49] update dependency tcgc 0.48.1 --- packages/http-client-python/package-lock.json | 14 +++++++------- packages/http-client-python/package.json | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/http-client-python/package-lock.json b/packages/http-client-python/package-lock.json index e2930e99fe..ff8ea4993b 100644 --- a/packages/http-client-python/package-lock.json +++ b/packages/http-client-python/package-lock.json @@ -1,12 +1,12 @@ { "name": "@typespec/http-client-python", - "version": "0.3.7", + "version": "0.3.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@typespec/http-client-python", - "version": "0.3.7", + "version": "0.3.8", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -21,7 +21,7 @@ "@azure-tools/typespec-azure-core": "~0.48.0", "@azure-tools/typespec-azure-resource-manager": "~0.48.0", "@azure-tools/typespec-azure-rulesets": "~0.48.0", - "@azure-tools/typespec-client-generator-core": "~0.48.0", + "@azure-tools/typespec-client-generator-core": "~0.48.1", "@types/js-yaml": "~4.0.5", "@types/node": "~22.5.4", "@types/semver": "7.5.8", @@ -45,7 +45,7 @@ "@azure-tools/typespec-azure-core": ">=0.48.0 <1.0.0", "@azure-tools/typespec-azure-resource-manager": ">=0.48.0 <1.0.0", "@azure-tools/typespec-azure-rulesets": ">=0.48.0 <3.0.0", - "@azure-tools/typespec-client-generator-core": ">=0.48.0 <1.0.0", + "@azure-tools/typespec-client-generator-core": ">=0.48.1 <1.0.0", "@typespec/compiler": ">=0.62.0 <1.0.0", "@typespec/http": ">=0.62.0 <1.0.0", "@typespec/openapi": ">=0.62.0 <1.0.0", @@ -237,9 +237,9 @@ } }, "node_modules/@azure-tools/typespec-client-generator-core": { - "version": "0.48.0", - "resolved": "https://registry.npmjs.org/@azure-tools/typespec-client-generator-core/-/typespec-client-generator-core-0.48.0.tgz", - "integrity": "sha512-+fmKjapz0kP7ONPZap8dgcIKIdQw+YBSrf89csbIyhPTcLnVAk/BKljo8FoNypKXwqKHenslLm0njBKPllkopg==", + "version": "0.48.1", + "resolved": "https://registry.npmjs.org/@azure-tools/typespec-client-generator-core/-/typespec-client-generator-core-0.48.1.tgz", + "integrity": "sha512-pYEZDExltNNLAaA12EwEag5VLESyPoKNQQ/6Olj4rJouA4cBjZDTW80VYgKuPQBt/uCtA0Yn6xxl0nH7TGOwWQ==", "dev": true, "dependencies": { "change-case": "~5.4.4", diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 413f34c07d..eaa02a1519 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -60,7 +60,7 @@ "@azure-tools/typespec-azure-resource-manager": ">=0.48.0 <1.0.0", "@azure-tools/typespec-autorest": ">=0.48.0 <1.0.0", "@azure-tools/typespec-azure-rulesets": ">=0.48.0 <3.0.0", - "@azure-tools/typespec-client-generator-core": ">=0.48.0 <1.0.0" + "@azure-tools/typespec-client-generator-core": ">=0.48.1 <1.0.0" }, "dependencies": { "js-yaml": "~4.1.0", @@ -77,7 +77,7 @@ "@azure-tools/typespec-azure-core": "~0.48.0", "@azure-tools/typespec-azure-rulesets": "~0.48.0", "@azure-tools/typespec-azure-resource-manager": "~0.48.0", - "@azure-tools/typespec-client-generator-core": "~0.48.0", + "@azure-tools/typespec-client-generator-core": "~0.48.1", "@azure-tools/cadl-ranch-specs": "~0.39.1", "@azure-tools/cadl-ranch-expect": "~0.15.6", "@types/js-yaml": "~4.0.5", From 60e4c1573514b61d1e3ac53b7a3eb2cda321a0c4 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 11 Nov 2024 14:57:15 +0800 Subject: [PATCH 02/49] add clientNamespace property --- .../emitter/src/code-model.ts | 5 ++++- .../http-client-python/emitter/src/types.ts | 17 +++++++++++++---- .../http-client-python/emitter/src/utils.ts | 7 +++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/http-client-python/emitter/src/code-model.ts b/packages/http-client-python/emitter/src/code-model.ts index 8595637ebc..6c51e1ab01 100644 --- a/packages/http-client-python/emitter/src/code-model.ts +++ b/packages/http-client-python/emitter/src/code-model.ts @@ -30,7 +30,7 @@ import { simpleTypesMap, typesMap, } from "./types.js"; -import { emitParamBase, getImplementation, removeUnderscoresFromNamespace } from "./utils.js"; +import { emitParamBase, getImplementation, removeUnderscoresFromNamespace, getClientNamespace } from "./utils.js"; function emitBasicMethod( context: PythonSdkContext, @@ -169,6 +169,7 @@ function emitOperationGroups( propertyName: operationGroup.name, operations: operations, operationGroups: emitOperationGroups(context, operationGroup, rootClient, name), + clientNamespace: getClientNamespace(client.clientNamespace, context.sdkPackage.rootNamespace), }); } } @@ -186,6 +187,7 @@ function emitOperationGroups( className: "", propertyName: "", operations: operations, + clientNamespace: getClientNamespace(client.clientNamespace, context.sdkPackage.rootNamespace), }); } } @@ -223,6 +225,7 @@ function emitClient( url, apiVersions: client.apiVersions, arm: context.arm, + clientNamespace: getClientNamespace(client.clientNamespace, context.sdkPackage.rootNamespace), }; } diff --git a/packages/http-client-python/emitter/src/types.ts b/packages/http-client-python/emitter/src/types.ts index 356ce0c54a..940fe12921 100644 --- a/packages/http-client-python/emitter/src/types.ts +++ b/packages/http-client-python/emitter/src/types.ts @@ -21,7 +21,7 @@ import { Type } from "@typespec/compiler"; import { HttpAuth, Visibility } from "@typespec/http"; import { dump } from "js-yaml"; import { PythonSdkContext } from "./lib.js"; -import { camelToSnakeCase, emitParamBase, getAddedOn, getImplementation } from "./utils.js"; +import { camelToSnakeCase, emitParamBase, getAddedOn, getImplementation, getClientNamespace } from "./utils.js"; export const typesMap = new Map>(); export const simpleTypesMap = new Map>(); @@ -75,7 +75,7 @@ export function getType( case "union": return emitUnion(context, type); case "enum": - return emitEnum(type); + return emitEnum(context, type); case "constant": return emitConstant(type)!; case "array": @@ -86,7 +86,7 @@ export function getType( case "duration": return emitDurationOrDateType(type); case "enumvalue": - return emitEnumMember(type, emitEnum(type.enumType)); + return emitEnumMember(type, emitEnum(context, type.enumType)); case "credential": return emitCredential(type); case "bytes": @@ -249,6 +249,7 @@ function emitProperty( }; } + function emitModel( context: PythonSdkContext, type: SdkModelType, @@ -289,6 +290,7 @@ function emitModel( usage: type.usage, isXml: type.usage & UsageFlags.Xml ? true : false, xmlMetadata: type.usage & UsageFlags.Xml ? getXmlMetadata(type) : undefined, + clientNamespace: getClientNamespace(type.clientNamespace, context.sdkPackage.rootNamespace), }; typesMap.set(type, newValue); @@ -314,7 +316,11 @@ function emitModel( return newValue; } -function emitEnum(type: SdkEnumType): Record { + +function emitEnum( + context: PythonSdkContext, + type: SdkEnumType +): Record { if (typesMap.has(type)) { return typesMap.get(type)!; } @@ -338,6 +344,7 @@ function emitEnum(type: SdkEnumType): Record { type: "combined", types, xmlMetadata: {}, + clientNamespace: getClientNamespace(type.clientNamespace, context.sdkPackage.rootNamespace), }; } const values: Record[] = []; @@ -352,6 +359,7 @@ function emitEnum(type: SdkEnumType): Record { values, xmlMetadata: {}, crossLanguageDefinitionId: type.crossLanguageDefinitionId, + clientNamespace: getClientNamespace(type.clientNamespace, context.sdkPackage.rootNamespace), }; for (const value of type.values) { newValue.values.push(emitEnumMember(value, newValue)); @@ -469,6 +477,7 @@ function emitUnion( type: "combined", types: type.variantTypes.map((x) => getType(context, x)), xmlMetadata: {}, + clientNamespace: getClientNamespace(type.clientNamespace, context.sdkPackage.rootNamespace), }); } diff --git a/packages/http-client-python/emitter/src/utils.ts b/packages/http-client-python/emitter/src/utils.ts index 6bdde071c1..a920404c05 100644 --- a/packages/http-client-python/emitter/src/utils.ts +++ b/packages/http-client-python/emitter/src/utils.ts @@ -199,3 +199,10 @@ export function isAzureCoreErrorResponse(t: SdkType | undefined): boolean { export function capitalize(name: string): string { return name[0].toUpperCase() + name.slice(1); } + +export function getClientNamespace(clientNamespace: string, rootNameSpace: string) { + if (["azure.core", "azure.resourcemanager"].some(item => clientNamespace.toLowerCase().startsWith(item))) { + return rootNameSpace; + } + return clientNamespace; +} From ac2b2257c7a4c900b2c48df7406eef31767b25d1 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 25 Nov 2024 15:34:34 +0800 Subject: [PATCH 03/49] init --- .../http-client-python/.vscode/launch.json | 2 +- .../emitter/src/code-model.ts | 6 +++ .../http-client-python/emitter/src/types.ts | 12 +++-- .../http-client-python/emitter/src/utils.ts | 6 ++- .../generator/pygen/codegen/_utils.py | 7 +++ .../pygen/codegen/models/base_builder.py | 1 + .../generator/pygen/codegen/models/client.py | 45 +++++++++------- .../pygen/codegen/models/code_model.py | 52 +++++++++++++++++-- .../pygen/codegen/models/combined_type.py | 4 +- .../pygen/codegen/models/enum_type.py | 24 ++++++--- .../generator/pygen/codegen/models/imports.py | 22 +++++--- .../pygen/codegen/models/lro_operation.py | 4 +- .../pygen/codegen/models/model_type.py | 28 +++++----- .../pygen/codegen/models/operation.py | 47 ++++++++++------- .../pygen/codegen/models/operation_group.py | 28 +++++----- .../pygen/codegen/models/paging_operation.py | 5 +- .../pygen/codegen/models/parameter.py | 12 +++-- .../pygen/codegen/models/primitive_types.py | 12 ++--- .../pygen/codegen/models/property.py | 7 +-- .../pygen/codegen/models/request_builder.py | 13 ++--- .../pygen/codegen/models/response.py | 3 +- .../generator/pygen/codegen/models/utils.py | 7 +++ .../codegen/serializers/base_serializer.py | 4 +- .../codegen/serializers/general_serializer.py | 10 ++-- .../codegen/serializers/model_serializer.py | 4 +- .../operation_groups_serializer.py | 4 +- .../request_builders_serializer.py | 6 ++- .../codegen/serializers/sample_serializer.py | 6 ++- .../codegen/serializers/test_serializer.py | 11 +++- .../codegen/serializers/types_serializer.py | 2 +- 30 files changed, 259 insertions(+), 135 deletions(-) diff --git a/packages/http-client-python/.vscode/launch.json b/packages/http-client-python/.vscode/launch.json index ab9e01f1a5..ad210f3787 100644 --- a/packages/http-client-python/.vscode/launch.json +++ b/packages/http-client-python/.vscode/launch.json @@ -26,7 +26,7 @@ "cwd": "${workspaceFolder}", "args": [ "compile", - "${workspaceFolder}/alpha/main.tsp", + "${workspaceFolder}/alpha/common-properties/main.tsp", "--emit=${workspaceFolder}", "--option=@typespec/http-client-python.flavor=azure" ], diff --git a/packages/http-client-python/emitter/src/code-model.ts b/packages/http-client-python/emitter/src/code-model.ts index 6c51e1ab01..ea0c956c0c 100644 --- a/packages/http-client-python/emitter/src/code-model.ts +++ b/packages/http-client-python/emitter/src/code-model.ts @@ -192,6 +192,12 @@ function emitOperationGroups( } } + // operation has same clientNamespace as the operation group + for (const og of operationGroups) { + for (const op of og.operations) { + op.clientNamespace = og.clientNamespace; + } + return operationGroups.length > 0 ? operationGroups : undefined; } diff --git a/packages/http-client-python/emitter/src/types.ts b/packages/http-client-python/emitter/src/types.ts index 940fe12921..37161ef500 100644 --- a/packages/http-client-python/emitter/src/types.ts +++ b/packages/http-client-python/emitter/src/types.ts @@ -21,7 +21,13 @@ import { Type } from "@typespec/compiler"; import { HttpAuth, Visibility } from "@typespec/http"; import { dump } from "js-yaml"; import { PythonSdkContext } from "./lib.js"; -import { camelToSnakeCase, emitParamBase, getAddedOn, getImplementation, getClientNamespace } from "./utils.js"; +import { + camelToSnakeCase, + emitParamBase, + getAddedOn, + getClientNamespace, + getImplementation, +} from "./utils.js"; export const typesMap = new Map>(); export const simpleTypesMap = new Map>(); @@ -249,7 +255,6 @@ function emitProperty( }; } - function emitModel( context: PythonSdkContext, type: SdkModelType, @@ -316,10 +321,9 @@ function emitModel( return newValue; } - function emitEnum( context: PythonSdkContext, - type: SdkEnumType + type: SdkEnumType, ): Record { if (typesMap.has(type)) { return typesMap.get(type)!; diff --git a/packages/http-client-python/emitter/src/utils.ts b/packages/http-client-python/emitter/src/utils.ts index a920404c05..21de15d4fb 100644 --- a/packages/http-client-python/emitter/src/utils.ts +++ b/packages/http-client-python/emitter/src/utils.ts @@ -201,7 +201,11 @@ export function capitalize(name: string): string { } export function getClientNamespace(clientNamespace: string, rootNameSpace: string) { - if (["azure.core", "azure.resourcemanager"].some(item => clientNamespace.toLowerCase().startsWith(item))) { + if ( + ["azure.core", "azure.resourcemanager"].some((item) => + clientNamespace.toLowerCase().startsWith(item), + ) + ) { return rootNameSpace; } return clientNamespace; diff --git a/packages/http-client-python/generator/pygen/codegen/_utils.py b/packages/http-client-python/generator/pygen/codegen/_utils.py index 726ccc3f2f..8097eefbd9 100644 --- a/packages/http-client-python/generator/pygen/codegen/_utils.py +++ b/packages/http-client-python/generator/pygen/codegen/_utils.py @@ -15,3 +15,10 @@ TYPESPEC_PACKAGE_MODE = ["azure-mgmt", "azure-dataplane", "generic"] VALID_PACKAGE_MODE = SWAGGER_PACKAGE_MODE + TYPESPEC_PACKAGE_MODE NAME_LENGTH_LIMIT = 40 + + +def get_unique_alias(relative_import: str, module_name: str = "models") -> str: + pass + +def get_parent_namespace(namespace: str) -> str: + return namespace.rsplit(".", 1)[0] diff --git a/packages/http-client-python/generator/pygen/codegen/models/base_builder.py b/packages/http-client-python/generator/pygen/codegen/models/base_builder.py index 87d8b02353..ed9b450868 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/base_builder.py +++ b/packages/http-client-python/generator/pygen/codegen/models/base_builder.py @@ -72,6 +72,7 @@ def __init__( self.api_versions: List[str] = yaml_data["apiVersions"] self.added_on: Optional[str] = yaml_data.get("addedOn") self.external_docs: Optional[Dict[str, Any]] = yaml_data.get("externalDocs") + self.client_namespace: str = yaml_data.get("clientNamespace", code_model.namespace) if code_model.options["version_tolerant"] and yaml_data.get("abstract"): _LOGGER.warning( diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index b2fe0fda10..a99c5a4d10 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -43,6 +43,7 @@ def __init__( self.parameters = parameters self.url: str = self.yaml_data["url"] # the base endpoint of the client. Can be parameterized or not self.legacy_filename: str = self.yaml_data.get("legacyFilename", "client") + self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) @property def description(self) -> str: @@ -188,7 +189,7 @@ def lookup_operation(self, operation_id: int) -> "OperationType": except StopIteration as exc: raise KeyError(f"No operation with id {operation_id} found.") from exc - def _imports_shared(self, async_mode: bool) -> FileImport: + def _imports_shared(self, async_mode: bool, **kwargs) -> FileImport: file_import = FileImport(self.code_model) file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) if self.code_model.options["azure_arm"]: @@ -206,17 +207,19 @@ def _imports_shared(self, async_mode: bool) -> FileImport: file_import.merge( gp.imports( async_mode, - relative_path=".." if async_mode else ".", - operation=True, + in_operation_file=True, + **kwargs, ) ) file_import.add_submodule_import( - "._configuration", + "_configuration", f"{self.name}Configuration", ImportType.LOCAL, + client_namespace=self.client_namespace, ) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_msrest_import( - relative_path=".." if async_mode else ".", + serialize_namespace=serialize_namespace, msrest_import_type=MsrestImportType.SerializerDeserializer, typing_section=TypingSection.REGULAR, ) @@ -277,8 +280,8 @@ def has_non_abstract_operations(self) -> bool: """Whether there is non-abstract operation in any operation group.""" return any(og.has_non_abstract_operations for og in self.operation_groups) - def imports(self, async_mode: bool) -> FileImport: - file_import = self._imports_shared(async_mode) + def imports(self, async_mode: bool, **kwargs) -> FileImport: + file_import = self._imports_shared(async_mode, **kwargs) if async_mode: file_import.add_submodule_import("typing", "Awaitable", ImportType.STDLIB) file_import.add_submodule_import( @@ -302,9 +305,10 @@ def imports(self, async_mode: bool) -> FileImport: ) for og in self.operation_groups: file_import.add_submodule_import( - f".{self.code_model.operations_folder_name}", + f"{self.code_model.operations_folder_name(og.client_namespace)}", og.class_name, ImportType.LOCAL, + client_namespace=og.client_namespace, ) if self.code_model.model_types and self.code_model.options["models_mode"] == "msrest": @@ -317,8 +321,8 @@ def imports(self, async_mode: bool) -> FileImport: file_import.add_submodule_import("copy", "deepcopy", ImportType.STDLIB) return file_import - def imports_for_multiapi(self, async_mode: bool) -> FileImport: - file_import = self._imports_shared(async_mode) + def imports_for_multiapi(self, async_mode: bool, **kwargs) -> FileImport: + file_import = self._imports_shared(async_mode, **kwargs) file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB, TypingSection.CONDITIONAL) try: mixin_operation = next(og for og in self.operation_groups if og.is_mixin) @@ -377,7 +381,7 @@ def sdk_moniker(self) -> str: def name(self) -> str: return f"{super().name}Configuration" - def _imports_shared(self, async_mode: bool) -> FileImport: + def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) file_import.add_submodule_import( "pipeline" if self.code_model.is_azure_flavor else "runtime", @@ -386,7 +390,8 @@ def _imports_shared(self, async_mode: bool) -> FileImport: ) file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) if self.code_model.options["package_version"]: - file_import.add_submodule_import(".._version" if async_mode else "._version", "VERSION", ImportType.LOCAL) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + file_import.add_submodule_import(f"{self.code_model.get_relative_import_path(serialize_namespace)}._version", "VERSION", ImportType.LOCAL) if self.code_model.options["azure_arm"]: policy = "AsyncARMChallengeAuthenticationPolicy" if async_mode else "ARMChallengeAuthenticationPolicy" file_import.add_submodule_import("azure.mgmt.core.policies", "ARMHttpLoggingPolicy", ImportType.SDKCORE) @@ -394,22 +399,22 @@ def _imports_shared(self, async_mode: bool) -> FileImport: return file_import - def imports(self, async_mode: bool) -> FileImport: - file_import = self._imports_shared(async_mode) + def imports(self, async_mode: bool, **kwargs) -> FileImport: + file_import = self._imports_shared(async_mode, **kwargs) for gp in self.parameters: if gp.method_location == ParameterMethodLocation.KWARG and gp not in self.parameters.kwargs_to_pop: continue file_import.merge( gp.imports( async_mode=async_mode, - relative_path=".." if async_mode else ".", - operation=True, + in_operation_file=True, + **kwargs, ) ) return file_import - def imports_for_multiapi(self, async_mode: bool) -> FileImport: - file_import = self._imports_shared(async_mode) + def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport: + file_import = self._imports_shared(async_mode, **kwargs) for gp in self.parameters: if ( gp.method_location == ParameterMethodLocation.KWARG @@ -420,8 +425,8 @@ def imports_for_multiapi(self, async_mode: bool) -> FileImport: file_import.merge( gp.imports_for_multiapi( async_mode=async_mode, - relative_path=".." if async_mode else ".", - operation=True, + in_operation_file=True, + **kwargs, ) ) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 7ddb90fe37..daf92cdf9f 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import List, Dict, Any, Set, Union, Literal +from typing import List, Dict, Any, Set, Union, Literal, Optional from .base import BaseType from .enum_type import EnumType @@ -11,12 +11,27 @@ from .combined_type import CombinedType from .client import Client from .request_builder import RequestBuilder, OverloadedRequestBuilder - +from .operation_group import OperationGroup +from .utils import NamespaceType def _is_legacy(options) -> bool: return not (options.get("version_tolerant") or options.get("low_level_client")) +class ClientNamespaceType: + def __init__( + self, + client_namespace: str, + clients: List[Client], + models: List[ModelType], + operation_groups: List[OperationGroup], + ): + self.client_namespace = client_namespace + self.clients = clients + self.models = models + self.operation_groups = operation_groups + + class CodeModel: # pylint: disable=too-many-public-methods, disable=too-many-instance-attributes """Top level code model @@ -71,6 +86,31 @@ def __init__( ] self.cross_language_package_id = self.yaml_data.get("crossLanguagePackageId") self.for_test: bool = False + self._client_namespace_types: Dict[str, ClientNamespaceType] = {} + # | serialize_namespace | imported_namespace | relative_import_path | + # |----------------------|--------------------|----------------------| + # |azure.test.operations | azure.test | .. | + # |azure.test.operations | azure.test.subtest | ..subtest | + # |azure.test.operations | azure | ... | + # |azure.test.aio.operations | azure.test | ... | + # |azure.test.subtest.aio.operations|azure.test| .... | + # |azure.test |azure.test.subtest | .subtest | + def get_relative_import_path(self, serialize_namespace: str, imported_namespace: Optional[str] = None, *, namespace_type = Optional[NamespaceType] = None) -> str: + imported_namespace = self.namespace if imported_namespace is None else imported_namespace + idx = 0 + while idx < min(len(serialize_namespace), len(imported_namespace)): + if serialize_namespace[idx] != imported_namespace[idx]: + break + idx += 1 + return "." * (len(serialize_namespace[idx:].strip(".").split(".")) + 1) + imported_namespace[idx:].strip(".") + + @property + def client_namespace_types(self) -> Dict[str, ClientNamespaceType]: + if not self._client_namespace_types: + for client in self.clients: + # TODO + pass + return self._client_namespace_types @property def has_form_data(self) -> bool: @@ -130,12 +170,14 @@ def need_mixin_abc(self) -> bool: def has_abstract_operations(self) -> bool: return any(c for c in self.clients if c.has_abstract_operations) - @property - def operations_folder_name(self) -> str: + @staticmethod + def operations_folder_name(self, client_namespace: str) -> str: """Get the name of the operations folder that holds operations.""" name = "operations" + client_namespace_type = self.client_namespace_types.get(client_namespace) + operation_groups = client_namespace_type.operation_groups if client_namespace_type else [] if self.options["version_tolerant"] and not any( - og for client in self.clients for og in client.operation_groups if not og.is_mixin + og for client in self.clients for og in operation_groups if not og.is_mixin ): name = f"_{name}" return name diff --git a/packages/http-client-python/generator/pygen/codegen/models/combined_type.py b/packages/http-client-python/generator/pygen/codegen/models/combined_type.py index 6afbe08ac9..0d221280e4 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/combined_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/combined_type.py @@ -30,6 +30,7 @@ def __init__( self.types = types # the types that this type is combining self.name = yaml_data.get("name") self._is_union_of_literals = all(i.type == "constant" for i in self.types) + self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) @property def serialization_type(self) -> str: @@ -112,9 +113,10 @@ def instance_check_template(self) -> str: def imports(self, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) if self.name and not kwargs.get("is_types_file"): file_import.add_submodule_import( - kwargs.pop("relative_path"), + self.code_model.get_relative_import_path(serialize_namespace), "_types", ImportType.LOCAL, TypingSection.TYPING, diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 56e712654f..61521061e2 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -8,6 +8,7 @@ from .base import BaseType from .imports import FileImport, ImportType, TypingSection + if TYPE_CHECKING: from .code_model import CodeModel @@ -75,7 +76,14 @@ def imports(self, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) file_import.merge(self.value_type.imports(**kwargs)) file_import.add_submodule_import("typing", "Literal", ImportType.STDLIB, TypingSection.REGULAR) - file_import.add_submodule_import("._enums", self.enum_type.name, ImportType.LOCAL, TypingSection.REGULAR) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace, self.enum_type.client_namespace), + self.enum_type.name, + ImportType.LOCAL, + TypingSection.REGULAR, + client_namespace=self.enum_type.client_namespace, + ) return file_import @@ -124,6 +132,7 @@ def __init__( self.value_type = value_type self.internal: bool = self.yaml_data.get("internal", False) self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") + self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) def __lt__(self, other): return self.name.lower() < other.name.lower() @@ -212,27 +221,28 @@ def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "EnumT ) def imports(self, **kwargs: Any) -> FileImport: - operation = kwargs.pop("operation", False) + in_operation_file = kwargs.pop("in_operation_file", False) file_import = FileImport(self.code_model) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) if self.code_model.options["models_mode"]: file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) - if not operation: + if not in_operation_file: file_import.add_submodule_import( - "..", + self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), "models", ImportType.LOCAL, TypingSection.TYPING, alias="_models", ) - file_import.merge(self.value_type.imports(operation=operation, **kwargs)) + file_import.merge(self.value_type.imports(in_operation_file=in_operation_file, **kwargs)) relative_path = kwargs.pop("relative_path", None) if self.code_model.options["models_mode"] and relative_path: # add import for enums in operations file file_import.add_submodule_import( - relative_path, + self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), "models", ImportType.LOCAL, alias="_models", - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), + typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), # TODO ) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/imports.py b/packages/http-client-python/generator/pygen/codegen/models/imports.py index c1f827b19c..6dfa570f3a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/imports.py +++ b/packages/http-client-python/generator/pygen/codegen/models/imports.py @@ -5,6 +5,7 @@ # -------------------------------------------------------------------------- from enum import Enum, auto from typing import Dict, List, Optional, Tuple, Union, Set, TYPE_CHECKING +from .._utils import get_parent_namespace if TYPE_CHECKING: from .code_model import CodeModel @@ -44,6 +45,7 @@ def __init__( submodule_name: Optional[str] = None, alias: Optional[str] = None, version_modules: Optional[Tuple[Tuple[Tuple[int, int], str, Optional[str]]]] = None, + client_namespace: Optional[str] = None, # namespace where the imported model is ): self.typing_section = typing_section self.import_type = import_type @@ -54,6 +56,7 @@ def __init__( # It's a list of "python version, module_name, comments". # The python version is in form of (major, minor), for instance (3, 9) stands for py3.9. self.version_modules = version_modules + self.client_namespace = client_namespace def __eq__(self, other): try: @@ -122,6 +125,8 @@ def add_submodule_import( typing_section: TypingSection = TypingSection.REGULAR, alias: Optional[str] = None, version_modules: Optional[Tuple[Tuple[Tuple[int, int], str, Optional[str]]]] = None, + *, + client_namespace: Optional[str] = None, # namespace where the imported model is ) -> None: """Add an import to this import block.""" self._append_import( @@ -132,6 +137,7 @@ def add_submodule_import( submodule_name=submodule_name, alias=alias, version_modules=version_modules, + client_namespace=client_namespace or self.code_model.namespace, ) ) @@ -259,7 +265,7 @@ def to_dict( def add_msrest_import( self, *, - relative_path: str, + serialize_namespace: str, msrest_import_type: MsrestImportType, typing_section: TypingSection, ): @@ -271,21 +277,25 @@ def add_msrest_import( if msrest_import_type == MsrestImportType.SerializerDeserializer: self.add_submodule_import("msrest", "Deserializer", ImportType.THIRDPARTY, typing_section) else: + # _serialization.py is always in root namespace + imported_namespace = self.code_model.namespace if self.code_model.options["multiapi"]: - relative_path += "." + # for multiapi, the namespace is azure.mgmt.xxx.v20XX_XX_XX while _serialization.py is in azure.mgmt.xxx + imported_namespace = get_parent_namespace(imported_namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, imported_namespace) if msrest_import_type == MsrestImportType.Module: self.add_submodule_import(relative_path, "_serialization", ImportType.LOCAL, typing_section) else: self.add_submodule_import( - f"{relative_path}_serialization", + f"{relative_path}._serialization", "Serializer", ImportType.LOCAL, - typing_section, + typing_section ) if msrest_import_type == MsrestImportType.SerializerDeserializer: self.add_submodule_import( - f"{relative_path}_serialization", + f"{relative_path}._serialization", "Deserializer", ImportType.LOCAL, - typing_section, + typing_section ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index 84647a037c..c455e7a7eb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -132,8 +132,8 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ): # used in the case if initial operation returns none # but final call returns a model - relative_path = "..." if async_mode else ".." - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + file_import.add_submodule_import(f"{self.code_model.get_relative_import_path(serialize_namespace)}._model_base", "_deserialize", ImportType.LOCAL) file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) file_import.add_submodule_import("typing", "cast", ImportType.STDLIB) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 80e21252b4..b4ee652663 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -82,6 +82,7 @@ def __init__( self.snake_case_name: str = self.yaml_data["snakeCaseName"] self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") self.usage: int = self.yaml_data.get("usage", UsageFlags.Input.value | UsageFlags.Output.value) + self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) @property def is_usage_output(self) -> bool: @@ -298,23 +299,24 @@ def type_description(self) -> str: def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) - relative_path = kwargs.pop("relative_path", None) - if relative_path: - # add import for models in operations or _types file + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) + # add import for models in operations or _types file + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias="_models", + typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), # TODO, + client_namespace=self.client_namespace, + ) + if self.is_form_data: file_import.add_submodule_import( - relative_path, - "models", + self.code_model.get_relative_import_path(serialize_namespace), + "_model_base", ImportType.LOCAL, - alias="_models", typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), ) - if self.is_form_data: - file_import.add_submodule_import( - relative_path, - "_model_base", - ImportType.LOCAL, - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), - ) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 06631ca907..24a2c1e5d9 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -37,6 +37,7 @@ from .parameter_list import ParameterList from .model_type import ModelType from .base import BaseType +from .combined_type import CombinedType from .request_builder import OverloadedRequestBuilder, RequestBuilder from ...utils import xml_serializable, json_serializable, NAME_LENGTH_LIMIT @@ -217,17 +218,22 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: # pyl file_import = FileImport(self.code_model) file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) - response_types = [r.type_annotation(async_mode=async_mode, operation=self) for r in self.responses if r.type] + response_types = [r.type_annotation(async_mode=async_mode) for r in self.responses if r.type] if len(set(response_types)) > 1: file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) if self.added_on: + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{'.' if async_mode else ''}.._validation", + f"{self.code_model.get_relative_import_path(serialize_namespace)}._validation", "api_version_validation", ImportType.LOCAL, ) return file_import + @property + def need_import_iobase(self) -> bool: + return self.parameters.has_body and isinstance(self.parameters.body_parameter.type, CombinedType) + def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport: if self.abstract: return FileImport(self.code_model) @@ -236,15 +242,15 @@ def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import.merge( param.imports_for_multiapi( async_mode, - operation=self, + need_import_iobase=self.need_import_iobase, **kwargs, ) ) for response in self.responses: - file_import.merge(response.imports_for_multiapi(async_mode=async_mode, operation=self, **kwargs)) + file_import.merge(response.imports_for_multiapi(async_mode=async_mode, need_import_iobase=self.need_import_iobase, **kwargs)) if self.code_model.options["models_mode"]: for exception in self.exceptions: - file_import.merge(exception.imports_for_multiapi(async_mode=async_mode, operation=self, **kwargs)) + file_import.merge(exception.imports_for_multiapi(async_mode=async_mode, need_import_iobase=self.need_import_iobase, **kwargs)) return file_import @staticmethod @@ -271,6 +277,7 @@ def get_request_builder_import( self, request_builder: Union[RequestBuilder, OverloadedRequestBuilder], async_mode: bool, + serialize_namespace: str, ) -> FileImport: """Helper method to get a request builder import.""" file_import = FileImport(self.code_model) @@ -292,8 +299,10 @@ def get_request_builder_import( alias="rest", ) if self.code_model.options["builders_visibility"] == "embedded" and async_mode: + operations_folder_name = self.code_model.operations_folder_name(self.client_namespace) + client_namespace = self.client_namespace + operations_folder_name file_import.add_submodule_import( - f"...{self.code_model.operations_folder_name}.{self.filename}", + f"{self.code_model.get_relative_import_path(serialize_namespace, client_namespace)}.{operations_folder_name}.{self.filename}", request_builder.name, import_type=ImportType.LOCAL, ) @@ -304,24 +313,26 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements ) -> FileImport: if self.abstract: return FileImport(self.code_model) + + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import = self._imports_shared(async_mode, **kwargs) for param in self.parameters.method: file_import.merge( param.imports( async_mode, - operation=self, + need_import_iobase=self.need_import_iobase, **kwargs, ) ) for response in self.responses: - file_import.merge(response.imports(async_mode=async_mode, operation=self, **kwargs)) + file_import.merge(response.imports(async_mode=async_mode, need_import_iobase=self.need_import_iobase, **kwargs)) if self.code_model.options["models_mode"]: for exception in self.exceptions: file_import.merge(exception.imports(async_mode=async_mode, **kwargs)) if self.parameters.has_body and self.parameters.body_parameter.flattened: - file_import.merge(self.parameters.body_parameter.type.imports(operation=self, **kwargs)) + file_import.merge(self.parameters.body_parameter.type.imports(need_import_iobase=self.need_import_iobase, **kwargs)) if not async_mode: for param in self.parameters.headers: if param.wire_name.lower() == "repeatability-request-id": @@ -371,7 +382,6 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements if self.deprecated: file_import.add_import("warnings", ImportType.STDLIB) - relative_path = "..." if async_mode else ".." if self.has_etag: file_import.add_submodule_import( "exceptions", @@ -379,8 +389,8 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements ImportType.SDKCORE, ) if not async_mode: - file_import.add_submodule_import(f"{relative_path}_vendor", "prep_if_match", ImportType.LOCAL) - file_import.add_submodule_import(f"{relative_path}_vendor", "prep_if_none_match", ImportType.LOCAL) + file_import.add_submodule_import(f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", "prep_if_match", ImportType.LOCAL) + file_import.add_submodule_import(f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", "prep_if_none_match", ImportType.LOCAL) if async_mode: file_import.add_submodule_import( "rest", @@ -411,32 +421,33 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements "distributed_trace", ImportType.SDKCORE, ) - file_import.merge(self.get_request_builder_import(self.request_builder, async_mode)) + file_import.merge(self.get_request_builder_import(self.request_builder, async_mode, serialize_namespace)) if self.overloads: file_import.add_submodule_import("typing", "overload", ImportType.STDLIB) if self.code_model.options["models_mode"] == "dpg": + relative_path = self.code_model.get_relative_import_path(serialize_namespace) if self.parameters.has_body: if self.has_form_data_body: file_import.add_submodule_import(relative_path, "_model_base", ImportType.LOCAL) elif xml_serializable(self.parameters.body_parameter.default_content_type): file_import.add_submodule_import( - f"{relative_path}_model_base", + f"{relative_path}._model_base", "_get_element", ImportType.LOCAL, ) elif json_serializable(self.parameters.body_parameter.default_content_type): file_import.add_submodule_import( - f"{relative_path}_model_base", + f"{relative_path}._model_base", "SdkJSONEncoder", ImportType.LOCAL, ) file_import.add_import("json", ImportType.STDLIB) if any(xml_serializable(str(r.default_content_type)) for r in self.responses): - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize_xml", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}._model_base", "_deserialize_xml", ImportType.LOCAL) elif any(r.type for r in self.responses): - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}._model_base", "_deserialize", ImportType.LOCAL) if self.default_error_deserialization or self.non_default_errors: - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}._model_base", "_deserialize", ImportType.LOCAL) return file_import def get_response_from_status(self, status_code: Optional[Union[str, int]]) -> ResponseType: diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index a6339a86bf..2bfd608be1 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -46,6 +46,7 @@ def __init__( for op_group in self.yaml_data.get("operationGroups", []) ] self.link_lro_initial_operations() + self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) @property def has_abstract_operations(self) -> bool: @@ -66,11 +67,11 @@ def base_class(self) -> str: base_classes.append(f"{self.client.name}MixinABC") return ", ".join(base_classes) - def imports_for_multiapi(self, async_mode: bool) -> FileImport: + def imports_for_multiapi(self, async_mode: bool, **kwargs) -> FileImport: file_import = FileImport(self.code_model) relative_path = ".." if async_mode else "." for operation in self.operations: - file_import.merge(operation.imports_for_multiapi(async_mode, relative_path=relative_path)) + file_import.merge(operation.imports_for_multiapi(async_mode, **kwargs)) if (self.code_model.model_types or self.code_model.enums) and self.code_model.options[ "models_mode" ] == "msrest": @@ -94,12 +95,13 @@ def need_validation(self) -> bool: """Whether any of its operations need validation""" return any(o for o in self.operations if o.need_validation) - def imports(self, async_mode: bool) -> FileImport: + def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) - relative_path = ("..." if async_mode else "..") + ("." if self.client.is_subclient else "") + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) for operation in self.operations: - file_import.merge(operation.imports(async_mode, relative_path=relative_path)) + file_import.merge(operation.imports(async_mode, **kwargs)) if not self.code_model.options["combine_operation_files"]: for og in self.operation_groups: file_import.add_submodule_import( @@ -108,16 +110,16 @@ def imports(self, async_mode: bool) -> FileImport: ImportType.LOCAL, ) # for multiapi - if ( - (self.code_model.public_model_types) - and self.code_model.options["models_mode"] == "msrest" - and not self.is_mixin - ): - file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") + # if ( + # (self.code_model.public_model_types) + # and self.code_model.options["models_mode"] == "msrest" + # and not self.is_mixin + # ): + # file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") if self.is_mixin: - file_import.add_submodule_import(".._vendor", f"{self.client.name}MixinABC", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}._vendor", f"{self.client.name}MixinABC", ImportType.LOCAL) if self.has_abstract_operations: - file_import.add_submodule_import(".._vendor", "raise_if_not_implemented", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}._vendor", "raise_if_not_implemented", ImportType.LOCAL) if all(o.abstract for o in self.operations): return file_import file_import.add_submodule_import("typing", "TypeVar", ImportType.STDLIB, TypingSection.CONDITIONAL) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index dd5ee82640..65d22952b9 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -145,10 +145,11 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ImportType.SDKCORE, ) if self.code_model.options["models_mode"] == "dpg": - relative_path = "..." if async_mode else ".." + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace) file_import.merge(self.item_type.imports(**kwargs)) if self.default_error_deserialization or any(r.type for r in self.responses): - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}._model_base", "_deserialize", ImportType.LOCAL) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter.py b/packages/http-client-python/generator/pygen/codegen/models/parameter.py index 7703c9f2e7..5ff32fdfbb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter.py @@ -159,19 +159,21 @@ def docstring_type(self, **kwargs: Any) -> str: def serialization_type(self) -> str: return self.type.serialization_type - def _imports_shared(self, async_mode: bool, **_: Any) -> FileImport: + def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) if self.optional and self.client_default_value is None: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace) if self.added_on and self.implementation != "Client": file_import.add_submodule_import( - f"{'.' if async_mode else ''}.._validation", + f"{relative_path}._validation", "api_version_validation", ImportType.LOCAL, ) if isinstance(self.type, CombinedType) and self.type.name: file_import.add_submodule_import( - "..." if async_mode else "..", + relative_path, "_types", ImportType.LOCAL, TypingSection.TYPING, @@ -272,9 +274,9 @@ def has_json_model_type(self) -> bool: def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import = super().imports(async_mode, **kwargs) if self.is_form_data: - relative_path = "..." if async_mode else ".." + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{relative_path}_vendor", + f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", "prepare_multipart_form_data", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py index bbb2bcf2af..caad87f0eb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py +++ b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py @@ -89,12 +89,7 @@ def imports(self, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) file_import.add_submodule_import("typing", "IO", ImportType.STDLIB) - operation = kwargs.get("operation") - if ( - isinstance(operation, OperationBase) - and operation.parameters.has_body - and isinstance(operation.parameters.body_parameter.type, CombinedType) - ): + if kwargs.get("need_import_iobase", False): file_import.add_submodule_import("io", "IOBase", ImportType.STDLIB) return file_import @@ -646,8 +641,9 @@ def docstring_type(self, **kwargs: Any) -> str: def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) - relative_path = "..." if kwargs.get("async_mode") else ".." - file_import.add_submodule_import(f"{relative_path}_vendor", self.name, ImportType.LOCAL) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace) + file_import.add_submodule_import(f"{relative_path}._vendor", self.name, ImportType.LOCAL) return file_import @property diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index 963437d94d..289ba1ee29 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -144,12 +144,13 @@ def imports(self, **kwargs) -> FileImport: file_import = FileImport(self.code_model) if self.is_discriminator and isinstance(self.type, EnumType): return file_import - file_import.merge(self.type.imports(**kwargs, relative_path="..", model_typing=True)) + file_import.merge(self.type.imports(**kwargs, model_typing=True)) if self.optional and self.client_default_value is None: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) - if self.code_model.options["models_mode"] == "dpg": + if self.code_model.options["models_mode"] == "dpg" and isinstance(self.type, ModelType): + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - ".._model_base", + f"{self.code_model.get_relative_import_path(serialize_namespace)}._model_base", "rest_discriminator" if self.is_discriminator else "rest_field", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/request_builder.py b/packages/http-client-python/generator/pygen/codegen/models/request_builder.py index 81e00d18b4..79e3e2809b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/request_builder.py +++ b/packages/http-client-python/generator/pygen/codegen/models/request_builder.py @@ -84,15 +84,12 @@ def response_docstring_text(self, **kwargs) -> str: def response_docstring_type(self, **kwargs) -> str: return f"~{self.code_model.core_library}.rest.HttpRequest" - def imports(self) -> FileImport: + def imports(self, **kwargs) -> FileImport: file_import = FileImport(self.code_model) - relative_path = ".." - if not self.code_model.options["builders_visibility"] == "embedded" and self.group_name: - relative_path = "..." if self.group_name else ".." if self.abstract: return file_import for parameter in self.parameters.method: - file_import.merge(parameter.imports(async_mode=False, relative_path=relative_path, operation=self)) + file_import.merge(parameter.imports(async_mode=False, **kwargs)) file_import.add_submodule_import( "rest", @@ -108,11 +105,7 @@ def imports(self) -> FileImport: ) file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, typing_section=TypingSection.CONDITIONAL) file_import.add_msrest_import( - relative_path=( - "..." - if (not self.code_model.options["builders_visibility"] == "embedded" and self.group_name) - else ".." - ), + serialize_namespace=kwargs.get("serialize_namespace", self.code_model.namespace), msrest_import_type=MsrestImportType.Serializer, typing_section=TypingSection.REGULAR, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/response.py b/packages/http-client-python/generator/pygen/codegen/models/response.py index 19a7d62e94..666605a71b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/response.py +++ b/packages/http-client-python/generator/pygen/codegen/models/response.py @@ -121,8 +121,9 @@ def _imports_shared(self, **kwargs: Any) -> FileImport: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) if isinstance(self.type, CombinedType) and self.type.name: async_mode = kwargs.get("async_mode", False) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - "..." if async_mode else "..", + self.code_model.get_relative_import_path(serialize_namespace), "_types", ImportType.LOCAL, TypingSection.TYPING, diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index e8472382e2..e786064959 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -4,6 +4,7 @@ # license information. # -------------------------------------------------------------------------- from typing import TypeVar, Dict +from enum import Enum T = TypeVar("T") OrderedSet = Dict[T, None] @@ -19,3 +20,9 @@ def add_to_pylint_disable(curr_str: str, entry: str) -> str: if curr_str: return f"{curr_str},{entry}" return f" # pylint: disable={entry}" + +class NamespaceType(str, Enum): + """Special signal for impports""" + MODEL = "model" + OPERATION = "operation" + diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py index 0ac623166a..16f3741622 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +from typing import Optional from jinja2 import Environment from ..models import ( FileImport, @@ -13,9 +14,10 @@ class BaseSerializer: """Base serializer for SDK root level files""" - def __init__(self, code_model: CodeModel, env: Environment): + def __init__(self, code_model: CodeModel, env: Environment, *, serialize_namespace: Optional[str] = None): self.code_model = code_model self.env = env + self.serialize_namespace = serialize_namespace or code_model.namespace def init_file_import(self) -> FileImport: return FileImport(self.code_model) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index 2a2c1c24d6..e597a740c8 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- import json -from typing import Any, List +from typing import Any, List, Optional from jinja2 import Environment from .import_serializer import FileImportSerializer, TypingSection from ..models.imports import MsrestImportType, FileImport @@ -21,8 +21,10 @@ class GeneralSerializer(BaseSerializer): """General serializer for SDK root level files""" - def __init__(self, code_model: CodeModel, env: Environment, async_mode: bool): - super().__init__(code_model, env) + def __init__( + self, code_model: CodeModel, env: Environment, async_mode: bool, *, serialize_namespace: Optional[str] = None + ): + super().__init__(code_model, env, serialize_namespace=serialize_namespace) self.async_mode = async_mode def serialize_setup_file(self) -> str: @@ -100,7 +102,7 @@ def serialize_vendor_file(self, clients: List[Client]) -> str: TypingSection.TYPING, ) file_import.add_msrest_import( - relative_path=".." if self.async_mode else ".", + serialize_namespace=self.serialize_namespace, msrest_import_type=MsrestImportType.SerializerDeserializer, typing_section=TypingSection.TYPING, ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 6ad0fab40a..449745fa2e 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -132,7 +132,7 @@ class MsrestModelSerializer(_ModelSerializer): def imports(self) -> FileImport: file_import = FileImport(self.code_model) file_import.add_msrest_import( - relative_path="..", + serialize_namespace=self.serialize_namespace, msrest_import_type=MsrestImportType.Module, typing_section=TypingSection.REGULAR, ) @@ -210,7 +210,7 @@ def super_call(self, model: ModelType) -> List[str]: def imports(self) -> FileImport: file_import = FileImport(self.code_model) file_import.add_submodule_import( - "..", + self.code_model.get_relative_import_path(self.serialize_namespace) "_model_base", ImportType.LOCAL, TypingSection.REGULAR, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py index 4c04efbcd2..b98cb5f0c0 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -32,8 +32,10 @@ def __init__( env: Environment, async_mode: bool, operation_group: Optional[OperationGroup] = None, + *, + serialize_namespace: Optional[str] = None, ): - super().__init__(code_model, env) + super().__init__(code_model, env, serialize_namespace=serialize_namespace) self.clients = clients self.async_mode = async_mode self.operation_group = operation_group diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py index f982984364..b178d66054 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import List +from typing import List, Optional from jinja2 import Environment from ..models import FileImport @@ -19,8 +19,10 @@ def __init__( code_model: CodeModel, env: Environment, request_builders: List[RequestBuilderType], + *, + serialize_namespace: Optional[str] = None, ) -> None: - super().__init__(code_model, env) + super().__init__(code_model, env, serialize_namespace=serialize_namespace) self.request_builders = request_builders self.group_name = request_builders[0].group_name diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py index 024c4d4034..a5c69713ed 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- import logging -from typing import Dict, Any, Union, Tuple +from typing import Dict, Any, Union, Tuple, Optional from jinja2 import Environment from ..models.operation import OperationBase @@ -34,8 +34,10 @@ def __init__( operation: OperationBase[Any], sample: Dict[str, Any], file_name: str, + *, + serialize_namespace: Optional[str] = None, ) -> None: - super().__init__(code_model, env) + super().__init__(code_model, env, serialize_namespace=serialize_namespace) self.operation_group = operation_group self.operation = operation self.sample = sample diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index 7ae55ab093..370b6554ce 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -137,8 +137,15 @@ def __init__( class TestGeneralSerializer(BaseSerializer): - def __init__(self, code_model: CodeModel, env: Environment, *, is_async: bool = False) -> None: - super().__init__(code_model, env) + def __init__( + self, + code_model: CodeModel, + env: Environment, + *, + is_async: bool = False, + serialize_namespace: Optional[str] = None, + ) -> None: + super().__init__(code_model, env, serialize_namespace=serialize_namespace) self.is_async = is_async @property diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 3e143f5209..9f5dc15499 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -18,7 +18,7 @@ def imports(self) -> FileImport: ImportType.STDLIB, ) for nu in self.code_model.named_unions: - file_import.merge(nu.imports(relative_path=".", model_typing=True, is_types_file=True)) + file_import.merge(nu.imports(model_typing=True, is_types_file=True, serialize_namespace=self.serialize_namespace)) return file_import def serialize(self) -> str: From 0a391bd432fd5b73cfb2b091e34595728a8d5748 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 26 Nov 2024 15:14:16 +0800 Subject: [PATCH 04/49] init2 --- .../emitter/src/code-model.ts | 5 - .../pygen/codegen/models/code_model.py | 15 +- .../pygen/codegen/models/operation_group.py | 3 + .../pygen/codegen/serializers/__init__.py | 444 +++++++++++++----- .../serializers/operations_init_serializer.py | 11 +- .../operations_folder_init.py.jinja2 | 6 +- .../generator/pygen/preprocess/__init__.py | 4 - 7 files changed, 343 insertions(+), 145 deletions(-) diff --git a/packages/http-client-python/emitter/src/code-model.ts b/packages/http-client-python/emitter/src/code-model.ts index ea0c956c0c..9de1bef03a 100644 --- a/packages/http-client-python/emitter/src/code-model.ts +++ b/packages/http-client-python/emitter/src/code-model.ts @@ -243,14 +243,9 @@ export function emitCodeModel( const codeModel: Record = { namespace: removeUnderscoresFromNamespace(sdkPackage.rootNamespace).toLowerCase(), clients: [], - subnamespaceToClients: {}, }; for (const client of sdkPackage.clients) { codeModel["clients"].push(emitClient(sdkContext, client)); - if (client.nameSpace === sdkPackage.rootNamespace) { - } else { - codeModel["subnamespaceToClients"][client.nameSpace] = emitClient(sdkContext, client); - } } // loop through models and enums since there may be some orphaned models needs to be generated for (const model of sdkPackage.models) { diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index daf92cdf9f..16ebfbb668 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -24,11 +24,13 @@ def __init__( client_namespace: str, clients: List[Client], models: List[ModelType], + enums: List[EnumType], operation_groups: List[OperationGroup], ): self.client_namespace = client_namespace self.clients = clients self.models = models + self.enums = enums self.operation_groups = operation_groups @@ -74,10 +76,6 @@ def __init__( self.clients: List[Client] = [ Client.from_yaml(client_yaml_data, self) for client_yaml_data in yaml_data["clients"] ] - self.subnamespace_to_clients: Dict[str, List[Client]] = { - subnamespace: [Client.from_yaml(client_yaml, self, is_subclient=True) for client_yaml in client_yamls] - for subnamespace, client_yamls in yaml_data.get("subnamespaceToClients", {}).items() - } if self.options["models_mode"] and self.model_types: self.sort_model_types() self.is_subnamespace = is_subnamespace @@ -87,6 +85,7 @@ def __init__( self.cross_language_package_id = self.yaml_data.get("crossLanguagePackageId") self.for_test: bool = False self._client_namespace_types: Dict[str, ClientNamespaceType] = {} + # | serialize_namespace | imported_namespace | relative_import_path | # |----------------------|--------------------|----------------------| # |azure.test.operations | azure.test | .. | @@ -122,15 +121,11 @@ def has_etag(self) -> bool: @property def has_operations(self) -> bool: - if any(c for c in self.clients if c.has_operations): - return True - return any(c for clients in self.subnamespace_to_clients.values() for c in clients if c.has_operations) + return any(c for c in self.clients if c.has_operations) @property def has_non_abstract_operations(self) -> bool: - return any(c for c in self.clients if c.has_non_abstract_operations) or any( - c for cs in self.subnamespace_to_clients.values() for c in cs if c.has_non_abstract_operations - ) + return any(c for c in self.clients if c.has_non_abstract_operations) def lookup_request_builder(self, request_builder_id: int) -> Union[RequestBuilder, OverloadedRequestBuilder]: """Find the request builder based off of id""" diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 2bfd608be1..363356e5dd 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -47,6 +47,9 @@ def __init__( ] self.link_lro_initial_operations() self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) + self.has_parent_operation_group: bool = False + for og in self.operation_groups: + og.has_parent_operation_group = True @property def has_abstract_operations(self) -> bool: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index c6ad4cee49..c45639c96e 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -15,6 +15,8 @@ OverloadedRequestBuilder, CodeModel, Client, + ModelType, + EnumType, ) from .enum_serializer import EnumSerializer from .general_serializer import GeneralSerializer @@ -65,7 +67,6 @@ def _sample_output_path(source_file_path: str) -> Path: return Path("/".join([to_snake_case(i) for i in after_examples.parts])) return Path("") - class JinjaSerializer(ReaderAndWriter): def __init__( self, @@ -144,54 +145,104 @@ def serialize(self) -> None: lstrip_blocks=True, ) - namespace_path = ( - Path(".") if self.code_model.options["no_namespace_folders"] else Path(*self._name_space().split(".")) - ) - - p = namespace_path.parent general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) - while p != Path("."): - # write pkgutil init file - self.write_file( - p / Path("__init__.py"), - general_serializer.serialize_pkgutil_init_file(), - ) - p = p.parent - # serialize main module - self._serialize_namespace_level( - env, - namespace_path, - [c for c in self.code_model.clients if c.has_operations], - ) - # serialize sub modules - for ( - subnamespace, - clients, - ) in self.code_model.subnamespace_to_clients.items(): - subnamespace_path = namespace_path / Path(subnamespace) - self._serialize_namespace_level(env, subnamespace_path, [c for c in clients if c.has_operations]) - - if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): - self._keep_patch_file(namespace_path / Path("models") / Path("_patch.py"), env) - - if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): - self._serialize_and_write_models_folder(env=env, namespace_path=namespace_path) - if not self.code_model.options["models_mode"]: - # keep models file if users ended up just writing a models file - if self.read_file(namespace_path / Path("models.py")): - self.write_file( - namespace_path / Path("models.py"), - self.read_file(namespace_path / Path("models.py")), - ) - if self.code_model.named_unions: - self.write_file( - namespace_path / Path("_types.py"), - TypesSerializer(code_model=self.code_model, env=env).serialize(), + for client_namespace, client_namespace_type in self.code_model.client_namespace_types.items(): + + if client_namespace == "": + # Write the setup file + if self.code_model.options["basic_setup_py"]: + self.write_file(Path("setup.py"), general_serializer.serialize_setup_file()) + + # add packaging files in root namespace (e.g. setup.py, README.md, etc.) + self._serialize_and_write_package_files(client_namespace) + + # add generated samples and generated tests + ... + + elif client_namespace_type.clients: + # add clients folder if there are clients in this namespace + self._serialize_client_and_config_files(client_namespace, general_serializer, client_namespace_type.clients, env) + else: + # add pkgutil init file if no clients in this namespace + self.write_file( + self.exec_path(client_namespace) / Path("__init__.py"), + general_serializer.serialize_pkgutil_init_file(), ) - - def _serialize_and_write_package_files(self, namespace_path: Path) -> None: - root_of_sdk = self._package_root_folder(namespace_path) + + # _model_base.py/_serialization.py/_vendor.py/py.typed/_types.py/_validation.py is always put in top level namespace + if client_namespace == self.code_model.namespace: + self._serialize_and_write_top_level_folder(env=env, namespace=client_namespace) + + # add models folder if there are models in this namespace + if (client_namespace_type.models or client_namespace_type.enums) and self.code_model.options["models_mode"]: + self._serialize_and_write_models_folder(env=env, namespace=client_namespace, models=client_namespace_type.models, enums=client_namespace_type.enums) + + if not self.code_model.options["models_mode"]: + # keep models file if users ended up just writing a models file + model_path = self.exec_path(client_namespace) / Path("models.py") + if self.read_file(model_path): + self.write_file( + model_path, + self.read_file(model_path), + ) + + # add operations folder if there are operations in this namespace + if client_namespace_type.operation_groups: + self._serialize_and_write_operations_folder(client_namespace_type.operation_groups, env=env, namespace=client_namespace) + if self.code_model.options["multiapi"]: + self._serialize_and_write_metadata(env=env, namespace_path=namespace_path) + + + + + + + + + + + # namespace_path = ( + # Path(".") if self.code_model.options["no_namespace_folders"] else Path(*self._name_space().split(".")) + # ) + + # p = namespace_path.parent + # general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) + # while p != Path("."): + # # write pkgutil init file + # self.write_file( + # p / Path("__init__.py"), + # general_serializer.serialize_pkgutil_init_file(), + # ) + # p = p.parent + + # # serialize main module + # self._serialize_namespace_level( + # env, + # namespace_path, + # [c for c in self.code_model.clients if c.has_operations], + # ) + + # if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): + # self._keep_patch_file(namespace_path / Path("models") / Path("_patch.py"), env) + + # if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): + # self._serialize_and_write_models_folder(env=env, namespace_path=namespace_path) + # if not self.code_model.options["models_mode"]: + # # keep models file if users ended up just writing a models file + # if self.read_file(namespace_path / Path("models.py")): + # self.write_file( + # namespace_path / Path("models.py"), + # self.read_file(namespace_path / Path("models.py")), + # ) + # if self.code_model.named_unions: + # self.write_file( + # namespace_path / Path("_types.py"), + # TypesSerializer(code_model=self.code_model, env=env).serialize(), + # ) + + def _serialize_and_write_package_files(self, namespace_path: str) -> None: + root_of_sdk = self.exec_path(namespace_path) if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: env = Environment( loader=PackageLoader("pygen.codegen", "templates/packaging_templates"), @@ -230,25 +281,49 @@ def _keep_patch_file(self, path_file: Path, env: Environment): PatchSerializer(env=env, code_model=self.code_model).serialize(), ) - def _serialize_and_write_models_folder(self, env: Environment, namespace_path: Path) -> None: + def _serialize_and_write_models_folder(self, env: Environment, namespace: str, models: List[ModelType], enums: List[EnumType]) -> None: # Write the models folder - models_path = namespace_path / Path("models") + models_namespace = namespace + ".models" + models_path = self.exec_path(models_namespace) serializer = DpgModelSerializer if self.code_model.options["models_mode"] == "dpg" else MsrestModelSerializer - if self.code_model.model_types: + if models: self.write_file( models_path / Path(f"{self.code_model.models_filename}.py"), - serializer(code_model=self.code_model, env=env).serialize(), + serializer(code_model=self.code_model, env=env, serialize_namespace=models_namespace).serialize(), ) - if self.code_model.enums: + if enums: self.write_file( models_path / Path(f"{self.code_model.enums_filename}.py"), - EnumSerializer(code_model=self.code_model, env=env).serialize(), + EnumSerializer(code_model=self.code_model, env=env, serialize_namespace=models_namespace).serialize(), ) self.write_file( models_path / Path("__init__.py"), ModelInitSerializer(code_model=self.code_model, env=env).serialize(), ) + self._keep_patch_file(models_path / Path("_patch.py"), env) + + + + # def _serialize_and_write_models_folder(self, env: Environment, namespace_path: Path) -> None: + # # Write the models folder + # models_path = namespace_path / Path("models") + # serializer = DpgModelSerializer if self.code_model.options["models_mode"] == "dpg" else MsrestModelSerializer + # if self.code_model.model_types: + # self.write_file( + # models_path / Path(f"{self.code_model.models_filename}.py"), + # serializer(code_model=self.code_model, env=env).serialize(), + # ) + # if self.code_model.enums: + # self.write_file( + # models_path / Path(f"{self.code_model.enums_filename}.py"), + # EnumSerializer(code_model=self.code_model, env=env).serialize(), + # ) + # self.write_file( + # models_path / Path("__init__.py"), + # ModelInitSerializer(code_model=self.code_model, env=env).serialize(), + # ) + def _serialize_and_write_rest_layer(self, env: Environment, namespace_path: Path) -> None: rest_path = namespace_path / Path(self.code_model.rest_layer_name) group_names = {rb.group_name for c in self.code_model.clients for rb in c.request_builders} @@ -327,26 +402,27 @@ def _serialize_and_write_operations_file( operation_group_async_serializer.serialize(), ) + def _serialize_and_write_operations_folder( - self, clients: List[Client], env: Environment, namespace_path: Path + self, operation_groups: List[OperationGroup], env: Environment, namespace: str ) -> None: + operations_folder_name = self.code_model.operations_folder_name(namespace) + # write sync operations init file operations_init_serializer = OperationsInitSerializer( - code_model=self.code_model, clients=clients, env=env, async_mode=False + code_model=self.code_model, operation_groups=operation_groups, env=env, async_mode=False ) self.write_file( - namespace_path / Path(self.code_model.operations_folder_name) / Path("__init__.py"), + self.exec_path(namespace) / operations_folder_name / Path("__init__.py"), operations_init_serializer.serialize(), ) # write async operations init file if self.has_aio_folder: - operations_async_init_serializer = OperationsInitSerializer( - code_model=self.code_model, clients=clients, env=env, async_mode=True - ) + operations_init_serializer.async_mode = True self.write_file( - namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("__init__.py"), - operations_async_init_serializer.serialize(), + self.exec_path(namespace) / "aio" / operations_folder_name / Path("__init__.py"), + operations_init_serializer.serialize(), ) if self.code_model.options["combine_operation_files"]: @@ -364,17 +440,54 @@ def _serialize_and_write_operations_folder( clients=clients, ) + # def _serialize_and_write_operations_folder( + # self, clients: List[Client], env: Environment, namespace_path: Path + # ) -> None: + # # write sync operations init file + # operations_init_serializer = OperationsInitSerializer( + # code_model=self.code_model, clients=clients, env=env, async_mode=False + # ) + # self.write_file( + # namespace_path / Path(self.code_model.operations_folder_name) / Path("__init__.py"), + # operations_init_serializer.serialize(), + # ) + + # # write async operations init file + # if self.has_aio_folder: + # operations_async_init_serializer = OperationsInitSerializer( + # code_model=self.code_model, clients=clients, env=env, async_mode=True + # ) + # self.write_file( + # namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("__init__.py"), + # operations_async_init_serializer.serialize(), + # ) + + # if self.code_model.options["combine_operation_files"]: + # self._serialize_and_write_operations_file( + # env=env, + # namespace_path=namespace_path, + # clients=clients, + # ) + # else: + # for operation_group in get_all_operation_groups_recursively(self.code_model.clients): + # self._serialize_and_write_operations_file( + # env=env, + # namespace_path=namespace_path, + # operation_group=operation_group, + # clients=clients, + # ) + def _serialize_and_write_version_file( self, - namespace_path: Path, + namespace: str, general_serializer: GeneralSerializer, ): def _read_version_file(original_version_file_name: str) -> str: - return self.read_file(namespace_path / original_version_file_name) + return self.read_file(self.exec_path(namespace) / original_version_file_name) def _write_version_file(original_version_file_name: str) -> None: self.write_file( - namespace_path / Path("_version.py"), + self.exec_path(namespace) / Path("_version.py"), _read_version_file(original_version_file_name), ) @@ -385,100 +498,191 @@ def _write_version_file(original_version_file_name: str) -> None: _write_version_file(original_version_file_name="version.py") elif self.code_model.options["package_version"]: self.write_file( - namespace_path / Path("_version.py"), + self.exec_path(namespace) / Path("_version.py"), general_serializer.serialize_version_file(), ) def _serialize_client_and_config_files( self, - namespace_path: Path, + namespace: str, general_serializer: GeneralSerializer, - async_mode: bool, clients: List[Client], + env: Environment, ) -> None: - if self.code_model.has_operations: - namespace_path = namespace_path / Path("aio") if async_mode else namespace_path - self.write_file( - namespace_path / Path(f"{self.code_model.client_filename}.py"), - general_serializer.serialize_service_client_file(clients), - ) - self.write_file( - namespace_path / Path("_configuration.py"), - general_serializer.serialize_config_file(clients), - ) + general_serializer.async_mode = False + self.write_file( + self.exec_path(namespace) / Path(f"{self.code_model.client_filename}.py"), + general_serializer.serialize_service_client_file(clients), + ) + self.write_file( + self.exec_path(namespace) / Path("_configuration.py"), + general_serializer.serialize_config_file(clients), + ) - def _serialize_and_write_top_level_folder( - self, env: Environment, namespace_path: Path, clients: List[Client] - ) -> None: - general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) + # if there was a patch file before, we keep it + self._keep_patch_file(self.exec_path(namespace) / Path("_patch.py"), env) + general_serializer.async_mode = True self.write_file( - namespace_path / Path("__init__.py"), - general_serializer.serialize_init_file(clients), + self.exec_path(namespace) / Path(f"aio/{self.code_model.client_filename}.py"), + general_serializer.serialize_service_client_file(clients), ) + self.write_file( + self.exec_path(namespace) / Path("aio/_configuration.py"), + general_serializer.serialize_config_file(clients), + ) + + if self.has_aio_folder: + self._keep_patch_file(self.exec_path(namespace) / Path("aio/_patch.py"), env) + + # def _serialize_client_and_config_files( + # self, + # namespace_path: Path, + # general_serializer: GeneralSerializer, + # async_mode: bool, + # clients: List[Client], + # ) -> None: + # if self.code_model.has_operations: + # namespace_path = namespace_path / Path("aio") if async_mode else namespace_path + # self.write_file( + # namespace_path / Path(f"{self.code_model.client_filename}.py"), + # general_serializer.serialize_service_client_file(clients), + # ) + # self.write_file( + # namespace_path / Path("_configuration.py"), + # general_serializer.serialize_config_file(clients), + # ) - # Write the service client - self._serialize_client_and_config_files(namespace_path, general_serializer, async_mode=False, clients=clients) + def _serialize_and_write_top_level_folder( + self, env: Environment, namespace: str + ) -> None: + general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) + + # write _vendor.py if self.code_model.need_vendored_code(async_mode=False): self.write_file( - namespace_path / Path("_vendor.py"), - general_serializer.serialize_vendor_file(clients), + self.exec_path(namespace) / Path("_vendor.py"), + general_serializer.serialize_vendor_file(self.code_model.clients), + ) + if self.code_model.need_vendored_code(async_mode=True): + self.write_file( + self.exec_path(namespace) / Path("aio/_vendor.py"), + general_serializer.serialize_vendor_file(self.code_model.clients), ) - self._serialize_and_write_version_file(namespace_path, general_serializer) + # write _version.py + self._serialize_and_write_version_file(namespace, general_serializer) # write the empty py.typed file - self.write_file(namespace_path / Path("py.typed"), "# Marker file for PEP 561.") + self.write_file(self.exec_path(namespace) / Path("py.typed"), "# Marker file for PEP 561.") + # write _serialization.py if not self.code_model.options["client_side_validation"] and not self.code_model.options["multiapi"]: self.write_file( - namespace_path / Path("_serialization.py"), + self.exec_path(namespace) / Path("_serialization.py"), general_serializer.serialize_serialization_file(), ) + + # write _model_base.py if self.code_model.options["models_mode"] == "dpg": self.write_file( - namespace_path / Path("_model_base.py"), + self.exec_path(namespace) / Path("_model_base.py"), general_serializer.serialize_model_base_file(), ) + # write _validation.py if any(og for client in self.code_model.clients for og in client.operation_groups if og.need_validation): self.write_file( - namespace_path / Path("_validation.py"), + self.exec_path(namespace) / Path("_validation.py"), general_serializer.serialize_validation_file(), ) + + # write apiview_mapping_python.json if self.code_model.options.get("emit_cross_language_definition_file"): self.write_file( - Path("./apiview_mapping_python.json"), + self.exec_path(namespace) / Path("apiview_mapping_python.json"), general_serializer.serialize_cross_language_definition_file(), ) - # Write the setup file - if self.code_model.options["basic_setup_py"]: - self.write_file(Path("setup.py"), general_serializer.serialize_setup_file()) - - def _serialize_and_write_aio_top_level_folder( - self, env: Environment, namespace_path: Path, clients: List[Client] - ) -> None: - aio_general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=True) - - aio_path = namespace_path / Path("aio") - - # Write the __init__ file - self.write_file( - aio_path / Path("__init__.py"), - aio_general_serializer.serialize_init_file(clients), - ) - - # Write the service client - self._serialize_client_and_config_files( - namespace_path, aio_general_serializer, async_mode=True, clients=clients - ) - if self.code_model.need_vendored_code(async_mode=True): + # write _types.py + if self.code_model.named_unions: self.write_file( - aio_path / Path("_vendor.py"), - aio_general_serializer.serialize_vendor_file(clients), + self.exec_path(namespace) / Path("_types.py"), + TypesSerializer(code_model=self.code_model, env=env).serialize(), ) + # def _serialize_and_write_top_level_folder( + # self, env: Environment, namespace_path: Path, clients: List[Client] + # ) -> None: + # general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) + + # self.write_file( + # namespace_path / Path("__init__.py"), + # general_serializer.serialize_init_file(clients), + # ) + + # # Write the service client + # self._serialize_client_and_config_files(namespace_path, general_serializer, async_mode=False, clients=clients) + # if self.code_model.need_vendored_code(async_mode=False): + # self.write_file( + # namespace_path / Path("_vendor.py"), + # general_serializer.serialize_vendor_file(clients), + # ) + + # self._serialize_and_write_version_file(namespace_path, general_serializer) + + # # write the empty py.typed file + # self.write_file(namespace_path / Path("py.typed"), "# Marker file for PEP 561.") + + # if not self.code_model.options["client_side_validation"] and not self.code_model.options["multiapi"]: + # self.write_file( + # namespace_path / Path("_serialization.py"), + # general_serializer.serialize_serialization_file(), + # ) + # if self.code_model.options["models_mode"] == "dpg": + # self.write_file( + # namespace_path / Path("_model_base.py"), + # general_serializer.serialize_model_base_file(), + # ) + + # if any(og for client in self.code_model.clients for og in client.operation_groups if og.need_validation): + # self.write_file( + # namespace_path / Path("_validation.py"), + # general_serializer.serialize_validation_file(), + # ) + # if self.code_model.options.get("emit_cross_language_definition_file"): + # self.write_file( + # Path("./apiview_mapping_python.json"), + # general_serializer.serialize_cross_language_definition_file(), + # ) + + # # Write the setup file + # if self.code_model.options["basic_setup_py"]: + # self.write_file(Path("setup.py"), general_serializer.serialize_setup_file()) + + # def _serialize_and_write_aio_top_level_folder( + # self, env: Environment, namespace_path: Path, clients: List[Client] + # ) -> None: + # aio_general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=True) + + # aio_path = namespace_path / Path("aio") + + # # Write the __init__ file + # self.write_file( + # aio_path / Path("__init__.py"), + # aio_general_serializer.serialize_init_file(clients), + # ) + + # # Write the service client + # self._serialize_client_and_config_files( + # namespace_path, aio_general_serializer, async_mode=True, clients=clients + # ) + # if self.code_model.need_vendored_code(async_mode=True): + # self.write_file( + # aio_path / Path("_vendor.py"), + # aio_general_serializer.serialize_vendor_file(clients), + # ) + def _serialize_and_write_metadata(self, env: Environment, namespace_path: Path) -> None: metadata_serializer = MetadataSerializer(self.code_model, env) self.write_file(namespace_path / Path("_metadata.json"), metadata_serializer.serialize()) @@ -493,6 +697,14 @@ def _name_space(self) -> str: return self._namespace_from_package_name + @property + def _exec_path_implimentation(self) -> Path: + """ Assume the process is running in the root folder of the package. If not, we need the path implementation.""" + return Path("../" * (self._name_space().count(".") + 1)) if self.code_model.options["no_namespace_folders"] else Path(".") + + def exec_path(self, namespace: str) -> Path: + return self._exec_path_implimentation / Path(*namespace.split(".")) + # find root folder where "setup.py" is def _package_root_folder(self, namespace_path: Path) -> Path: return namespace_path / Path("../" * (self._name_space().count(".") + 1)) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py index d0d30c03eb..75ebcdd1d4 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py @@ -7,19 +7,19 @@ from jinja2 import Environment from ..models.operation_group import OperationGroup -from ..models import CodeModel, Client +from ..models import CodeModel, OperationGroup class OperationsInitSerializer: def __init__( self, code_model: CodeModel, - clients: List[Client], + operation_groups: List[OperationGroup], env: Environment, async_mode: bool, ) -> None: self.code_model = code_model - self.clients = clients + self.operation_groups = [og for og in self.operation_groups if not og.has_parent_operation_group] self.env = env self.async_mode = async_mode @@ -29,8 +29,7 @@ def _get_filename(operation_group: OperationGroup) -> str: return [ f"from .{_get_filename(og)} import {og.class_name} # type: ignore" - for client in self.clients - for og in client.operation_groups + for og in self.operation_groups ] def serialize(self) -> str: @@ -40,5 +39,5 @@ def serialize(self) -> str: code_model=self.code_model, async_mode=self.async_mode, operation_group_imports=self.operation_group_imports, - clients=self.clients, + operation_groups=self.operation_groups, ) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 index e4ad6e4368..9fbe74c65c 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 @@ -7,11 +7,9 @@ {{ op_tools.serialize(operation_group_imports()) }} {{ keywords.patch_imports() }} __all__ = [ - {% for client in clients %} - {% for operation_group in client.operation_groups %} + {% for operation_group in operation_groups %} '{{ operation_group.class_name }}', - {% endfor %} - {% endfor %} + {% endfor %} ] {{ keywords.extend_all }} _patch_sdk() diff --git a/packages/http-client-python/generator/pygen/preprocess/__init__.py b/packages/http-client-python/generator/pygen/preprocess/__init__.py index cfddddb860..d1407930f0 100644 --- a/packages/http-client-python/generator/pygen/preprocess/__init__.py +++ b/packages/http-client-python/generator/pygen/preprocess/__init__.py @@ -501,10 +501,6 @@ def update_yaml(self, yaml_data: Dict[str, Any]) -> None: for client in yaml_data["clients"]: self.update_client(client) self.update_operation_groups(yaml_data, client) - for clients in yaml_data["subnamespaceToClients"].values(): - for client in clients: - self.update_client(client) - self.update_operation_groups(yaml_data, client) if yaml_data.get("namespace"): yaml_data["namespace"] = pad_builtin_namespaces(yaml_data["namespace"]) From 80ea4709e68794ab10860621f9b9b4d6d3271936 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 27 Nov 2024 17:38:21 +0800 Subject: [PATCH 05/49] init3 --- .../emitter/src/code-model.ts | 6 +- .../http-client-python/emitter/src/lib.ts | 2 + .../http-client-python/emitter/src/types.ts | 7 +- .../http-client-python/emitter/src/utils.ts | 8 +- .../generator/pygen/codegen/__init__.py | 2 + .../pygen/codegen/models/code_model.py | 41 +++- .../pygen/codegen/serializers/__init__.py | 199 +++++++++++------- .../operation_groups_serializer.py | 18 +- 8 files changed, 171 insertions(+), 112 deletions(-) diff --git a/packages/http-client-python/emitter/src/code-model.ts b/packages/http-client-python/emitter/src/code-model.ts index 9de1bef03a..f13cfe4d7b 100644 --- a/packages/http-client-python/emitter/src/code-model.ts +++ b/packages/http-client-python/emitter/src/code-model.ts @@ -169,7 +169,7 @@ function emitOperationGroups( propertyName: operationGroup.name, operations: operations, operationGroups: emitOperationGroups(context, operationGroup, rootClient, name), - clientNamespace: getClientNamespace(client.clientNamespace, context.sdkPackage.rootNamespace), + clientNamespace: getClientNamespace(context, client.clientNamespace), }); } } @@ -187,7 +187,7 @@ function emitOperationGroups( className: "", propertyName: "", operations: operations, - clientNamespace: getClientNamespace(client.clientNamespace, context.sdkPackage.rootNamespace), + clientNamespace: getClientNamespace(context, client.clientNamespace), }); } } @@ -231,7 +231,7 @@ function emitClient( url, apiVersions: client.apiVersions, arm: context.arm, - clientNamespace: getClientNamespace(client.clientNamespace, context.sdkPackage.rootNamespace), + clientNamespace: getClientNamespace(context, client.clientNamespace), }; } diff --git a/packages/http-client-python/emitter/src/lib.ts b/packages/http-client-python/emitter/src/lib.ts index 91aed8fb4b..8150253615 100644 --- a/packages/http-client-python/emitter/src/lib.ts +++ b/packages/http-client-python/emitter/src/lib.ts @@ -17,6 +17,7 @@ export interface PythonEmitterOptions { debug?: boolean; flavor?: "azure"; "examples-dir"?: string; + "enable-typespec-namespace"?: boolean; } export interface PythonSdkContext @@ -43,6 +44,7 @@ const EmitterOptionsSchema: JSONSchemaType = { debug: { type: "boolean", nullable: true }, flavor: { type: "string", nullable: true }, "examples-dir": { type: "string", nullable: true, format: "absolute-path" }, + "enable-typespec-namespace": { type: "boolean", nullable: true }, }, required: [], }; diff --git a/packages/http-client-python/emitter/src/types.ts b/packages/http-client-python/emitter/src/types.ts index 37161ef500..4f13a5b96f 100644 --- a/packages/http-client-python/emitter/src/types.ts +++ b/packages/http-client-python/emitter/src/types.ts @@ -295,7 +295,7 @@ function emitModel( usage: type.usage, isXml: type.usage & UsageFlags.Xml ? true : false, xmlMetadata: type.usage & UsageFlags.Xml ? getXmlMetadata(type) : undefined, - clientNamespace: getClientNamespace(type.clientNamespace, context.sdkPackage.rootNamespace), + clientNamespace: getClientNamespace(context, type.clientNamespace), }; typesMap.set(type, newValue); @@ -348,7 +348,6 @@ function emitEnum( type: "combined", types, xmlMetadata: {}, - clientNamespace: getClientNamespace(type.clientNamespace, context.sdkPackage.rootNamespace), }; } const values: Record[] = []; @@ -363,7 +362,7 @@ function emitEnum( values, xmlMetadata: {}, crossLanguageDefinitionId: type.crossLanguageDefinitionId, - clientNamespace: getClientNamespace(type.clientNamespace, context.sdkPackage.rootNamespace), + clientNamespace: getClientNamespace(context, type.clientNamespace), }; for (const value of type.values) { newValue.values.push(emitEnumMember(value, newValue)); @@ -481,7 +480,7 @@ function emitUnion( type: "combined", types: type.variantTypes.map((x) => getType(context, x)), xmlMetadata: {}, - clientNamespace: getClientNamespace(type.clientNamespace, context.sdkPackage.rootNamespace), + clientNamespace: getClientNamespace(context, type.clientNamespace), }); } diff --git a/packages/http-client-python/emitter/src/utils.ts b/packages/http-client-python/emitter/src/utils.ts index 21de15d4fb..6038ff6767 100644 --- a/packages/http-client-python/emitter/src/utils.ts +++ b/packages/http-client-python/emitter/src/utils.ts @@ -200,13 +200,17 @@ export function capitalize(name: string): string { return name[0].toUpperCase() + name.slice(1); } -export function getClientNamespace(clientNamespace: string, rootNameSpace: string) { +export function getClientNamespace(context: PythonSdkContext,clientNamespace: string) { + const options = context.emitContext.options + if ([undefined, false].includes(options["enable-typespec-namespace"])) { + return context.sdkPackage.rootNamespace; + } if ( ["azure.core", "azure.resourcemanager"].some((item) => clientNamespace.toLowerCase().startsWith(item), ) ) { - return rootNameSpace; + return context.sdkPackage.rootNamespace; } return clientNamespace; } diff --git a/packages/http-client-python/generator/pygen/codegen/__init__.py b/packages/http-client-python/generator/pygen/codegen/__init__.py index 11d829263c..3035f33e59 100644 --- a/packages/http-client-python/generator/pygen/codegen/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/__init__.py @@ -40,6 +40,7 @@ class OptionsRetriever: "generate-test": False, "from-typespec": False, "emit-cross-language-definition-file": False, + "enable-typespec-namespace": False, } @property @@ -299,6 +300,7 @@ def _build_code_model_options(self) -> Dict[str, Any]: "flavor", "company_name", "emit_cross_language_definition_file", + "enable_typespec_namespace", ] return {f: getattr(self.options_retriever, f) for f in flags} diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 16ebfbb668..31d57db39f 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -13,6 +13,7 @@ from .request_builder import RequestBuilder, OverloadedRequestBuilder from .operation_group import OperationGroup from .utils import NamespaceType +from ..serializers.utils import get_all_operation_groups_recursively def _is_legacy(options) -> bool: return not (options.get("version_tolerant") or options.get("low_level_client")) @@ -22,10 +23,10 @@ class ClientNamespaceType: def __init__( self, client_namespace: str, - clients: List[Client], - models: List[ModelType], - enums: List[EnumType], - operation_groups: List[OperationGroup], + clients: List[Client] = [], + models: List[ModelType] = [], + enums: List[EnumType] = [], + operation_groups: List[OperationGroup] = [], ): self.client_namespace = client_namespace self.clients = clients @@ -61,8 +62,6 @@ def __init__( self, yaml_data: Dict[str, Any], options: Dict[str, Any], - *, - is_subnamespace: bool = False, ) -> None: self.yaml_data = yaml_data self.options = options @@ -78,13 +77,13 @@ def __init__( ] if self.options["models_mode"] and self.model_types: self.sort_model_types() - self.is_subnamespace = is_subnamespace self.named_unions: List[CombinedType] = [ t for t in self.types_map.values() if isinstance(t, CombinedType) and t.name ] self.cross_language_package_id = self.yaml_data.get("crossLanguagePackageId") self.for_test: bool = False self._client_namespace_types: Dict[str, ClientNamespaceType] = {} + self.has_subnamespace = False # | serialize_namespace | imported_namespace | relative_import_path | # |----------------------|--------------------|----------------------| @@ -107,8 +106,32 @@ def get_relative_import_path(self, serialize_namespace: str, imported_namespace: def client_namespace_types(self) -> Dict[str, ClientNamespaceType]: if not self._client_namespace_types: for client in self.clients: - # TODO - pass + if client.client_namespace not in self._client_namespace_types: + self._client_namespace_types[client.client_namespace] = ClientNamespaceType(client.client_namespace) + self._client_namespace_types[client.client_namespace].clients.append(client) + for model in self.model_types: + if model.client_namespace not in self._client_namespace_types: + self._client_namespace_types[model.client_namespace] = ClientNamespaceType(model.client_namespace) + self._client_namespace_types[model.client_namespace].models.append(model) + for enum in self.enums: + if enum.client_namespace not in self._client_namespace_types: + self._client_namespace_types[enum.client_namespace] = ClientNamespaceType(enum.client_namespace) + self._client_namespace_types[enum.client_namespace].enums.append(enum) + for operation_group in get_all_operation_groups_recursively(self.clients): + if operation_group.client_namespace not in self._client_namespace_types: + self._client_namespace_types[operation_group.client_namespace] = ClientNamespaceType(operation_group.client_namespace) + self._client_namespace_types[operation_group.client_namespace].operation_groups.append(operation_group) + + if len(self._client_namespace_types.keys()) > 1: + self.has_subnamespace = True + + # insert namespace to make sure it is continuous(e.g. ("", "azure", "azure.mgmt", "azure.mgmt.service", ...)) + longest_namespace = sorted(self._client_namespace_types.keys())[-1] + namespace_parts = longest_namespace.split(".") + for idx in range(len(namespace_parts) + 1): + namespace = ".".join(namespace_parts[:idx]) + if namespace not in self._client_namespace_types: + self._client_namespace_types[namespace] = ClientNamespaceType(namespace) return self._client_namespace_types @property diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index c45639c96e..82a00e72d0 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -86,54 +86,54 @@ def has_aio_folder(self) -> bool: def has_operations_folder(self) -> bool: return self.code_model.options["show_operations"] and bool(self.code_model.has_operations) - def _serialize_namespace_level(self, env: Environment, namespace_path: Path, clients: List[Client]) -> None: - # if there was a patch file before, we keep it - self._keep_patch_file(namespace_path / Path("_patch.py"), env) - if self.has_aio_folder: - self._keep_patch_file(namespace_path / Path("aio") / Path("_patch.py"), env) + # def _serialize_namespace_level(self, env: Environment, namespace_path: Path, clients: List[Client]) -> None: + # # if there was a patch file before, we keep it + # self._keep_patch_file(namespace_path / Path("_patch.py"), env) + # if self.has_aio_folder: + # self._keep_patch_file(namespace_path / Path("aio") / Path("_patch.py"), env) - if self.has_operations_folder: - self._keep_patch_file( - namespace_path / Path(self.code_model.operations_folder_name) / Path("_patch.py"), - env, - ) - if self.has_aio_folder: - self._keep_patch_file( - namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("_patch.py"), - env, - ) - self._serialize_and_write_top_level_folder(env=env, namespace_path=namespace_path, clients=clients) + # if self.has_operations_folder: + # self._keep_patch_file( + # namespace_path / Path(self.code_model.operations_folder_name) / Path("_patch.py"), + # env, + # ) + # if self.has_aio_folder: + # self._keep_patch_file( + # namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("_patch.py"), + # env, + # ) + # self._serialize_and_write_top_level_folder(env=env, namespace_path=namespace_path, clients=clients) - if any(c for c in self.code_model.clients if c.operation_groups): - if self.code_model.options["builders_visibility"] != "embedded": - self._serialize_and_write_rest_layer(env=env, namespace_path=namespace_path) - if self.has_aio_folder: - self._serialize_and_write_aio_top_level_folder( - env=env, - namespace_path=namespace_path, - clients=clients, - ) + # if any(c for c in self.code_model.clients if c.operation_groups): + # if self.code_model.options["builders_visibility"] != "embedded": + # self._serialize_and_write_rest_layer(env=env, namespace_path=namespace_path) + # if self.has_aio_folder: + # self._serialize_and_write_aio_top_level_folder( + # env=env, + # namespace_path=namespace_path, + # clients=clients, + # ) - if self.has_operations_folder: - self._serialize_and_write_operations_folder(clients, env=env, namespace_path=namespace_path) - if self.code_model.options["multiapi"]: - self._serialize_and_write_metadata(env=env, namespace_path=namespace_path) - if self.code_model.options["package_mode"]: - self._serialize_and_write_package_files(namespace_path=namespace_path) - - if ( - self.code_model.options["show_operations"] - and self.code_model.has_operations - and self.code_model.options["generate_sample"] - ): - self._serialize_and_write_sample(env, namespace_path) - - if ( - self.code_model.options["show_operations"] - and self.code_model.has_operations - and self.code_model.options["generate_test"] - ): - self._serialize_and_write_test(env, namespace_path) + # if self.has_operations_folder: + # self._serialize_and_write_operations_folder(clients, env=env, namespace_path=namespace_path) + # if self.code_model.options["multiapi"]: + # self._serialize_and_write_metadata(env=env, namespace_path=namespace_path) + # if self.code_model.options["package_mode"]: + # self._serialize_and_write_package_files(namespace_path=namespace_path) + + # if ( + # self.code_model.options["show_operations"] + # and self.code_model.has_operations + # and self.code_model.options["generate_sample"] + # ): + # self._serialize_and_write_sample(env, namespace_path) + + # if ( + # self.code_model.options["show_operations"] + # and self.code_model.has_operations + # and self.code_model.options["generate_test"] + # ): + # self._serialize_and_write_test(env, namespace_path) def serialize(self) -> None: env = Environment( @@ -148,18 +148,23 @@ def serialize(self) -> None: general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) for client_namespace, client_namespace_type in self.code_model.client_namespace_types.items(): - if client_namespace == "": # Write the setup file if self.code_model.options["basic_setup_py"]: - self.write_file(Path("setup.py"), general_serializer.serialize_setup_file()) + self.write_file(self.exec_path(client_namespace) / Path("setup.py"), general_serializer.serialize_setup_file()) # add packaging files in root namespace (e.g. setup.py, README.md, etc.) self._serialize_and_write_package_files(client_namespace) # add generated samples and generated tests - ... - + if ( + self.code_model.options["show_operations"] + and self.code_model.has_operations + ): + if self.code_model.options["generate_sample"]: + self._serialize_and_write_sample(env, namespace=client_namespace) + if self.code_model.options["generate_test"]: + self._serialize_and_write_test(env, namespace=client_namespace) elif client_namespace_type.clients: # add clients folder if there are clients in this namespace self._serialize_client_and_config_files(client_namespace, general_serializer, client_namespace_type.clients, env) @@ -182,16 +187,13 @@ def serialize(self) -> None: # keep models file if users ended up just writing a models file model_path = self.exec_path(client_namespace) / Path("models.py") if self.read_file(model_path): - self.write_file( - model_path, - self.read_file(model_path), - ) + self.write_file(model_path, self.read_file(model_path)) # add operations folder if there are operations in this namespace if client_namespace_type.operation_groups: self._serialize_and_write_operations_folder(client_namespace_type.operation_groups, env=env, namespace=client_namespace) if self.code_model.options["multiapi"]: - self._serialize_and_write_metadata(env=env, namespace_path=namespace_path) + self._serialize_and_write_metadata(env=env, namespace=client_namespace) @@ -241,8 +243,8 @@ def serialize(self) -> None: # TypesSerializer(code_model=self.code_model, env=env).serialize(), # ) - def _serialize_and_write_package_files(self, namespace_path: str) -> None: - root_of_sdk = self.exec_path(namespace_path) + def _serialize_and_write_package_files(self, namespace: str) -> None: + root_of_sdk = self.exec_path(namespace) if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: env = Environment( loader=PackageLoader("pygen.codegen", "templates/packaging_templates"), @@ -370,44 +372,80 @@ def _serialize_and_write_single_rest_layer( def _serialize_and_write_operations_file( self, env: Environment, - clients: List[Client], - namespace_path: Path, + operation_groups: List[OperationGroup], + namespace: str, operation_group: Optional[OperationGroup] = None, ) -> None: filename = operation_group.filename if operation_group else "_operations" # write first sync file operation_group_serializer = OperationGroupsSerializer( code_model=self.code_model, - clients=clients, + operation_groups=operation_groups if operation_groups else [operation_group], env=env, async_mode=False, - operation_group=operation_group, ) + operations_folder_name = self.code_model.operations_folder_name(namespace) self.write_file( - namespace_path / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py"), + self.exec_path(namespace) / Path(operations_folder_name) / Path(f"{filename}.py"), operation_group_serializer.serialize(), ) if self.has_aio_folder: # write async operation group and operation files - operation_group_async_serializer = OperationGroupsSerializer( - code_model=self.code_model, - clients=clients, - env=env, - async_mode=True, - operation_group=operation_group, - ) + operation_group_serializer.async_mode = True self.write_file( - (namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py")), - operation_group_async_serializer.serialize(), + (self.exec_path(namespace) / Path("aio") / Path(operations_folder_name) / Path(f"{filename}.py")), + operation_group_serializer.serialize(), ) + # def _serialize_and_write_operations_file( + # self, + # env: Environment, + # clients: List[Client], + # namespace_path: Path, + # operation_group: Optional[OperationGroup] = None, + # ) -> None: + # filename = operation_group.filename if operation_group else "_operations" + # # write first sync file + # operation_group_serializer = OperationGroupsSerializer( + # code_model=self.code_model, + # clients=clients, + # env=env, + # async_mode=False, + # operation_group=operation_group, + # ) + # self.write_file( + # namespace_path / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py"), + # operation_group_serializer.serialize(), + # ) + + # if self.has_aio_folder: + # # write async operation group and operation files + # operation_group_async_serializer = OperationGroupsSerializer( + # code_model=self.code_model, + # clients=clients, + # env=env, + # async_mode=True, + # operation_group=operation_group, + # ) + # self.write_file( + # (namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py")), + # operation_group_async_serializer.serialize(), + # ) + + def _serialize_and_write_operations_folder( self, operation_groups: List[OperationGroup], env: Environment, namespace: str ) -> None: operations_folder_name = self.code_model.operations_folder_name(namespace) + + # if there was a patch file before, we keep it + self._keep_patch_file(self.exec_path(namespace) / Path("_patch.py"), env) + if self.has_aio_folder: + self._keep_patch_file(self.exec_path(namespace) / Path("aio") / Path("_patch.py"), env) + # write sync operations init file operations_init_serializer = OperationsInitSerializer( code_model=self.code_model, operation_groups=operation_groups, env=env, async_mode=False @@ -428,16 +466,15 @@ def _serialize_and_write_operations_folder( if self.code_model.options["combine_operation_files"]: self._serialize_and_write_operations_file( env=env, - namespace_path=namespace_path, - clients=clients, + namespace=namespace, + operation_groups=operation_groups, ) else: - for operation_group in get_all_operation_groups_recursively(self.code_model.clients): + for operation_group in operation_groups: self._serialize_and_write_operations_file( env=env, - namespace_path=namespace_path, + namespace=namespace, operation_group=operation_group, - clients=clients, ) # def _serialize_and_write_operations_folder( @@ -683,9 +720,9 @@ def _serialize_and_write_top_level_folder( # aio_general_serializer.serialize_vendor_file(clients), # ) - def _serialize_and_write_metadata(self, env: Environment, namespace_path: Path) -> None: + def _serialize_and_write_metadata(self, env: Environment, namespace: str) -> None: metadata_serializer = MetadataSerializer(self.code_model, env) - self.write_file(namespace_path / Path("_metadata.json"), metadata_serializer.serialize()) + self.write_file(self.exec_path(namespace) / Path("_metadata.json"), metadata_serializer.serialize()) @property def _namespace_from_package_name(self) -> str: @@ -718,8 +755,8 @@ def _additional_folder(self) -> Path: return Path("/".join(namespace_config.split(".")[num_of_package_namespace:])) return Path("") - def _serialize_and_write_sample(self, env: Environment, namespace_path: Path): - out_path = self._package_root_folder(namespace_path) / Path("generated_samples") + def _serialize_and_write_sample(self, env: Environment, namespace: str): + out_path = self.exec_path(namespace) / Path("generated_samples") for client in self.code_model.clients: for op_group in client.operation_groups: for operation in op_group.operations: @@ -751,9 +788,9 @@ def _serialize_and_write_sample(self, env: Environment, namespace_path: Path): log_error = f"error happens in sample {file}: {e}" _LOGGER.error(log_error) - def _serialize_and_write_test(self, env: Environment, namespace_path: Path): + def _serialize_and_write_test(self, env: Environment, namespace: Path): self.code_model.for_test = True - out_path = self._package_root_folder(namespace_path) / Path("generated_tests") + out_path = self.exec_path(namespace) / Path("generated_tests") general_serializer = TestGeneralSerializer(code_model=self.code_model, env=env) self.write_file(out_path / "conftest.py", general_serializer.serialize_conftest()) if not self.code_model.options["azure_arm"]: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py index b98cb5f0c0..75fae457ca 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -28,25 +28,22 @@ class OperationGroupsSerializer(BaseSerializer): def __init__( self, code_model: CodeModel, - clients: List[Client], + operation_groups: List[OperationGroup], env: Environment, async_mode: bool, - operation_group: Optional[OperationGroup] = None, *, serialize_namespace: Optional[str] = None, ): super().__init__(code_model, env, serialize_namespace=serialize_namespace) - self.clients = clients + self.operation_groups = operation_groups self.async_mode = async_mode - self.operation_group = operation_group def _get_request_builders( self, operation_group: OperationGroup ) -> List[Union[OverloadedRequestBuilder, RequestBuilder]]: return [ r - for client in self.clients - for r in client.request_builders + for r in operation_group.client.request_builders if r.client.name == operation_group.client.name and r.group_name == operation_group.identify_name and not r.is_overload @@ -55,13 +52,8 @@ def _get_request_builders( ] def serialize(self) -> str: - if self.operation_group: - operation_groups = [self.operation_group] - else: - operation_groups = get_all_operation_groups_recursively(self.clients) - imports = FileImport(self.code_model) - for operation_group in operation_groups: + for operation_group in self.operation_groups: imports.merge( operation_group.imports( async_mode=self.async_mode, @@ -72,7 +64,7 @@ def serialize(self) -> str: return template.render( code_model=self.code_model, - operation_groups=operation_groups, + operation_groups=self.operation_groups, imports=FileImportSerializer( imports, async_mode=self.async_mode, From d7f6e87a5d51645b30fdc8b541fb9d861dd4633a Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 2 Dec 2024 18:16:47 +0800 Subject: [PATCH 06/49] init4 --- .../generator/pygen/codegen/models/base.py | 5 +- .../pygen/codegen/models/code_model.py | 46 +++++++++++++----- .../pygen/codegen/models/combined_type.py | 5 +- .../pygen/codegen/models/constant_type.py | 5 +- .../pygen/codegen/models/credential_types.py | 3 +- .../pygen/codegen/models/dictionary_type.py | 5 +- .../pygen/codegen/models/enum_type.py | 18 +++---- .../pygen/codegen/models/list_type.py | 5 +- .../pygen/codegen/models/model_type.py | 21 ++++---- .../pygen/codegen/models/operation.py | 6 +-- .../pygen/codegen/models/operation_group.py | 18 +++---- .../pygen/codegen/models/parameter.py | 5 +- .../pygen/codegen/models/primitive_types.py | 48 +++++++------------ .../pygen/codegen/models/property.py | 5 +- .../pygen/codegen/models/response.py | 10 ++-- .../generator/pygen/codegen/models/utils.py | 1 + .../codegen/serializers/builder_serializer.py | 41 ++++++++-------- .../codegen/serializers/client_serializer.py | 10 ++-- .../codegen/serializers/general_serializer.py | 2 + .../operation_groups_serializer.py | 1 + .../serializers/parameter_serializer.py | 9 ++-- .../templates/client_container.py.jinja2 | 2 +- .../templates/config_container.py.jinja2 | 2 +- 23 files changed, 140 insertions(+), 133 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/base.py b/packages/http-client-python/generator/pygen/codegen/models/base.py index 516cade3bc..c6bb12e85e 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/base.py +++ b/packages/http-client-python/generator/pygen/codegen/models/base.py @@ -87,8 +87,7 @@ def xml_serialization_ctxt(self) -> Optional[str]: attrs_list.append("'text': True") return ", ".join(attrs_list) - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: """The tag recognized by 'msrest' as a serialization/deserialization. 'str', 'int', 'float', 'bool' or @@ -103,7 +102,7 @@ def serialization_type(self) -> str: @property def msrest_deserialization_key(self) -> str: - return self.serialization_type + return self.serialization_type() @property def client_default_value(self) -> Any: diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 31d57db39f..2bc7953939 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -84,6 +84,7 @@ def __init__( self.for_test: bool = False self._client_namespace_types: Dict[str, ClientNamespaceType] = {} self.has_subnamespace = False + self._operations_folder_name: Dict[str, str] = {} # | serialize_namespace | imported_namespace | relative_import_path | # |----------------------|--------------------|----------------------| @@ -93,8 +94,18 @@ def __init__( # |azure.test.aio.operations | azure.test | ... | # |azure.test.subtest.aio.operations|azure.test| .... | # |azure.test |azure.test.subtest | .subtest | - def get_relative_import_path(self, serialize_namespace: str, imported_namespace: Optional[str] = None, *, namespace_type = Optional[NamespaceType] = None) -> str: - imported_namespace = self.namespace if imported_namespace is None else imported_namespace + def get_relative_import_path(self, serialize_namespace: str, imported_namespace: Optional[str] = None, *, namespace_type: NamespaceType = NamespaceType.MODEL, async_mode: bool = False) -> str: + if imported_namespace is None: + imported_namespace = self.namespace + else: + async_namespace = ".aio" if async_mode else "" + if namespace_type == NamespaceType.MODEL: + module_namespace = ".models" + elif namespace_type == NamespaceType.OPERATION: + module_namespace = f".{self.operations_folder_name(imported_namespace)}" + else: + module_namespace = "" + imported_namespace = imported_namespace + async_namespace + module_namespace idx = 0 while idx < min(len(serialize_namespace), len(imported_namespace)): if serialize_namespace[idx] != imported_namespace[idx]: @@ -102,6 +113,18 @@ def get_relative_import_path(self, serialize_namespace: str, imported_namespace: idx += 1 return "." * (len(serialize_namespace[idx:].strip(".").split(".")) + 1) + imported_namespace[idx:].strip(".") + @property + def need_unique_model_alias(self) -> bool: + return self.options["enable_typespec_namespace"] and self.has_subnamespace + + def get_unique_models_alias(self, serialize_namespace: str, imported_namespace: str) -> str: + if not self.need_unique_model_alias: + return "_models." + relative_path = self.get_relative_import_path(serialize_namespace, imported_namespace) + dot_num = max(relative_path.count(".") - 1, 0) + parts = [""] + [p for p in relative_path.split(".") if p] + return "_".join(parts) + str(dot_num) + @property def client_namespace_types(self) -> Dict[str, ClientNamespaceType]: if not self._client_namespace_types: @@ -188,17 +211,18 @@ def need_mixin_abc(self) -> bool: def has_abstract_operations(self) -> bool: return any(c for c in self.clients if c.has_abstract_operations) - @staticmethod def operations_folder_name(self, client_namespace: str) -> str: """Get the name of the operations folder that holds operations.""" - name = "operations" - client_namespace_type = self.client_namespace_types.get(client_namespace) - operation_groups = client_namespace_type.operation_groups if client_namespace_type else [] - if self.options["version_tolerant"] and not any( - og for client in self.clients for og in operation_groups if not og.is_mixin - ): - name = f"_{name}" - return name + if client_namespace not in self._operations_folder_name: + name = "operations" + client_namespace_type = self.client_namespace_types.get(client_namespace) + operation_groups = client_namespace_type.operation_groups if client_namespace_type else [] + if self.options["version_tolerant"] and not any( + og for client in self.clients for og in operation_groups if not og.is_mixin + ): + name = f"_{name}" + self._operations_folder_name[client_namespace] = name + return self._operations_folder_name[client_namespace] @property def description(self) -> str: diff --git a/packages/http-client-python/generator/pygen/codegen/models/combined_type.py b/packages/http-client-python/generator/pygen/codegen/models/combined_type.py index 0d221280e4..88452b5db1 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/combined_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/combined_type.py @@ -32,8 +32,7 @@ def __init__( self._is_union_of_literals = all(i.type == "constant" for i in self.types) self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: """The tag recognized by 'msrest' as a serialization/deserialization. 'str', 'int', 'float', 'bool' or @@ -46,7 +45,7 @@ def serialization_type(self) -> str: """ if not all(t for t in self.types if t.type == "constant"): raise ValueError("Shouldn't get serialization type of a combinedtype") - return self.types[0].serialization_type + return self.types[0].serialization_type(**kwargs) @property def client_default_value(self) -> Any: diff --git a/packages/http-client-python/generator/pygen/codegen/models/constant_type.py b/packages/http-client-python/generator/pygen/codegen/models/constant_type.py index 02e1bfaca2..b99b469a16 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/constant_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/constant_type.py @@ -56,14 +56,13 @@ def description(self, *, is_operation_file: bool) -> str: f"Default value is {self.get_declaration()}.", ) - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: """Returns the serialization value for msrest. :return: The serialization value for msrest :rtype: str """ - return self.value_type.serialization_type + return self.value_type.serialization_type(**kwargs) def docstring_text(self, **kwargs: Any) -> str: return "constant" diff --git a/packages/http-client-python/generator/pygen/codegen/models/credential_types.py b/packages/http-client-python/generator/pygen/codegen/models/credential_types.py index 82dd74302a..7dd617235d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/credential_types.py +++ b/packages/http-client-python/generator/pygen/codegen/models/credential_types.py @@ -131,8 +131,7 @@ def get_json_template_representation( def docstring_text(self, **kwargs: Any) -> str: return "credential" - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return self.docstring_type() @classmethod diff --git a/packages/http-client-python/generator/pygen/codegen/models/dictionary_type.py b/packages/http-client-python/generator/pygen/codegen/models/dictionary_type.py index 38b83dcbed..e467c626e7 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/dictionary_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/dictionary_type.py @@ -34,14 +34,13 @@ def __init__( def encode(self) -> Optional[str]: return self.element_type.encode if hasattr(self.element_type, "encode") else None # type: ignore - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: """Returns the serialization value for msrest. :return: The serialization value for msrest :rtype: str """ - return f"{{{self.element_type.serialization_type}}}" + return f"{{{self.element_type.serialization_type(**kwargs)}}}" def type_annotation(self, **kwargs: Any) -> str: """The python type used for type annotation diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 61521061e2..429496ba33 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -64,9 +64,8 @@ def get_json_template_representation( client_default_value_declaration=client_default_value_declaration, ) - @property - def serialization_type(self) -> str: - return self.value_type.serialization_type + def serialization_type(self, **kwargs: Any) -> str: + return self.value_type.serialization_type(**kwargs) @property def instance_check_template(self) -> str: @@ -137,14 +136,13 @@ def __init__( def __lt__(self, other): return self.name.lower() < other.name.lower() - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: """Returns the serialization value for msrest. :return: The serialization value for msrest :rtype: str """ - return self.value_type.serialization_type + return self.value_type.serialization_type(**kwargs) def description(self, *, is_operation_file: bool) -> str: possible_values = [self.get_declaration(v.value) for v in self.values] @@ -169,7 +167,9 @@ def type_annotation(self, **kwargs: Any) -> str: :rtype: str """ if self.code_model.options["models_mode"]: - module_name = "_models." if kwargs.get("need_module_name", True) else "" + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) + module_name = model_alias if kwargs.get("need_model_alias", True) else "" file_name = f"{self.code_model.enums_filename}." if self.internal else "" model_name = module_name + file_name + self.name # we don't need quoted annotation in operation files, and need it in model folder files. @@ -232,7 +232,7 @@ def imports(self, **kwargs: Any) -> FileImport: "models", ImportType.LOCAL, TypingSection.TYPING, - alias="_models", + alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), ) file_import.merge(self.value_type.imports(in_operation_file=in_operation_file, **kwargs)) relative_path = kwargs.pop("relative_path", None) @@ -242,7 +242,7 @@ def imports(self, **kwargs: Any) -> FileImport: self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), "models", ImportType.LOCAL, - alias="_models", + alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), # TODO ) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/list_type.py b/packages/http-client-python/generator/pygen/codegen/models/list_type.py index 8c29ed6561..1f7d3bfda1 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/list_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/list_type.py @@ -29,9 +29,8 @@ def __init__( def encode(self) -> Optional[str]: return self.element_type.encode if hasattr(self.element_type, "encode") else None # type: ignore - @property - def serialization_type(self) -> str: - return f"[{self.element_type.serialization_type}]" + def serialization_type(self, **kwargs: Any) -> str: + return f"[{self.element_type.serialization_type(**kwargs)}]" def type_annotation(self, **kwargs: Any) -> str: if ( diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index b4ee652663..81412cea9c 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -255,8 +255,7 @@ class JSONModelType(ModelType): def type_annotation(self, **kwargs: Any) -> str: return "ET.Element" if self.is_xml else "JSON" - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "object" def docstring_type(self, **kwargs: Any) -> str: @@ -282,13 +281,15 @@ class GeneratedModelType(ModelType): def type_annotation(self, **kwargs: Any) -> str: is_operation_file = kwargs.pop("is_operation_file", False) skip_quote = kwargs.get("skip_quote", False) - module_name = "_models." if kwargs.get("need_module_name", True) else "" + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) + module_name = model_alias if kwargs.get("need_model_alias", True) else "" file_name = f"{self.code_model.models_filename}." if self.internal else "" retval = module_name + file_name + self.name return retval if is_operation_file or skip_quote else f'"{retval}"' def docstring_type(self, **kwargs: Any) -> str: - return f"~{self.code_model.namespace}.models.{self.type_annotation(need_module_name=False, skip_quote=True)}" + return f"~{self.code_model.namespace}.models.{self.type_annotation(need_model_alias=False, skip_quote=True)}" def docstring_text(self, **kwargs: Any) -> str: return self.name @@ -323,9 +324,8 @@ def imports(self, **kwargs: Any) -> FileImport: class MsrestModelType(GeneratedModelType): base = "msrest" - @property - def serialization_type(self) -> str: - return self.type_annotation(skip_quote=True) if self.internal else self.name + def serialization_type(self, **kwargs: Any) -> str: + return self.type_annotation(skip_quote=True, **kwargs) if self.internal else self.name @property def instance_check_template(self) -> str: @@ -340,12 +340,11 @@ def imports(self, **kwargs: Any) -> FileImport: class DPGModelType(GeneratedModelType): base = "dpg" - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return ( - self.type_annotation(skip_quote=True) + self.type_annotation(skip_quote=True, **kwargs) if self.internal - else self.type_annotation(need_module_name=False, skip_quote=True) + else self.type_annotation(need_model_alias=False, skip_quote=True, **kwargs) ) @property diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 24a2c1e5d9..9c0f7b0c6d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -19,7 +19,7 @@ from .request_builder_parameter import RequestBuilderParameter -from .utils import OrderedSet, add_to_pylint_disable +from .utils import OrderedSet, add_to_pylint_disable, NamespaceType from .base_builder import BaseBuilder from .imports import FileImport, ImportType, TypingSection from .response import ( @@ -299,10 +299,8 @@ def get_request_builder_import( alias="rest", ) if self.code_model.options["builders_visibility"] == "embedded" and async_mode: - operations_folder_name = self.code_model.operations_folder_name(self.client_namespace) - client_namespace = self.client_namespace + operations_folder_name file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace, client_namespace)}.{operations_folder_name}.{self.filename}", + f"{self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace, namespace_type=NamespaceType.OPERATION)}.{self.filename}", request_builder.name, import_type=ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 363356e5dd..79cbdd3a07 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -10,7 +10,7 @@ from .base import BaseModel from .operation import get_operation from .imports import FileImport, ImportType, TypingSection -from .utils import add_to_pylint_disable +from .utils import add_to_pylint_disable, NamespaceType from .lro_operation import LROOperation from .lro_paging_operation import LROPagingOperation from ...utils import NAME_LENGTH_LIMIT @@ -102,23 +102,23 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace) for operation in self.operations: file_import.merge(operation.imports(async_mode, **kwargs)) if not self.code_model.options["combine_operation_files"]: for og in self.operation_groups: file_import.add_submodule_import( - ".", + self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace, namespace_type=NamespaceType.OPERATION, async_mode=async_mode), og.class_name, ImportType.LOCAL, ) # for multiapi - # if ( - # (self.code_model.public_model_types) - # and self.code_model.options["models_mode"] == "msrest" - # and not self.is_mixin - # ): - # file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") + if ( + (self.code_model.public_model_types) + and self.code_model.options["models_mode"] == "msrest" + and not self.is_mixin + ): + file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") if self.is_mixin: file_import.add_submodule_import(f"{relative_path}._vendor", f"{self.client.name}MixinABC", ImportType.LOCAL) if self.has_abstract_operations: diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter.py b/packages/http-client-python/generator/pygen/codegen/models/parameter.py index 5ff32fdfbb..507646256a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter.py @@ -155,9 +155,8 @@ def docstring_text(self, **kwargs: Any) -> str: def docstring_type(self, **kwargs: Any) -> str: return self.type.docstring_type(**kwargs) - @property - def serialization_type(self) -> str: - return self.type.serialization_type + def serialization_type(self, **kwargs: Any) -> str: + return self.type.serialization_type(**kwargs) def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) diff --git a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py index caad87f0eb..f0f2c05980 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py +++ b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py @@ -49,8 +49,7 @@ def default_template_representation_declaration(self) -> str: class BooleanType(PrimitiveType): - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "bool" def docstring_type(self, **kwargs: Any) -> str: @@ -66,8 +65,7 @@ def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: super().__init__(yaml_data=yaml_data, code_model=code_model) self.type = "IO" - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return self.type def docstring_type(self, **kwargs: Any) -> str: @@ -102,8 +100,7 @@ class BinaryIteratorType(PrimitiveType): def _iterator_name(self, **kwargs: Any) -> str: return "AsyncIterator" if kwargs.pop("async_mode") else "Iterator" - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "IO" def docstring_type(self, **kwargs: Any) -> str: @@ -130,8 +127,7 @@ def instance_check_template(self) -> str: class AnyType(PrimitiveType): - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "object" def docstring_type(self, **kwargs: Any) -> str: @@ -155,8 +151,7 @@ def instance_check_template(self) -> str: class AnyObjectType(PrimitiveType): - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "object" def docstring_type(self, **kwargs: Any) -> str: @@ -234,8 +229,7 @@ def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: if yaml_data.get("encode") == "string": self.encode = "str" - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "int" def docstring_type(self, **kwargs: Any) -> str: @@ -254,8 +248,7 @@ def instance_check_template(self) -> str: class FloatType(NumberType): - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "float" def docstring_type(self, **kwargs: Any) -> str: @@ -274,8 +267,7 @@ def instance_check_template(self) -> str: class DecimalType(NumberType): - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "decimal" def docstring_type(self, **kwargs: Any) -> str: @@ -337,8 +329,7 @@ def validation(self) -> Optional[Dict[str, Union[bool, int, str]]]: def get_declaration(self, value) -> str: return f'"{value}"' - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "str" def docstring_type(self, **kwargs: Any) -> str: @@ -358,8 +349,7 @@ def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: else "rfc7231" ) - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: formats_to_attribute_type = { "rfc3339": "iso-8601", "rfc7231": "rfc-1123", @@ -405,8 +395,7 @@ def serialize_sample_value(value: Any) -> str: class TimeType(PrimitiveType): - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "time" def docstring_type(self, **kwargs: Any) -> str: @@ -452,8 +441,7 @@ class UnixTimeType(PrimitiveType): def encode(self) -> str: return "unix-timestamp" - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "unix-time" def docstring_type(self, **kwargs: Any) -> str: @@ -495,8 +483,7 @@ def serialize_sample_value(value: Any) -> str: class DateType(PrimitiveType): - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "date" def docstring_type(self, **kwargs: Any) -> str: @@ -538,8 +525,7 @@ def serialize_sample_value(value: Any) -> str: class DurationType(PrimitiveType): - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return "duration" def docstring_type(self, **kwargs: Any) -> str: @@ -585,8 +571,7 @@ def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: super().__init__(yaml_data=yaml_data, code_model=code_model) self.encode = yaml_data.get("encode", "base64") - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: if self.encode == "base64url": return "base64" return "bytearray" @@ -623,8 +608,7 @@ def imports(self, **kwargs: Any) -> FileImport: def instance_check_template(self) -> str: return f"isinstance({{}}, {self.name})" - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: return self.name diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index 289ba1ee29..ab7f7eae5c 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -76,9 +76,8 @@ def constant(self) -> bool: def is_input(self): return not (self.constant or self.readonly or self.is_discriminator) - @property - def serialization_type(self) -> str: - return self.type.serialization_type + def serialization_type(self, **kwargs: Any) -> str: + return self.type.serialization_type(**kwargs) @property def msrest_deserialization_key(self) -> str: diff --git a/packages/http-client-python/generator/pygen/codegen/models/response.py b/packages/http-client-python/generator/pygen/codegen/models/response.py index 666605a71b..6dd4c98601 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/response.py +++ b/packages/http-client-python/generator/pygen/codegen/models/response.py @@ -29,9 +29,8 @@ def __init__( self.wire_name: str = yaml_data["wireName"] self.type = type - @property - def serialization_type(self) -> str: - return self.type.serialization_type + def serialization_type(self, **kwargs: Any) -> str: + return self.type.serialization_type(**kwargs) @classmethod def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "ResponseHeader": @@ -88,10 +87,9 @@ def is_stream_response(self) -> bool: ) return retval - @property - def serialization_type(self) -> str: + def serialization_type(self, **kwargs: Any) -> str: if self.type: - return self.type.serialization_type + return self.type.serialization_type(**kwargs) return "None" def type_annotation(self, **kwargs: Any) -> str: diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index e786064959..20f924d8d3 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -23,6 +23,7 @@ def add_to_pylint_disable(curr_str: str, entry: str) -> str: class NamespaceType(str, Enum): """Special signal for impports""" + NONE = "none" MODEL = "model" OPERATION = "operation" diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index de7d6c2086..b2b85520b2 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -196,10 +196,11 @@ def is_json_model_type(parameters: ParameterListType) -> bool: class _BuilderBaseSerializer(Generic[BuilderType]): - def __init__(self, code_model: CodeModel, async_mode: bool) -> None: + def __init__(self, code_model: CodeModel, async_mode: bool, serialize_namespace: Optional[str] = None) -> None: self.code_model = code_model self.async_mode = async_mode - self.parameter_serializer = ParameterSerializer() + self.serialize_namespace = code_model.namespace if serialize_namespace is None else serialize_namespace + self.parameter_serializer = ParameterSerializer(self.serialize_namespace) @property @abstractmethod @@ -680,7 +681,7 @@ def _serialize_body_parameter(self, builder: OperationType) -> List[str]: serialization_ctxt_cmd = f", {ser_ctxt_name}={ser_ctxt_name}" if xml_serialization_ctxt else "" create_body_call = ( f"_{body_kwarg_name} = self._serialize.body({body_param.client_name}, " - f"'{body_param.type.serialization_type}'{is_xml_cmd}{serialization_ctxt_cmd})" + f"'{body_param.type.serialization_type(serialize_namespace=self.serialize_namespace)}'{is_xml_cmd}{serialization_ctxt_cmd})" ) elif self.code_model.options["models_mode"] == "dpg": if json_serializable(body_param.default_content_type): @@ -911,7 +912,7 @@ def response_headers(self, response: Response) -> List[str]: retval: List[str] = [ ( f"response_headers['{response_header.wire_name}']=self._deserialize(" - f"'{response_header.serialization_type}', response.headers.get('{response_header.wire_name}'))" + f"'{response_header.serialization_type(serialize_namespace=self.serialize_namespace)}', response.headers.get('{response_header.wire_name}'))" ) for response_header in response.headers ] @@ -945,7 +946,7 @@ def response_deserialization( pylint_disable = " # pylint: disable=protected-access" if self.code_model.options["models_mode"] == "msrest": deserialize_code.append("deserialized = self._deserialize(") - deserialize_code.append(f" '{response.serialization_type}',{pylint_disable}") + deserialize_code.append(f" '{response.serialization_type(serialize_namespace=self.serialize_namespace)}',{pylint_disable}") deserialize_code.append(" pipeline_response.http_response") deserialize_code.append(")") elif self.code_model.options["models_mode"] == "dpg": @@ -1101,7 +1102,7 @@ def error_map(self, builder: OperationType) -> List[str]: if isinstance(e.type, ModelType): if self.code_model.options["models_mode"] == "msrest": error_model_str = ( - f", model=self._deserialize(" f"_models.{e.type.serialization_type}, response)" + f", model=self._deserialize(" f"{e.type.serialization_type(serialize_namespace=self.serialize_namespace)}, response)" ) elif self.code_model.options["models_mode"] == "dpg": error_model_str = f", model=_deserialize(_models.{e.type.name}, response.json())" @@ -1173,13 +1174,13 @@ class OperationSerializer(_OperationSerializer[Operation]): ... class _PagingOperationSerializer(_OperationSerializer[PagingOperationType]): - def __init__(self, code_model: CodeModel, async_mode: bool) -> None: + def __init__(self, code_model: CodeModel, async_mode: bool, serialize_namespace: Optional[str] = None) -> None: # for pylint reasons need to redefine init # probably because inheritance is going too deep - super().__init__(code_model, async_mode) - self.code_model = code_model - self.async_mode = async_mode - self.parameter_serializer = ParameterSerializer() + super().__init__(code_model, async_mode, serialize_namespace) + # self.code_model = code_model + # self.async_mode = async_mode + # self.parameter_serializer = ParameterSerializer(self.serialize_namespace) def serialize_path(self, builder: PagingOperationType) -> List[str]: return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) @@ -1269,10 +1270,10 @@ def _extract_data_callback(self, builder: PagingOperationType) -> List[str]: deserialized = "pipeline_response.http_response.json()" if self.code_model.options["models_mode"] == "msrest": suffix = ".http_response" if hasattr(builder, "initial_operation") else "" - deserialize_type = response.serialization_type + deserialize_type = response.serialization_type(serialize_namespace=self.serialize_namespace) pylint_disable = " # pylint: disable=protected-access" if isinstance(response.type, ModelType) and not response.type.internal: - deserialize_type = f'"{response.serialization_type}"' + deserialize_type = f'"{response.serialization_type(serialize_namespace=self.serialize_namespace)}"' pylint_disable = "" deserialized = ( f"self._deserialize(\n {deserialize_type},{pylint_disable}\n pipeline_response{suffix}\n)" @@ -1338,13 +1339,13 @@ class PagingOperationSerializer(_PagingOperationSerializer[PagingOperation]): .. class _LROOperationSerializer(_OperationSerializer[LROOperationType]): - def __init__(self, code_model: CodeModel, async_mode: bool) -> None: + def __init__(self, code_model: CodeModel, async_mode: bool, serialize_namespace: Optional[str] = None) -> None: # for pylint reasons need to redefine init # probably because inheritance is going too deep - super().__init__(code_model, async_mode) - self.code_model = code_model - self.async_mode = async_mode - self.parameter_serializer = ParameterSerializer() + super().__init__(code_model, async_mode, serialize_namespace) + # self.code_model = code_model + # self.async_mode = async_mode + # self.parameter_serializer = ParameterSerializer() def serialize_path(self, builder: LROOperationType) -> List[str]: return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) @@ -1487,6 +1488,7 @@ def get_operation_serializer( builder: Operation, code_model, async_mode: bool, + serialize_namespace: Optional[str] = None, ) -> Union[ OperationSerializer, PagingOperationSerializer, @@ -1505,4 +1507,5 @@ def get_operation_serializer( ret_cls = LROOperationSerializer elif builder.operation_type == "paging": ret_cls = PagingOperationSerializer - return ret_cls(code_model, async_mode) + serialize_namespace = code_model.namespace if serialize_namespace is None else serialize_namespace + return ret_cls(code_model, async_mode, serialize_namespace) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py index b99f471fbd..22540a1a85 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py @@ -12,9 +12,10 @@ class ClientSerializer: - def __init__(self, client: Client) -> None: + def __init__(self, client: Client, serialize_namespace: str) -> None: self.client = client - self.parameter_serializer = ParameterSerializer() + self.parameter_serializer = ParameterSerializer(serialize_namespace) + self.serialize_namespace = serialize_namespace def _init_signature(self, async_mode: bool) -> str: pylint_disable = "" @@ -244,9 +245,10 @@ def serialize_path(self) -> List[str]: class ConfigSerializer: - def __init__(self, client: Client) -> None: + def __init__(self, client: Client, serialize_namespace: str) -> None: self.client = client - self.parameter_serializer = ParameterSerializer() + self.parameter_serializer = ParameterSerializer(serialize_namespace) + self.serialize_namespace = serialize_namespace def _init_signature(self, async_mode: bool) -> str: return self.parameter_serializer.serialize_method( diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index e597a740c8..9d34f4501f 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -82,6 +82,7 @@ def serialize_service_client_file(self, clients: List[Client]) -> str: async_mode=self.async_mode, get_serializer=ClientSerializer, imports=FileImportSerializer(imports), + serialize_namespace=self.serialize_namespace, ) def serialize_vendor_file(self, clients: List[Client]) -> str: @@ -161,6 +162,7 @@ def serialize_config_file(self, clients: List[Client]) -> str: imports=FileImportSerializer(imports), get_serializer=ConfigSerializer, clients=clients, + serialize_namespace=self.serialize_namespace, ) def serialize_version_file(self) -> str: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py index 75fae457ca..0836136db1 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -74,6 +74,7 @@ def serialize(self) -> str: get_operation_serializer, code_model=self.code_model, async_mode=self.async_mode, + serialize_namespace=self.serialize_namespace, ), request_builder_serializer=RequestBuilderSerializer( self.code_model, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py index f18cd7e1ec..99cbffba67 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py @@ -52,8 +52,11 @@ class PopKwargType(Enum): class ParameterSerializer: - @staticmethod - def serialize_parameter(parameter: ParameterType, serializer_name: str) -> str: + + def __init__(self, serialize_namespace: str) -> None: + self.serialize_namespace = serialize_namespace + + def serialize_parameter(self, parameter: ParameterType, serializer_name: str) -> str: optional_parameters = [] if parameter.skip_url_encoding: @@ -88,7 +91,7 @@ def serialize_parameter(parameter: ParameterType, serializer_name: str) -> str: parameters = [ f'"{origin_name.lstrip("_")}"', "q" if parameter.explode else origin_name, - f"'{type.serialization_type}'", + f"'{type.serialization_type(serialize_namespace=self.serialize_namespace)}'", *optional_parameters, ] parameters_line = ", ".join(parameters) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/client_container.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/client_container.py.jinja2 index 33de339aff..25d4dca8d7 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/client_container.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/client_container.py.jinja2 @@ -7,6 +7,6 @@ {{ imports }} {% for client in clients %} - {% set serializer = get_serializer(client) %} + {% set serializer = get_serializer(client, serialize_namespace) %} {% include "client.py.jinja2" %} {% endfor %} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/config_container.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/config_container.py.jinja2 index 9a3c263e2f..72179cd890 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/config_container.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/config_container.py.jinja2 @@ -11,6 +11,6 @@ VERSION = "unknown" {% endif %} {% for client in clients %} - {% set serializer = get_serializer(client) %} + {% set serializer = get_serializer(client, serialize_namespace) %} {% include "config.py.jinja2" %} {% endfor %} From 4146dc49c361a04d2815472225e73ea038c6fc0f Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 2 Dec 2024 19:16:40 +0800 Subject: [PATCH 07/49] init5 --- .../pygen/codegen/models/code_model.py | 9 + .../pygen/codegen/models/operation.py | 2 +- .../pygen/codegen/models/paging_operation.py | 2 +- .../pygen/codegen/serializers/__init__.py | 286 +----------------- .../codegen/serializers/base_serializer.py | 9 +- .../codegen/serializers/general_serializer.py | 10 +- .../codegen/serializers/model_serializer.py | 9 +- .../operation_groups_serializer.py | 7 +- .../request_builders_serializer.py | 5 + .../codegen/serializers/test_serializer.py | 2 +- 10 files changed, 41 insertions(+), 300 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 2bc7953939..6c3def7ec3 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -224,6 +224,15 @@ def operations_folder_name(self, client_namespace: str) -> str: self._operations_folder_name[client_namespace] = name return self._operations_folder_name[client_namespace] + def get_serialize_namespace(self, client_namespace: str, async_mode: bool = False, namespace_type: NamespaceType = NamespaceType.NONE) -> str: + if namespace_type == NamespaceType.NONE: + return client_namespace + (".aio" if async_mode else "") + if namespace_type == NamespaceType.MODEL: + return client_namespace + ".models" + + operations_folder_name = self.operations_folder_name(client_namespace) + return client_namespace + (".aio." if async_mode else ".") + + operations_folder_name + @property def description(self) -> str: return self.clients[0].description diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 9c0f7b0c6d..827f764326 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -402,7 +402,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements ImportType.SDKCORE, ) if self.code_model.options["builders_visibility"] == "embedded" and not async_mode: - file_import.merge(self.request_builder.imports()) + file_import.merge(self.request_builder.imports(**kwargs)) file_import.add_submodule_import( f"{'' if self.code_model.is_azure_flavor else 'runtime.'}pipeline", "PipelineResponse", diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index 65d22952b9..0c232cb9b2 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -117,7 +117,7 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: and self.code_model.options["builders_visibility"] == "embedded" and not async_mode ): - file_import.merge(self.next_request_builder.imports()) + file_import.merge(self.next_request_builder.imports(**kwargs)) return file_import @property diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 82a00e72d0..d02a0bbac9 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -36,7 +36,6 @@ extract_sample_name, get_namespace_from_package_name, get_namespace_config, - get_all_operation_groups_recursively, ) _LOGGER = logging.getLogger(__name__) @@ -86,55 +85,6 @@ def has_aio_folder(self) -> bool: def has_operations_folder(self) -> bool: return self.code_model.options["show_operations"] and bool(self.code_model.has_operations) - # def _serialize_namespace_level(self, env: Environment, namespace_path: Path, clients: List[Client]) -> None: - # # if there was a patch file before, we keep it - # self._keep_patch_file(namespace_path / Path("_patch.py"), env) - # if self.has_aio_folder: - # self._keep_patch_file(namespace_path / Path("aio") / Path("_patch.py"), env) - - # if self.has_operations_folder: - # self._keep_patch_file( - # namespace_path / Path(self.code_model.operations_folder_name) / Path("_patch.py"), - # env, - # ) - # if self.has_aio_folder: - # self._keep_patch_file( - # namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("_patch.py"), - # env, - # ) - # self._serialize_and_write_top_level_folder(env=env, namespace_path=namespace_path, clients=clients) - - # if any(c for c in self.code_model.clients if c.operation_groups): - # if self.code_model.options["builders_visibility"] != "embedded": - # self._serialize_and_write_rest_layer(env=env, namespace_path=namespace_path) - # if self.has_aio_folder: - # self._serialize_and_write_aio_top_level_folder( - # env=env, - # namespace_path=namespace_path, - # clients=clients, - # ) - - # if self.has_operations_folder: - # self._serialize_and_write_operations_folder(clients, env=env, namespace_path=namespace_path) - # if self.code_model.options["multiapi"]: - # self._serialize_and_write_metadata(env=env, namespace_path=namespace_path) - # if self.code_model.options["package_mode"]: - # self._serialize_and_write_package_files(namespace_path=namespace_path) - - # if ( - # self.code_model.options["show_operations"] - # and self.code_model.has_operations - # and self.code_model.options["generate_sample"] - # ): - # self._serialize_and_write_sample(env, namespace_path) - - # if ( - # self.code_model.options["show_operations"] - # and self.code_model.has_operations - # and self.code_model.options["generate_test"] - # ): - # self._serialize_and_write_test(env, namespace_path) - def serialize(self) -> None: env = Environment( loader=PackageLoader("pygen.codegen", "templates"), @@ -197,52 +147,6 @@ def serialize(self) -> None: - - - - - - - - # namespace_path = ( - # Path(".") if self.code_model.options["no_namespace_folders"] else Path(*self._name_space().split(".")) - # ) - - # p = namespace_path.parent - # general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) - # while p != Path("."): - # # write pkgutil init file - # self.write_file( - # p / Path("__init__.py"), - # general_serializer.serialize_pkgutil_init_file(), - # ) - # p = p.parent - - # # serialize main module - # self._serialize_namespace_level( - # env, - # namespace_path, - # [c for c in self.code_model.clients if c.has_operations], - # ) - - # if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): - # self._keep_patch_file(namespace_path / Path("models") / Path("_patch.py"), env) - - # if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): - # self._serialize_and_write_models_folder(env=env, namespace_path=namespace_path) - # if not self.code_model.options["models_mode"]: - # # keep models file if users ended up just writing a models file - # if self.read_file(namespace_path / Path("models.py")): - # self.write_file( - # namespace_path / Path("models.py"), - # self.read_file(namespace_path / Path("models.py")), - # ) - # if self.code_model.named_unions: - # self.write_file( - # namespace_path / Path("_types.py"), - # TypesSerializer(code_model=self.code_model, env=env).serialize(), - # ) - def _serialize_and_write_package_files(self, namespace: str) -> None: root_of_sdk = self.exec_path(namespace) if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: @@ -291,12 +195,12 @@ def _serialize_and_write_models_folder(self, env: Environment, namespace: str, m if models: self.write_file( models_path / Path(f"{self.code_model.models_filename}.py"), - serializer(code_model=self.code_model, env=env, serialize_namespace=models_namespace).serialize(), + serializer(code_model=self.code_model, env=env, client_namespace=models_namespace).serialize(), ) if enums: self.write_file( models_path / Path(f"{self.code_model.enums_filename}.py"), - EnumSerializer(code_model=self.code_model, env=env, serialize_namespace=models_namespace).serialize(), + EnumSerializer(code_model=self.code_model, env=env, client_namespace=models_namespace).serialize(), ) self.write_file( models_path / Path("__init__.py"), @@ -306,26 +210,6 @@ def _serialize_and_write_models_folder(self, env: Environment, namespace: str, m self._keep_patch_file(models_path / Path("_patch.py"), env) - - # def _serialize_and_write_models_folder(self, env: Environment, namespace_path: Path) -> None: - # # Write the models folder - # models_path = namespace_path / Path("models") - # serializer = DpgModelSerializer if self.code_model.options["models_mode"] == "dpg" else MsrestModelSerializer - # if self.code_model.model_types: - # self.write_file( - # models_path / Path(f"{self.code_model.models_filename}.py"), - # serializer(code_model=self.code_model, env=env).serialize(), - # ) - # if self.code_model.enums: - # self.write_file( - # models_path / Path(f"{self.code_model.enums_filename}.py"), - # EnumSerializer(code_model=self.code_model, env=env).serialize(), - # ) - # self.write_file( - # models_path / Path("__init__.py"), - # ModelInitSerializer(code_model=self.code_model, env=env).serialize(), - # ) - def _serialize_and_write_rest_layer(self, env: Environment, namespace_path: Path) -> None: rest_path = namespace_path / Path(self.code_model.rest_layer_name) group_names = {rb.group_name for c in self.code_model.clients for rb in c.request_builders} @@ -399,48 +283,11 @@ def _serialize_and_write_operations_file( ) - # def _serialize_and_write_operations_file( - # self, - # env: Environment, - # clients: List[Client], - # namespace_path: Path, - # operation_group: Optional[OperationGroup] = None, - # ) -> None: - # filename = operation_group.filename if operation_group else "_operations" - # # write first sync file - # operation_group_serializer = OperationGroupsSerializer( - # code_model=self.code_model, - # clients=clients, - # env=env, - # async_mode=False, - # operation_group=operation_group, - # ) - # self.write_file( - # namespace_path / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py"), - # operation_group_serializer.serialize(), - # ) - - # if self.has_aio_folder: - # # write async operation group and operation files - # operation_group_async_serializer = OperationGroupsSerializer( - # code_model=self.code_model, - # clients=clients, - # env=env, - # async_mode=True, - # operation_group=operation_group, - # ) - # self.write_file( - # (namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py")), - # operation_group_async_serializer.serialize(), - # ) - - def _serialize_and_write_operations_folder( self, operation_groups: List[OperationGroup], env: Environment, namespace: str ) -> None: operations_folder_name = self.code_model.operations_folder_name(namespace) - # if there was a patch file before, we keep it self._keep_patch_file(self.exec_path(namespace) / Path("_patch.py"), env) if self.has_aio_folder: @@ -476,44 +323,6 @@ def _serialize_and_write_operations_folder( namespace=namespace, operation_group=operation_group, ) - - # def _serialize_and_write_operations_folder( - # self, clients: List[Client], env: Environment, namespace_path: Path - # ) -> None: - # # write sync operations init file - # operations_init_serializer = OperationsInitSerializer( - # code_model=self.code_model, clients=clients, env=env, async_mode=False - # ) - # self.write_file( - # namespace_path / Path(self.code_model.operations_folder_name) / Path("__init__.py"), - # operations_init_serializer.serialize(), - # ) - - # # write async operations init file - # if self.has_aio_folder: - # operations_async_init_serializer = OperationsInitSerializer( - # code_model=self.code_model, clients=clients, env=env, async_mode=True - # ) - # self.write_file( - # namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("__init__.py"), - # operations_async_init_serializer.serialize(), - # ) - - # if self.code_model.options["combine_operation_files"]: - # self._serialize_and_write_operations_file( - # env=env, - # namespace_path=namespace_path, - # clients=clients, - # ) - # else: - # for operation_group in get_all_operation_groups_recursively(self.code_model.clients): - # self._serialize_and_write_operations_file( - # env=env, - # namespace_path=namespace_path, - # operation_group=operation_group, - # clients=clients, - # ) - def _serialize_and_write_version_file( self, namespace: str, @@ -547,6 +356,7 @@ def _serialize_client_and_config_files( env: Environment, ) -> None: general_serializer.async_mode = False + general_serializer.client_namespace = namespace self.write_file( self.exec_path(namespace) / Path(f"{self.code_model.client_filename}.py"), general_serializer.serialize_service_client_file(clients), @@ -572,24 +382,6 @@ def _serialize_client_and_config_files( if self.has_aio_folder: self._keep_patch_file(self.exec_path(namespace) / Path("aio/_patch.py"), env) - # def _serialize_client_and_config_files( - # self, - # namespace_path: Path, - # general_serializer: GeneralSerializer, - # async_mode: bool, - # clients: List[Client], - # ) -> None: - # if self.code_model.has_operations: - # namespace_path = namespace_path / Path("aio") if async_mode else namespace_path - # self.write_file( - # namespace_path / Path(f"{self.code_model.client_filename}.py"), - # general_serializer.serialize_service_client_file(clients), - # ) - # self.write_file( - # namespace_path / Path("_configuration.py"), - # general_serializer.serialize_config_file(clients), - # ) - def _serialize_and_write_top_level_folder( self, env: Environment, namespace: str ) -> None: @@ -648,78 +440,6 @@ def _serialize_and_write_top_level_folder( TypesSerializer(code_model=self.code_model, env=env).serialize(), ) - # def _serialize_and_write_top_level_folder( - # self, env: Environment, namespace_path: Path, clients: List[Client] - # ) -> None: - # general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) - - # self.write_file( - # namespace_path / Path("__init__.py"), - # general_serializer.serialize_init_file(clients), - # ) - - # # Write the service client - # self._serialize_client_and_config_files(namespace_path, general_serializer, async_mode=False, clients=clients) - # if self.code_model.need_vendored_code(async_mode=False): - # self.write_file( - # namespace_path / Path("_vendor.py"), - # general_serializer.serialize_vendor_file(clients), - # ) - - # self._serialize_and_write_version_file(namespace_path, general_serializer) - - # # write the empty py.typed file - # self.write_file(namespace_path / Path("py.typed"), "# Marker file for PEP 561.") - - # if not self.code_model.options["client_side_validation"] and not self.code_model.options["multiapi"]: - # self.write_file( - # namespace_path / Path("_serialization.py"), - # general_serializer.serialize_serialization_file(), - # ) - # if self.code_model.options["models_mode"] == "dpg": - # self.write_file( - # namespace_path / Path("_model_base.py"), - # general_serializer.serialize_model_base_file(), - # ) - - # if any(og for client in self.code_model.clients for og in client.operation_groups if og.need_validation): - # self.write_file( - # namespace_path / Path("_validation.py"), - # general_serializer.serialize_validation_file(), - # ) - # if self.code_model.options.get("emit_cross_language_definition_file"): - # self.write_file( - # Path("./apiview_mapping_python.json"), - # general_serializer.serialize_cross_language_definition_file(), - # ) - - # # Write the setup file - # if self.code_model.options["basic_setup_py"]: - # self.write_file(Path("setup.py"), general_serializer.serialize_setup_file()) - - # def _serialize_and_write_aio_top_level_folder( - # self, env: Environment, namespace_path: Path, clients: List[Client] - # ) -> None: - # aio_general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=True) - - # aio_path = namespace_path / Path("aio") - - # # Write the __init__ file - # self.write_file( - # aio_path / Path("__init__.py"), - # aio_general_serializer.serialize_init_file(clients), - # ) - - # # Write the service client - # self._serialize_client_and_config_files( - # namespace_path, aio_general_serializer, async_mode=True, clients=clients - # ) - # if self.code_model.need_vendored_code(async_mode=True): - # self.write_file( - # aio_path / Path("_vendor.py"), - # aio_general_serializer.serialize_vendor_file(clients), - # ) - def _serialize_and_write_metadata(self, env: Environment, namespace: str) -> None: metadata_serializer = MetadataSerializer(self.code_model, env) self.write_file(self.exec_path(namespace) / Path("_metadata.json"), metadata_serializer.serialize()) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py index 16f3741622..0620f48b5c 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py @@ -14,10 +14,15 @@ class BaseSerializer: """Base serializer for SDK root level files""" - def __init__(self, code_model: CodeModel, env: Environment, *, serialize_namespace: Optional[str] = None): + def __init__(self, code_model: CodeModel, env: Environment, async_mode: bool = False, *, client_namespace: Optional[str] = None): self.code_model = code_model self.env = env - self.serialize_namespace = serialize_namespace or code_model.namespace + self.async_mode = async_mode + self.client_namespace = code_model.namespace if client_namespace is None else client_namespace def init_file_import(self) -> FileImport: return FileImport(self.code_model) + + @property + def serialize_namespace(self) -> str: + return self.code_model.get_serialize_namespace(self.client_namespace, async_mode=self.async_mode) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index 9d34f4501f..d46a6f6211 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -21,12 +21,6 @@ class GeneralSerializer(BaseSerializer): """General serializer for SDK root level files""" - def __init__( - self, code_model: CodeModel, env: Environment, async_mode: bool, *, serialize_namespace: Optional[str] = None - ): - super().__init__(code_model, env, serialize_namespace=serialize_namespace) - self.async_mode = async_mode - def serialize_setup_file(self) -> str: template = self.env.get_template("packaging_templates/setup.py.jinja2") params = {} @@ -74,7 +68,7 @@ def serialize_service_client_file(self, clients: List[Client]) -> str: imports = FileImport(self.code_model) for client in clients: - imports.merge(client.imports(self.async_mode)) + imports.merge(client.imports(self.async_mode, serliaze_namespace=self.serialize_namespace)) return template.render( code_model=self.code_model, @@ -155,7 +149,7 @@ def serialize_config_file(self, clients: List[Client]) -> str: template = self.env.get_template("config_container.py.jinja2") imports = FileImport(self.code_model) for client in self.code_model.clients: - imports.merge(client.config.imports(self.async_mode)) + imports.merge(client.config.imports(self.async_mode, serialize_namespace=self.serialize_namespace)) return template.render( code_model=self.code_model, async_mode=self.async_mode, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 449745fa2e..acaa7f7a04 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -10,7 +10,7 @@ from ..models.imports import FileImport, TypingSection, MsrestImportType, ImportType from .import_serializer import FileImportSerializer from .base_serializer import BaseSerializer - +from ..models.utils import NamespaceType def _documentation_string(prop: Property, description_keyword: str, docstring_type_keyword: str) -> List[str]: retval: List[str] = [] @@ -127,6 +127,9 @@ def pylint_disable(self, model: ModelType) -> str: def global_pylint_disables(self) -> str: return "" + @property + def serialize_namespace(self) -> str: + return self.code_model.get_serialize_namespace(self.client_namespace, namespace_type=NamespaceType.MODEL) class MsrestModelSerializer(_ModelSerializer): def imports(self) -> FileImport: @@ -219,9 +222,9 @@ def imports(self) -> FileImport: for model in self.code_model.model_types: if model.base == "json": continue - file_import.merge(model.imports(is_operation_file=False)) + file_import.merge(model.imports(is_operation_file=False, serialize_namespace=self.serialize_namespace)) for prop in model.properties: - file_import.merge(prop.imports()) + file_import.merge(prop.imports(serialize_namespace=self.serialize_namespace)) if model.is_polymorphic: file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) if not model.internal and self.init_line(model): diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py index 0836136db1..b0e2ed1629 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -16,6 +16,7 @@ Client, FileImport, ) +from ..models.utils import NamespaceType from .import_serializer import FileImportSerializer from .builder_serializer import ( get_operation_serializer, @@ -34,7 +35,7 @@ def __init__( *, serialize_namespace: Optional[str] = None, ): - super().__init__(code_model, env, serialize_namespace=serialize_namespace) + super().__init__(code_model, env, async_mode, serialize_namespace=serialize_namespace) self.operation_groups = operation_groups self.async_mode = async_mode @@ -50,6 +51,9 @@ def _get_request_builders( and not r.abstract and not r.is_lro # lro has already initial builder ] + @property + def serialize_namespace(self) -> str: + return self.code_model.get_serialize_namespace(self.client_namespace, async_mode=self.async_mode, namespace_type=NamespaceType.OPERATION) def serialize(self) -> str: imports = FileImport(self.code_model) @@ -57,6 +61,7 @@ def serialize(self) -> str: imports.merge( operation_group.imports( async_mode=self.async_mode, + serialize_namespace=self.serialize_namespace, ) ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py index b178d66054..2a9f56f3e5 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py @@ -7,6 +7,7 @@ from jinja2 import Environment from ..models import FileImport +from ..models.utils import NamespaceType from .import_serializer import FileImportSerializer from ..models import CodeModel, RequestBuilderType from .builder_serializer import RequestBuilderSerializer @@ -41,6 +42,10 @@ def serialize_init(self) -> str: request_builders=[r for r in self.request_builders if not r.is_overload], ) + @property + def serialize_namespace(self) -> str: + return self.code_model.get_serialize_namespace(self.client_namespace, namespace_type=NamespaceType.OPERATION) + def serialize_request_builders(self) -> str: template = self.env.get_template("request_builders.py.jinja2") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index 370b6554ce..35b43fe8aa 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -145,7 +145,7 @@ def __init__( is_async: bool = False, serialize_namespace: Optional[str] = None, ) -> None: - super().__init__(code_model, env, serialize_namespace=serialize_namespace) + super().__init__(code_model, env, is_async, serialize_namespace=serialize_namespace) self.is_async = is_async @property From f0d01be9800a56e7552035cab5073dc46179088c Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 3 Dec 2024 11:51:50 +0800 Subject: [PATCH 08/49] format --- .../emitter/src/code-model.ts | 10 +- .../http-client-python/emitter/src/utils.ts | 7 +- .../generator/pygen/codegen/_utils.py | 1 + .../generator/pygen/codegen/models/client.py | 4 +- .../pygen/codegen/models/code_model.py | 24 +++-- .../generator/pygen/codegen/models/imports.py | 10 +- .../pygen/codegen/models/lro_operation.py | 6 +- .../pygen/codegen/models/operation.py | 34 +++++-- .../pygen/codegen/models/operation_group.py | 11 ++- .../generator/pygen/codegen/models/utils.py | 3 +- .../pygen/codegen/serializers/__init__.py | 94 ++++++++++--------- .../codegen/serializers/base_serializer.py | 9 +- .../codegen/serializers/builder_serializer.py | 7 +- .../codegen/serializers/model_serializer.py | 4 +- .../operation_groups_serializer.py | 5 +- .../serializers/operations_init_serializer.py | 5 +- .../serializers/parameter_serializer.py | 2 +- .../codegen/serializers/types_serializer.py | 4 +- .../test_azure_arm_resource_async.py | 4 +- .../mock_api_tests/test_azure_arm_resource.py | 4 +- 20 files changed, 159 insertions(+), 89 deletions(-) diff --git a/packages/http-client-python/emitter/src/code-model.ts b/packages/http-client-python/emitter/src/code-model.ts index f13cfe4d7b..3f24de2b8b 100644 --- a/packages/http-client-python/emitter/src/code-model.ts +++ b/packages/http-client-python/emitter/src/code-model.ts @@ -30,7 +30,12 @@ import { simpleTypesMap, typesMap, } from "./types.js"; -import { emitParamBase, getImplementation, removeUnderscoresFromNamespace, getClientNamespace } from "./utils.js"; +import { + emitParamBase, + getClientNamespace, + getImplementation, + removeUnderscoresFromNamespace, +} from "./utils.js"; function emitBasicMethod( context: PythonSdkContext, @@ -195,7 +200,8 @@ function emitOperationGroups( // operation has same clientNamespace as the operation group for (const og of operationGroups) { for (const op of og.operations) { - op.clientNamespace = og.clientNamespace; + op.clientNamespace = getClientNamespace(context, og.clientNamespace); + } } return operationGroups.length > 0 ? operationGroups : undefined; diff --git a/packages/http-client-python/emitter/src/utils.ts b/packages/http-client-python/emitter/src/utils.ts index 6038ff6767..98407a009c 100644 --- a/packages/http-client-python/emitter/src/utils.ts +++ b/packages/http-client-python/emitter/src/utils.ts @@ -200,8 +200,11 @@ export function capitalize(name: string): string { return name[0].toUpperCase() + name.slice(1); } -export function getClientNamespace(context: PythonSdkContext,clientNamespace: string) { - const options = context.emitContext.options +export function getClientNamespace( + context: PythonSdkContext, + clientNamespace: string, +) { + const options = context.emitContext.options; if ([undefined, false].includes(options["enable-typespec-namespace"])) { return context.sdkPackage.rootNamespace; } diff --git a/packages/http-client-python/generator/pygen/codegen/_utils.py b/packages/http-client-python/generator/pygen/codegen/_utils.py index 8097eefbd9..a59235464a 100644 --- a/packages/http-client-python/generator/pygen/codegen/_utils.py +++ b/packages/http-client-python/generator/pygen/codegen/_utils.py @@ -20,5 +20,6 @@ def get_unique_alias(relative_import: str, module_name: str = "models") -> str: pass + def get_parent_namespace(namespace: str) -> str: return namespace.rsplit(".", 1)[0] diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index a99c5a4d10..c40f01ed78 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -391,7 +391,9 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) if self.code_model.options["package_version"]: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - file_import.add_submodule_import(f"{self.code_model.get_relative_import_path(serialize_namespace)}._version", "VERSION", ImportType.LOCAL) + file_import.add_submodule_import( + f"{self.code_model.get_relative_import_path(serialize_namespace)}._version", "VERSION", ImportType.LOCAL + ) if self.code_model.options["azure_arm"]: policy = "AsyncARMChallengeAuthenticationPolicy" if async_mode else "ARMChallengeAuthenticationPolicy" file_import.add_submodule_import("azure.mgmt.core.policies", "ARMHttpLoggingPolicy", ImportType.SDKCORE) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 6c3def7ec3..96042d53d1 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -15,6 +15,7 @@ from .utils import NamespaceType from ..serializers.utils import get_all_operation_groups_recursively + def _is_legacy(options) -> bool: return not (options.get("version_tolerant") or options.get("low_level_client")) @@ -94,7 +95,14 @@ def __init__( # |azure.test.aio.operations | azure.test | ... | # |azure.test.subtest.aio.operations|azure.test| .... | # |azure.test |azure.test.subtest | .subtest | - def get_relative_import_path(self, serialize_namespace: str, imported_namespace: Optional[str] = None, *, namespace_type: NamespaceType = NamespaceType.MODEL, async_mode: bool = False) -> str: + def get_relative_import_path( + self, + serialize_namespace: str, + imported_namespace: Optional[str] = None, + *, + namespace_type: NamespaceType = NamespaceType.MODEL, + async_mode: bool = False, + ) -> str: if imported_namespace is None: imported_namespace = self.namespace else: @@ -142,12 +150,14 @@ def client_namespace_types(self) -> Dict[str, ClientNamespaceType]: self._client_namespace_types[enum.client_namespace].enums.append(enum) for operation_group in get_all_operation_groups_recursively(self.clients): if operation_group.client_namespace not in self._client_namespace_types: - self._client_namespace_types[operation_group.client_namespace] = ClientNamespaceType(operation_group.client_namespace) + self._client_namespace_types[operation_group.client_namespace] = ClientNamespaceType( + operation_group.client_namespace + ) self._client_namespace_types[operation_group.client_namespace].operation_groups.append(operation_group) - + if len(self._client_namespace_types.keys()) > 1: self.has_subnamespace = True - + # insert namespace to make sure it is continuous(e.g. ("", "azure", "azure.mgmt", "azure.mgmt.service", ...)) longest_namespace = sorted(self._client_namespace_types.keys())[-1] namespace_parts = longest_namespace.split(".") @@ -224,14 +234,16 @@ def operations_folder_name(self, client_namespace: str) -> str: self._operations_folder_name[client_namespace] = name return self._operations_folder_name[client_namespace] - def get_serialize_namespace(self, client_namespace: str, async_mode: bool = False, namespace_type: NamespaceType = NamespaceType.NONE) -> str: + def get_serialize_namespace( + self, client_namespace: str, async_mode: bool = False, namespace_type: NamespaceType = NamespaceType.NONE + ) -> str: if namespace_type == NamespaceType.NONE: return client_namespace + (".aio" if async_mode else "") if namespace_type == NamespaceType.MODEL: return client_namespace + ".models" operations_folder_name = self.operations_folder_name(client_namespace) - return client_namespace + (".aio." if async_mode else ".") + + operations_folder_name + return client_namespace + (".aio." if async_mode else ".") + +operations_folder_name @property def description(self) -> str: diff --git a/packages/http-client-python/generator/pygen/codegen/models/imports.py b/packages/http-client-python/generator/pygen/codegen/models/imports.py index 6dfa570f3a..ab64d0ff21 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/imports.py +++ b/packages/http-client-python/generator/pygen/codegen/models/imports.py @@ -287,15 +287,9 @@ def add_msrest_import( self.add_submodule_import(relative_path, "_serialization", ImportType.LOCAL, typing_section) else: self.add_submodule_import( - f"{relative_path}._serialization", - "Serializer", - ImportType.LOCAL, - typing_section + f"{relative_path}._serialization", "Serializer", ImportType.LOCAL, typing_section ) if msrest_import_type == MsrestImportType.SerializerDeserializer: self.add_submodule_import( - f"{relative_path}._serialization", - "Deserializer", - ImportType.LOCAL, - typing_section + f"{relative_path}._serialization", "Deserializer", ImportType.LOCAL, typing_section ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index c455e7a7eb..69fb8e46ad 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -133,7 +133,11 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: # used in the case if initial operation returns none # but final call returns a model serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - file_import.add_submodule_import(f"{self.code_model.get_relative_import_path(serialize_namespace)}._model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import( + f"{self.code_model.get_relative_import_path(serialize_namespace)}._model_base", + "_deserialize", + ImportType.LOCAL, + ) file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) file_import.add_submodule_import("typing", "cast", ImportType.STDLIB) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 827f764326..87aef35991 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -247,10 +247,18 @@ def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport: ) ) for response in self.responses: - file_import.merge(response.imports_for_multiapi(async_mode=async_mode, need_import_iobase=self.need_import_iobase, **kwargs)) + file_import.merge( + response.imports_for_multiapi( + async_mode=async_mode, need_import_iobase=self.need_import_iobase, **kwargs + ) + ) if self.code_model.options["models_mode"]: for exception in self.exceptions: - file_import.merge(exception.imports_for_multiapi(async_mode=async_mode, need_import_iobase=self.need_import_iobase, **kwargs)) + file_import.merge( + exception.imports_for_multiapi( + async_mode=async_mode, need_import_iobase=self.need_import_iobase, **kwargs + ) + ) return file_import @staticmethod @@ -311,7 +319,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements ) -> FileImport: if self.abstract: return FileImport(self.code_model) - + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import = self._imports_shared(async_mode, **kwargs) @@ -324,13 +332,17 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements ) ) for response in self.responses: - file_import.merge(response.imports(async_mode=async_mode, need_import_iobase=self.need_import_iobase, **kwargs)) + file_import.merge( + response.imports(async_mode=async_mode, need_import_iobase=self.need_import_iobase, **kwargs) + ) if self.code_model.options["models_mode"]: for exception in self.exceptions: file_import.merge(exception.imports(async_mode=async_mode, **kwargs)) if self.parameters.has_body and self.parameters.body_parameter.flattened: - file_import.merge(self.parameters.body_parameter.type.imports(need_import_iobase=self.need_import_iobase, **kwargs)) + file_import.merge( + self.parameters.body_parameter.type.imports(need_import_iobase=self.need_import_iobase, **kwargs) + ) if not async_mode: for param in self.parameters.headers: if param.wire_name.lower() == "repeatability-request-id": @@ -387,8 +399,16 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements ImportType.SDKCORE, ) if not async_mode: - file_import.add_submodule_import(f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", "prep_if_match", ImportType.LOCAL) - file_import.add_submodule_import(f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", "prep_if_none_match", ImportType.LOCAL) + file_import.add_submodule_import( + f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", + "prep_if_match", + ImportType.LOCAL, + ) + file_import.add_submodule_import( + f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", + "prep_if_none_match", + ImportType.LOCAL, + ) if async_mode: file_import.add_submodule_import( "rest", diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 79cbdd3a07..dbe0c690c4 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -108,7 +108,12 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: if not self.code_model.options["combine_operation_files"]: for og in self.operation_groups: file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace, namespace_type=NamespaceType.OPERATION, async_mode=async_mode), + self.code_model.get_relative_import_path( + serialize_namespace, + self.client_namespace, + namespace_type=NamespaceType.OPERATION, + async_mode=async_mode, + ), og.class_name, ImportType.LOCAL, ) @@ -120,7 +125,9 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ): file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") if self.is_mixin: - file_import.add_submodule_import(f"{relative_path}._vendor", f"{self.client.name}MixinABC", ImportType.LOCAL) + file_import.add_submodule_import( + f"{relative_path}._vendor", f"{self.client.name}MixinABC", ImportType.LOCAL + ) if self.has_abstract_operations: file_import.add_submodule_import(f"{relative_path}._vendor", "raise_if_not_implemented", ImportType.LOCAL) if all(o.abstract for o in self.operations): diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index 20f924d8d3..a1a04101a5 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -21,9 +21,10 @@ def add_to_pylint_disable(curr_str: str, entry: str) -> str: return f"{curr_str},{entry}" return f" # pylint: disable={entry}" + class NamespaceType(str, Enum): """Special signal for impports""" + NONE = "none" MODEL = "model" OPERATION = "operation" - diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index d02a0bbac9..f39bed0159 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -66,6 +66,7 @@ def _sample_output_path(source_file_path: str) -> Path: return Path("/".join([to_snake_case(i) for i in after_examples.parts])) return Path("") + class JinjaSerializer(ReaderAndWriter): def __init__( self, @@ -99,54 +100,60 @@ def serialize(self) -> None: for client_namespace, client_namespace_type in self.code_model.client_namespace_types.items(): if client_namespace == "": - # Write the setup file - if self.code_model.options["basic_setup_py"]: - self.write_file(self.exec_path(client_namespace) / Path("setup.py"), general_serializer.serialize_setup_file()) - - # add packaging files in root namespace (e.g. setup.py, README.md, etc.) - self._serialize_and_write_package_files(client_namespace) - - # add generated samples and generated tests - if ( - self.code_model.options["show_operations"] - and self.code_model.has_operations - ): - if self.code_model.options["generate_sample"]: - self._serialize_and_write_sample(env, namespace=client_namespace) - if self.code_model.options["generate_test"]: - self._serialize_and_write_test(env, namespace=client_namespace) + # Write the setup file + if self.code_model.options["basic_setup_py"]: + self.write_file( + self.exec_path(client_namespace) / Path("setup.py"), general_serializer.serialize_setup_file() + ) + + # add packaging files in root namespace (e.g. setup.py, README.md, etc.) + self._serialize_and_write_package_files(client_namespace) + + # add generated samples and generated tests + if self.code_model.options["show_operations"] and self.code_model.has_operations: + if self.code_model.options["generate_sample"]: + self._serialize_and_write_sample(env, namespace=client_namespace) + if self.code_model.options["generate_test"]: + self._serialize_and_write_test(env, namespace=client_namespace) elif client_namespace_type.clients: - # add clients folder if there are clients in this namespace - self._serialize_client_and_config_files(client_namespace, general_serializer, client_namespace_type.clients, env) + # add clients folder if there are clients in this namespace + self._serialize_client_and_config_files( + client_namespace, general_serializer, client_namespace_type.clients, env + ) else: - # add pkgutil init file if no clients in this namespace - self.write_file( - self.exec_path(client_namespace) / Path("__init__.py"), - general_serializer.serialize_pkgutil_init_file(), - ) - + # add pkgutil init file if no clients in this namespace + self.write_file( + self.exec_path(client_namespace) / Path("__init__.py"), + general_serializer.serialize_pkgutil_init_file(), + ) + # _model_base.py/_serialization.py/_vendor.py/py.typed/_types.py/_validation.py is always put in top level namespace if client_namespace == self.code_model.namespace: self._serialize_and_write_top_level_folder(env=env, namespace=client_namespace) - + # add models folder if there are models in this namespace if (client_namespace_type.models or client_namespace_type.enums) and self.code_model.options["models_mode"]: - self._serialize_and_write_models_folder(env=env, namespace=client_namespace, models=client_namespace_type.models, enums=client_namespace_type.enums) - + self._serialize_and_write_models_folder( + env=env, + namespace=client_namespace, + models=client_namespace_type.models, + enums=client_namespace_type.enums, + ) + if not self.code_model.options["models_mode"]: # keep models file if users ended up just writing a models file model_path = self.exec_path(client_namespace) / Path("models.py") if self.read_file(model_path): - self.write_file(model_path, self.read_file(model_path)) - + self.write_file(model_path, self.read_file(model_path)) + # add operations folder if there are operations in this namespace if client_namespace_type.operation_groups: - self._serialize_and_write_operations_folder(client_namespace_type.operation_groups, env=env, namespace=client_namespace) + self._serialize_and_write_operations_folder( + client_namespace_type.operation_groups, env=env, namespace=client_namespace + ) if self.code_model.options["multiapi"]: self._serialize_and_write_metadata(env=env, namespace=client_namespace) - - def _serialize_and_write_package_files(self, namespace: str) -> None: root_of_sdk = self.exec_path(namespace) if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: @@ -187,7 +194,9 @@ def _keep_patch_file(self, path_file: Path, env: Environment): PatchSerializer(env=env, code_model=self.code_model).serialize(), ) - def _serialize_and_write_models_folder(self, env: Environment, namespace: str, models: List[ModelType], enums: List[EnumType]) -> None: + def _serialize_and_write_models_folder( + self, env: Environment, namespace: str, models: List[ModelType], enums: List[EnumType] + ) -> None: # Write the models folder models_namespace = namespace + ".models" models_path = self.exec_path(models_namespace) @@ -208,7 +217,6 @@ def _serialize_and_write_models_folder(self, env: Environment, namespace: str, m ) self._keep_patch_file(models_path / Path("_patch.py"), env) - def _serialize_and_write_rest_layer(self, env: Environment, namespace_path: Path) -> None: rest_path = namespace_path / Path(self.code_model.rest_layer_name) @@ -282,7 +290,6 @@ def _serialize_and_write_operations_file( operation_group_serializer.serialize(), ) - def _serialize_and_write_operations_folder( self, operation_groups: List[OperationGroup], env: Environment, namespace: str ) -> None: @@ -323,6 +330,7 @@ def _serialize_and_write_operations_folder( namespace=namespace, operation_group=operation_group, ) + def _serialize_and_write_version_file( self, namespace: str, @@ -382,9 +390,7 @@ def _serialize_client_and_config_files( if self.has_aio_folder: self._keep_patch_file(self.exec_path(namespace) / Path("aio/_patch.py"), env) - def _serialize_and_write_top_level_folder( - self, env: Environment, namespace: str - ) -> None: + def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str) -> None: general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) # write _vendor.py @@ -411,7 +417,7 @@ def _serialize_and_write_top_level_folder( self.exec_path(namespace) / Path("_serialization.py"), general_serializer.serialize_serialization_file(), ) - + # write _model_base.py if self.code_model.options["models_mode"] == "dpg": self.write_file( @@ -456,9 +462,13 @@ def _name_space(self) -> str: @property def _exec_path_implimentation(self) -> Path: - """ Assume the process is running in the root folder of the package. If not, we need the path implementation.""" - return Path("../" * (self._name_space().count(".") + 1)) if self.code_model.options["no_namespace_folders"] else Path(".") - + """Assume the process is running in the root folder of the package. If not, we need the path implementation.""" + return ( + Path("../" * (self._name_space().count(".") + 1)) + if self.code_model.options["no_namespace_folders"] + else Path(".") + ) + def exec_path(self, namespace: str) -> Path: return self._exec_path_implimentation / Path(*namespace.split(".")) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py index 0620f48b5c..4405338fc0 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/base_serializer.py @@ -14,7 +14,14 @@ class BaseSerializer: """Base serializer for SDK root level files""" - def __init__(self, code_model: CodeModel, env: Environment, async_mode: bool = False, *, client_namespace: Optional[str] = None): + def __init__( + self, + code_model: CodeModel, + env: Environment, + async_mode: bool = False, + *, + client_namespace: Optional[str] = None + ): self.code_model = code_model self.env = env self.async_mode = async_mode diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index b2b85520b2..be3a7fb043 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -946,7 +946,9 @@ def response_deserialization( pylint_disable = " # pylint: disable=protected-access" if self.code_model.options["models_mode"] == "msrest": deserialize_code.append("deserialized = self._deserialize(") - deserialize_code.append(f" '{response.serialization_type(serialize_namespace=self.serialize_namespace)}',{pylint_disable}") + deserialize_code.append( + f" '{response.serialization_type(serialize_namespace=self.serialize_namespace)}',{pylint_disable}" + ) deserialize_code.append(" pipeline_response.http_response") deserialize_code.append(")") elif self.code_model.options["models_mode"] == "dpg": @@ -1102,7 +1104,8 @@ def error_map(self, builder: OperationType) -> List[str]: if isinstance(e.type, ModelType): if self.code_model.options["models_mode"] == "msrest": error_model_str = ( - f", model=self._deserialize(" f"{e.type.serialization_type(serialize_namespace=self.serialize_namespace)}, response)" + f", model=self._deserialize(" + f"{e.type.serialization_type(serialize_namespace=self.serialize_namespace)}, response)" ) elif self.code_model.options["models_mode"] == "dpg": error_model_str = f", model=_deserialize(_models.{e.type.name}, response.json())" diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index acaa7f7a04..9d271c19b9 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -12,6 +12,7 @@ from .base_serializer import BaseSerializer from ..models.utils import NamespaceType + def _documentation_string(prop: Property, description_keyword: str, docstring_type_keyword: str) -> List[str]: retval: List[str] = [] sphinx_prefix = f":{description_keyword} {prop.client_name}:" @@ -131,6 +132,7 @@ def global_pylint_disables(self) -> str: def serialize_namespace(self) -> str: return self.code_model.get_serialize_namespace(self.client_namespace, namespace_type=NamespaceType.MODEL) + class MsrestModelSerializer(_ModelSerializer): def imports(self) -> FileImport: file_import = FileImport(self.code_model) @@ -213,7 +215,7 @@ def super_call(self, model: ModelType) -> List[str]: def imports(self) -> FileImport: file_import = FileImport(self.code_model) file_import.add_submodule_import( - self.code_model.get_relative_import_path(self.serialize_namespace) + self.code_model.get_relative_import_path(self.serialize_namespace), "_model_base", ImportType.LOCAL, TypingSection.REGULAR, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py index b0e2ed1629..0e431e049c 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -51,9 +51,12 @@ def _get_request_builders( and not r.abstract and not r.is_lro # lro has already initial builder ] + @property def serialize_namespace(self) -> str: - return self.code_model.get_serialize_namespace(self.client_namespace, async_mode=self.async_mode, namespace_type=NamespaceType.OPERATION) + return self.code_model.get_serialize_namespace( + self.client_namespace, async_mode=self.async_mode, namespace_type=NamespaceType.OPERATION + ) def serialize(self) -> str: imports = FileImport(self.code_model) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py index 75ebcdd1d4..c5659d5c4a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py @@ -27,10 +27,7 @@ def operation_group_imports(self) -> List[str]: def _get_filename(operation_group: OperationGroup) -> str: return "_operations" if self.code_model.options["combine_operation_files"] else operation_group.filename - return [ - f"from .{_get_filename(og)} import {og.class_name} # type: ignore" - for og in self.operation_groups - ] + return [f"from .{_get_filename(og)} import {og.class_name} # type: ignore" for og in self.operation_groups] def serialize(self) -> str: operation_group_init_template = self.env.get_template("operations_folder_init.py.jinja2") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py index 99cbffba67..f83feedb4a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py @@ -52,7 +52,7 @@ class PopKwargType(Enum): class ParameterSerializer: - + def __init__(self, serialize_namespace: str) -> None: self.serialize_namespace = serialize_namespace diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 9f5dc15499..f628e8f023 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -18,7 +18,9 @@ def imports(self) -> FileImport: ImportType.STDLIB, ) for nu in self.code_model.named_unions: - file_import.merge(nu.imports(model_typing=True, is_types_file=True, serialize_namespace=self.serialize_namespace)) + file_import.merge( + nu.imports(model_typing=True, is_types_file=True, serialize_namespace=self.serialize_namespace) + ) return file_import def serialize(self) -> str: diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_azure_arm_resource_async.py b/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_azure_arm_resource_async.py index 1e8fbe0213..abf03c911c 100644 --- a/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_azure_arm_resource_async.py +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_azure_arm_resource_async.py @@ -36,9 +36,7 @@ async def test_client_signature(credential, authentication_policy): # make sure signautre order is correct await client.top_level.get(RESOURCE_GROUP_NAME, "top") # make sure signautre name is correct - await client.top_level.get( - resource_group_name=RESOURCE_GROUP_NAME, top_level_tracked_resource_name="top" - ) + await client.top_level.get(resource_group_name=RESOURCE_GROUP_NAME, top_level_tracked_resource_name="top") @pytest.mark.asyncio diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/test_azure_arm_resource.py b/packages/http-client-python/generator/test/azure/mock_api_tests/test_azure_arm_resource.py index d79bc05168..ecc550477e 100644 --- a/packages/http-client-python/generator/test/azure/mock_api_tests/test_azure_arm_resource.py +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/test_azure_arm_resource.py @@ -35,9 +35,7 @@ def test_client_signature(credential, authentication_policy): # make sure signautre order is correct client.top_level.get(RESOURCE_GROUP_NAME, "top") # make sure signautre name is correct - client.top_level.get( - resource_group_name=RESOURCE_GROUP_NAME, top_level_tracked_resource_name="top" - ) + client.top_level.get(resource_group_name=RESOURCE_GROUP_NAME, top_level_tracked_resource_name="top") def test_top_level_begin_create_or_replace(client): From 3613073b5828132528ca3828cd64bae8e41d59e7 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 3 Dec 2024 15:42:30 +0800 Subject: [PATCH 09/49] format --- .../generator/pygen/codegen/_utils.py | 8 ++-- .../generator/pygen/codegen/models/client.py | 5 +-- .../pygen/codegen/models/code_model.py | 42 +++++++++---------- .../pygen/codegen/models/enum_type.py | 6 +-- .../generator/pygen/codegen/models/imports.py | 5 --- .../pygen/codegen/models/model_type.py | 5 +-- .../generator/pygen/codegen/models/utils.py | 16 ++++++- .../operation_groups_serializer.py | 2 - .../pygen/codegen/serializers/utils.py | 14 +------ 9 files changed, 47 insertions(+), 56 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/_utils.py b/packages/http-client-python/generator/pygen/codegen/_utils.py index a59235464a..b7133e3aa0 100644 --- a/packages/http-client-python/generator/pygen/codegen/_utils.py +++ b/packages/http-client-python/generator/pygen/codegen/_utils.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +from typing import List DEFAULT_HEADER_TEXT = ( "Copyright (c) {company_name} Corporation. All rights reserved.\n" @@ -17,9 +18,8 @@ NAME_LENGTH_LIMIT = 40 -def get_unique_alias(relative_import: str, module_name: str = "models") -> str: - pass - - def get_parent_namespace(namespace: str) -> str: return namespace.rsplit(".", 1)[0] + + + diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index c40f01ed78..f0a0049e8f 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -207,15 +207,14 @@ def _imports_shared(self, async_mode: bool, **kwargs) -> FileImport: file_import.merge( gp.imports( async_mode, - in_operation_file=True, + is_operation_file=True, **kwargs, ) ) file_import.add_submodule_import( - "_configuration", + "._configuration", f"{self.name}Configuration", ImportType.LOCAL, - client_namespace=self.client_namespace, ) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_msrest_import( diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 96042d53d1..6cd4e4eab7 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -12,8 +12,7 @@ from .client import Client from .request_builder import RequestBuilder, OverloadedRequestBuilder from .operation_group import OperationGroup -from .utils import NamespaceType -from ..serializers.utils import get_all_operation_groups_recursively +from .utils import NamespaceType, get_all_operation_groups_recursively def _is_legacy(options) -> bool: @@ -23,13 +22,11 @@ def _is_legacy(options) -> bool: class ClientNamespaceType: def __init__( self, - client_namespace: str, clients: List[Client] = [], models: List[ModelType] = [], enums: List[EnumType] = [], operation_groups: List[OperationGroup] = [], ): - self.client_namespace = client_namespace self.clients = clients self.models = models self.enums = enums @@ -86,6 +83,7 @@ def __init__( self._client_namespace_types: Dict[str, ClientNamespaceType] = {} self.has_subnamespace = False self._operations_folder_name: Dict[str, str] = {} + self._relative_import_path: Dict[str, str] = {} # | serialize_namespace | imported_namespace | relative_import_path | # |----------------------|--------------------|----------------------| @@ -114,17 +112,21 @@ def get_relative_import_path( else: module_namespace = "" imported_namespace = imported_namespace + async_namespace + module_namespace - idx = 0 - while idx < min(len(serialize_namespace), len(imported_namespace)): - if serialize_namespace[idx] != imported_namespace[idx]: - break - idx += 1 - return "." * (len(serialize_namespace[idx:].strip(".").split(".")) + 1) + imported_namespace[idx:].strip(".") + + key = f"{serialize_namespace}-{imported_namespace}" + if key not in self._relative_import_path: + idx = 0 + while idx < min(len(serialize_namespace), len(imported_namespace)): + if serialize_namespace[idx] != imported_namespace[idx]: + break + idx += 1 + self._relative_import_path[key] = "." * (len(serialize_namespace[idx:].strip(".").split(".")) + 1) + imported_namespace[idx:].strip(".") + return self._relative_import_path[key] @property def need_unique_model_alias(self) -> bool: - return self.options["enable_typespec_namespace"] and self.has_subnamespace - + return self.has_subnamespace and self.options["enable_typespec_namespace"] + def get_unique_models_alias(self, serialize_namespace: str, imported_namespace: str) -> str: if not self.need_unique_model_alias: return "_models." @@ -138,21 +140,19 @@ def client_namespace_types(self) -> Dict[str, ClientNamespaceType]: if not self._client_namespace_types: for client in self.clients: if client.client_namespace not in self._client_namespace_types: - self._client_namespace_types[client.client_namespace] = ClientNamespaceType(client.client_namespace) + self._client_namespace_types[client.client_namespace] = ClientNamespaceType() self._client_namespace_types[client.client_namespace].clients.append(client) for model in self.model_types: if model.client_namespace not in self._client_namespace_types: - self._client_namespace_types[model.client_namespace] = ClientNamespaceType(model.client_namespace) + self._client_namespace_types[model.client_namespace] = ClientNamespaceType() self._client_namespace_types[model.client_namespace].models.append(model) for enum in self.enums: if enum.client_namespace not in self._client_namespace_types: - self._client_namespace_types[enum.client_namespace] = ClientNamespaceType(enum.client_namespace) + self._client_namespace_types[enum.client_namespace] = ClientNamespaceType() self._client_namespace_types[enum.client_namespace].enums.append(enum) for operation_group in get_all_operation_groups_recursively(self.clients): if operation_group.client_namespace not in self._client_namespace_types: - self._client_namespace_types[operation_group.client_namespace] = ClientNamespaceType( - operation_group.client_namespace - ) + self._client_namespace_types[operation_group.client_namespace] = ClientNamespaceType() self._client_namespace_types[operation_group.client_namespace].operation_groups.append(operation_group) if len(self._client_namespace_types.keys()) > 1: @@ -164,7 +164,7 @@ def client_namespace_types(self) -> Dict[str, ClientNamespaceType]: for idx in range(len(namespace_parts) + 1): namespace = ".".join(namespace_parts[:idx]) if namespace not in self._client_namespace_types: - self._client_namespace_types[namespace] = ClientNamespaceType(namespace) + self._client_namespace_types[namespace] = ClientNamespaceType() return self._client_namespace_types @property @@ -227,9 +227,7 @@ def operations_folder_name(self, client_namespace: str) -> str: name = "operations" client_namespace_type = self.client_namespace_types.get(client_namespace) operation_groups = client_namespace_type.operation_groups if client_namespace_type else [] - if self.options["version_tolerant"] and not any( - og for client in self.clients for og in operation_groups if not og.is_mixin - ): + if self.options["version_tolerant"] and all(og.is_mixin for og in operation_groups): name = f"_{name}" self._operations_folder_name[client_namespace] = name return self._operations_folder_name[client_namespace] diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 429496ba33..855e397671 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -221,7 +221,7 @@ def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "EnumT ) def imports(self, **kwargs: Any) -> FileImport: - in_operation_file = kwargs.pop("in_operation_file", False) + in_operation_file = kwargs.get("in_operation_file", False) file_import = FileImport(self.code_model) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) if self.code_model.options["models_mode"]: @@ -234,7 +234,7 @@ def imports(self, **kwargs: Any) -> FileImport: TypingSection.TYPING, alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), ) - file_import.merge(self.value_type.imports(in_operation_file=in_operation_file, **kwargs)) + file_import.merge(self.value_type.imports(**kwargs)) relative_path = kwargs.pop("relative_path", None) if self.code_model.options["models_mode"] and relative_path: # add import for enums in operations file @@ -243,6 +243,6 @@ def imports(self, **kwargs: Any) -> FileImport: "models", ImportType.LOCAL, alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), # TODO + typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), ) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/imports.py b/packages/http-client-python/generator/pygen/codegen/models/imports.py index ab64d0ff21..36707ac987 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/imports.py +++ b/packages/http-client-python/generator/pygen/codegen/models/imports.py @@ -45,7 +45,6 @@ def __init__( submodule_name: Optional[str] = None, alias: Optional[str] = None, version_modules: Optional[Tuple[Tuple[Tuple[int, int], str, Optional[str]]]] = None, - client_namespace: Optional[str] = None, # namespace where the imported model is ): self.typing_section = typing_section self.import_type = import_type @@ -56,7 +55,6 @@ def __init__( # It's a list of "python version, module_name, comments". # The python version is in form of (major, minor), for instance (3, 9) stands for py3.9. self.version_modules = version_modules - self.client_namespace = client_namespace def __eq__(self, other): try: @@ -125,8 +123,6 @@ def add_submodule_import( typing_section: TypingSection = TypingSection.REGULAR, alias: Optional[str] = None, version_modules: Optional[Tuple[Tuple[Tuple[int, int], str, Optional[str]]]] = None, - *, - client_namespace: Optional[str] = None, # namespace where the imported model is ) -> None: """Add an import to this import block.""" self._append_import( @@ -137,7 +133,6 @@ def add_submodule_import( submodule_name=submodule_name, alias=alias, version_modules=version_modules, - client_namespace=client_namespace or self.code_model.namespace, ) ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 81412cea9c..4410c81538 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -307,9 +307,8 @@ def imports(self, **kwargs: Any) -> FileImport: relative_path, "models", ImportType.LOCAL, - alias="_models", - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), # TODO, - client_namespace=self.client_namespace, + alias=self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), + typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), ) if self.is_form_data: file_import.add_submodule_import( diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index a1a04101a5..6693630416 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -3,7 +3,10 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import TypeVar, Dict +from typing import TypeVar, Dict, List +from .client import Client +from .operation_group import OperationGroup + from enum import Enum T = TypeVar("T") @@ -28,3 +31,14 @@ class NamespaceType(str, Enum): NONE = "none" MODEL = "model" OPERATION = "operation" + +def get_all_operation_groups_recursively(clients: List[Client]) -> List[OperationGroup]: + operation_groups = [] + queue = [] + for client in clients: + queue.extend(client.operation_groups) + while queue: + operation_groups.append(queue.pop(0)) + if operation_groups[-1].operation_groups: + queue.extend(operation_groups[-1].operation_groups) + return operation_groups diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py index 0e431e049c..952d2f167a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -7,13 +7,11 @@ import functools from jinja2 import Environment -from .utils import get_all_operation_groups_recursively from ..models import ( CodeModel, OperationGroup, RequestBuilder, OverloadedRequestBuilder, - Client, FileImport, ) from ..models.utils import NamespaceType diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py index 7cd6f7c926..bd10b31577 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py @@ -4,10 +4,9 @@ # license information. # -------------------------------------------------------------------------- import json -from typing import Optional, List, Any +from typing import Optional, Any from pathlib import Path -from ..models import Client, OperationGroup def method_signature_and_response_type_annotation_template( @@ -35,17 +34,6 @@ def get_namespace_from_package_name(package_name: Optional[str]) -> str: return (package_name or "").replace("-", ".") -def get_all_operation_groups_recursively(clients: List[Client]) -> List[OperationGroup]: - operation_groups = [] - queue = [] - for client in clients: - queue.extend(client.operation_groups) - while queue: - operation_groups.append(queue.pop(0)) - if operation_groups[-1].operation_groups: - queue.extend(operation_groups[-1].operation_groups) - return operation_groups - def _improve_json_string(template_representation: str) -> Any: origin = template_representation.split("\n") From 5ea817c8b6e182c35bd385505a65ccdb532730c7 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 3 Dec 2024 17:00:20 +0800 Subject: [PATCH 10/49] review --- packages/http-client-python/.vscode/launch.json | 2 +- .../generator/pygen/codegen/models/client.py | 1 - .../generator/pygen/codegen/models/code_model.py | 14 ++++++++++++-- .../generator/pygen/codegen/models/property.py | 2 ++ .../generator/pygen/codegen/models/utils.py | 14 +------------- .../serializers/operation_groups_serializer.py | 4 ++-- .../serializers/operations_init_serializer.py | 2 +- .../codegen/serializers/parameter_serializer.py | 9 ++++----- 8 files changed, 23 insertions(+), 25 deletions(-) diff --git a/packages/http-client-python/.vscode/launch.json b/packages/http-client-python/.vscode/launch.json index ad210f3787..ab9e01f1a5 100644 --- a/packages/http-client-python/.vscode/launch.json +++ b/packages/http-client-python/.vscode/launch.json @@ -26,7 +26,7 @@ "cwd": "${workspaceFolder}", "args": [ "compile", - "${workspaceFolder}/alpha/common-properties/main.tsp", + "${workspaceFolder}/alpha/main.tsp", "--emit=${workspaceFolder}", "--option=@typespec/http-client-python.flavor=azure" ], diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index f0a0049e8f..221c81c2eb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -307,7 +307,6 @@ def imports(self, async_mode: bool, **kwargs) -> FileImport: f"{self.code_model.operations_folder_name(og.client_namespace)}", og.class_name, ImportType.LOCAL, - client_namespace=og.client_namespace, ) if self.code_model.model_types and self.code_model.options["models_mode"] == "msrest": diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 6cd4e4eab7..89293602ff 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -12,12 +12,22 @@ from .client import Client from .request_builder import RequestBuilder, OverloadedRequestBuilder from .operation_group import OperationGroup -from .utils import NamespaceType, get_all_operation_groups_recursively +from .utils import NamespaceType def _is_legacy(options) -> bool: return not (options.get("version_tolerant") or options.get("low_level_client")) +def get_all_operation_groups_recursively(clients: List[Client]) -> List[OperationGroup]: + operation_groups = [] + queue = [] + for client in clients: + queue.extend(client.operation_groups) + while queue: + operation_groups.append(queue.pop(0)) + if operation_groups[-1].operation_groups: + queue.extend(operation_groups[-1].operation_groups) + return operation_groups class ClientNamespaceType: def __init__( @@ -241,7 +251,7 @@ def get_serialize_namespace( return client_namespace + ".models" operations_folder_name = self.operations_folder_name(client_namespace) - return client_namespace + (".aio." if async_mode else ".") + +operations_folder_name + return client_namespace + (".aio." if async_mode else ".") + operations_folder_name @property def description(self) -> str: diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index ab7f7eae5c..8c21c431dd 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -140,6 +140,8 @@ def validation(self) -> Optional[Dict[str, Any]]: return retval or None def imports(self, **kwargs) -> FileImport: + from .model_type import ModelType + file_import = FileImport(self.code_model) if self.is_discriminator and isinstance(self.type, EnumType): return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index 6693630416..082bfa1296 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -3,9 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import TypeVar, Dict, List -from .client import Client -from .operation_group import OperationGroup +from typing import TypeVar, Dict from enum import Enum @@ -32,13 +30,3 @@ class NamespaceType(str, Enum): MODEL = "model" OPERATION = "operation" -def get_all_operation_groups_recursively(clients: List[Client]) -> List[OperationGroup]: - operation_groups = [] - queue = [] - for client in clients: - queue.extend(client.operation_groups) - while queue: - operation_groups.append(queue.pop(0)) - if operation_groups[-1].operation_groups: - queue.extend(operation_groups[-1].operation_groups) - return operation_groups diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py index 952d2f167a..9fcbe41f61 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -31,9 +31,9 @@ def __init__( env: Environment, async_mode: bool, *, - serialize_namespace: Optional[str] = None, + client_namespace: Optional[str] = None, ): - super().__init__(code_model, env, async_mode, serialize_namespace=serialize_namespace) + super().__init__(code_model, env, async_mode, client_namespace=client_namespace) self.operation_groups = operation_groups self.async_mode = async_mode diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py index c5659d5c4a..4f49832f36 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py @@ -19,7 +19,7 @@ def __init__( async_mode: bool, ) -> None: self.code_model = code_model - self.operation_groups = [og for og in self.operation_groups if not og.has_parent_operation_group] + self.operation_groups = [og for og in operation_groups if not og.has_parent_operation_group] self.env = env self.async_mode = async_mode diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py index f83feedb4a..1a27d30400 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py @@ -109,8 +109,7 @@ def serialize_parameter(self, parameter: ParameterType, serializer_name: str) -> return f"[{serialize_line} if q is not None else '' for q in {origin_name}]" return serialize_line - @staticmethod - def serialize_path( + def serialize_path(self, parameters: Union[ List[Parameter], List[RequestBuilderParameter], @@ -124,7 +123,7 @@ def serialize_path( [ ' "{}": {},'.format( path_parameter.wire_name, - ParameterSerializer.serialize_parameter(path_parameter, serializer_name), + self.serialize_parameter(path_parameter, serializer_name), ) for path_parameter in parameters ] @@ -132,8 +131,8 @@ def serialize_path( retval.append("}") return retval - @staticmethod def serialize_query_header( + self, param: Parameter, kwarg_name: str, serializer_name: str, @@ -149,7 +148,7 @@ def serialize_query_header( set_parameter = "_{}['{}'] = {}".format( kwarg_name, param.wire_name, - ParameterSerializer.serialize_parameter(param, serializer_name), + self.serialize_parameter(param, serializer_name), ) if not param.optional and (param.in_method_signature or param.constant): retval = [set_parameter] From ed61e31fae0c5942e8c6dfcc66717e8423572141 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 3 Dec 2024 17:35:50 +0800 Subject: [PATCH 11/49] review --- packages/http-client-python/emitter/src/utils.ts | 6 +++--- .../generator/pygen/codegen/models/code_model.py | 4 ++-- .../generator/pygen/codegen/models/enum_type.py | 2 +- .../generator/pygen/codegen/models/model_type.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/http-client-python/emitter/src/utils.ts b/packages/http-client-python/emitter/src/utils.ts index 98407a009c..f8c54ed9b5 100644 --- a/packages/http-client-python/emitter/src/utils.ts +++ b/packages/http-client-python/emitter/src/utils.ts @@ -206,14 +206,14 @@ export function getClientNamespace clientNamespace.toLowerCase().startsWith(item), ) ) { - return context.sdkPackage.rootNamespace; + return context.sdkPackage.rootNamespace.toLowerCase(); } - return clientNamespace; + return clientNamespace.toLowerCase(); } diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 89293602ff..cb242beb1b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -108,7 +108,7 @@ def get_relative_import_path( serialize_namespace: str, imported_namespace: Optional[str] = None, *, - namespace_type: NamespaceType = NamespaceType.MODEL, + namespace_type: NamespaceType = NamespaceType.NONE, async_mode: bool = False, ) -> str: if imported_namespace is None: @@ -139,7 +139,7 @@ def need_unique_model_alias(self) -> bool: def get_unique_models_alias(self, serialize_namespace: str, imported_namespace: str) -> str: if not self.need_unique_model_alias: - return "_models." + return "_models" relative_path = self.get_relative_import_path(serialize_namespace, imported_namespace) dot_num = max(relative_path.count(".") - 1, 0) parts = [""] + [p for p in relative_path.split(".") if p] diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 855e397671..973c75dd75 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -169,7 +169,7 @@ def type_annotation(self, **kwargs: Any) -> str: if self.code_model.options["models_mode"]: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) - module_name = model_alias if kwargs.get("need_model_alias", True) else "" + module_name = f"{model_alias}." if kwargs.get("need_model_alias", True) else "" file_name = f"{self.code_model.enums_filename}." if self.internal else "" model_name = module_name + file_name + self.name # we don't need quoted annotation in operation files, and need it in model folder files. diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 4410c81538..718af0b2f6 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -283,7 +283,7 @@ def type_annotation(self, **kwargs: Any) -> str: skip_quote = kwargs.get("skip_quote", False) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) - module_name = model_alias if kwargs.get("need_model_alias", True) else "" + module_name = f"{model_alias}." if kwargs.get("need_model_alias", True) else "" file_name = f"{self.code_model.models_filename}." if self.internal else "" retval = module_name + file_name + self.name return retval if is_operation_file or skip_quote else f'"{retval}"' @@ -307,7 +307,7 @@ def imports(self, **kwargs: Any) -> FileImport: relative_path, "models", ImportType.LOCAL, - alias=self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), + alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), ) if self.is_form_data: From 682b381da97b6250ec2fde90a9d04c1f751fb9fe Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 4 Dec 2024 11:31:48 +0800 Subject: [PATCH 12/49] review --- .../emitter/src/external-process.ts | 1 + .../generator/pygen/codegen/models/client.py | 2 +- .../pygen/codegen/models/code_model.py | 18 +++++++++--------- .../generator/pygen/codegen/models/imports.py | 4 ++-- .../pygen/codegen/models/lro_operation.py | 2 +- .../pygen/codegen/models/operation.py | 16 ++++++++-------- .../pygen/codegen/models/operation_group.py | 4 ++-- .../pygen/codegen/models/paging_operation.py | 2 +- .../pygen/codegen/models/parameter.py | 4 ++-- .../pygen/codegen/models/primitive_types.py | 4 ++-- .../generator/pygen/codegen/models/property.py | 2 +- .../pygen/codegen/serializers/__init__.py | 4 ++-- 12 files changed, 32 insertions(+), 31 deletions(-) diff --git a/packages/http-client-python/emitter/src/external-process.ts b/packages/http-client-python/emitter/src/external-process.ts index 1e3ca74fb6..23ddfe2eed 100644 --- a/packages/http-client-python/emitter/src/external-process.ts +++ b/packages/http-client-python/emitter/src/external-process.ts @@ -22,6 +22,7 @@ export async function saveCodeModelAsYaml(name: string, codemodel: unknown): Pro const filename = createTempPath(".yaml", name); const yamlStr = jsyaml.dump(codemodel); await writeFile(filename, yamlStr); + await writeFile(joinPaths("C:/dev/typespec/packages/http-client-python", "alpha", "output.yaml"), yamlStr); return filename; } diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index 221c81c2eb..06161b01e5 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -390,7 +390,7 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: if self.code_model.options["package_version"]: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}._version", "VERSION", ImportType.LOCAL + f"{self.code_model.get_relative_import_path(serialize_namespace)}_version", "VERSION", ImportType.LOCAL ) if self.code_model.options["azure_arm"]: policy = "AsyncARMChallengeAuthenticationPolicy" if async_mode else "ARMChallengeAuthenticationPolicy" diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index cb242beb1b..4745b01a47 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -32,15 +32,15 @@ def get_all_operation_groups_recursively(clients: List[Client]) -> List[Operatio class ClientNamespaceType: def __init__( self, - clients: List[Client] = [], - models: List[ModelType] = [], - enums: List[EnumType] = [], - operation_groups: List[OperationGroup] = [], + clients: Optional[List[Client]] = None, + models: Optional[List[ModelType]] = None, + enums: Optional[List[EnumType]] = None, + operation_groups: Optional[List[OperationGroup]] = None, ): - self.clients = clients - self.models = models - self.enums = enums - self.operation_groups = operation_groups + self.clients = clients or [] + self.models = models or [] + self.enums = enums or [] + self.operation_groups = operation_groups or [] class CodeModel: # pylint: disable=too-many-public-methods, disable=too-many-instance-attributes @@ -130,7 +130,7 @@ def get_relative_import_path( if serialize_namespace[idx] != imported_namespace[idx]: break idx += 1 - self._relative_import_path[key] = "." * (len(serialize_namespace[idx:].strip(".").split(".")) + 1) + imported_namespace[idx:].strip(".") + self._relative_import_path[key] = "." * len(serialize_namespace[idx:].strip(".").split(".")) + imported_namespace[idx:].strip(".") return self._relative_import_path[key] @property diff --git a/packages/http-client-python/generator/pygen/codegen/models/imports.py b/packages/http-client-python/generator/pygen/codegen/models/imports.py index 36707ac987..ae4a9acb75 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/imports.py +++ b/packages/http-client-python/generator/pygen/codegen/models/imports.py @@ -282,9 +282,9 @@ def add_msrest_import( self.add_submodule_import(relative_path, "_serialization", ImportType.LOCAL, typing_section) else: self.add_submodule_import( - f"{relative_path}._serialization", "Serializer", ImportType.LOCAL, typing_section + f"{relative_path}_serialization", "Serializer", ImportType.LOCAL, typing_section ) if msrest_import_type == MsrestImportType.SerializerDeserializer: self.add_submodule_import( - f"{relative_path}._serialization", "Deserializer", ImportType.LOCAL, typing_section + f"{relative_path}_serialization", "Deserializer", ImportType.LOCAL, typing_section ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index 69fb8e46ad..3085872c7a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -134,7 +134,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: # but final call returns a model serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}._model_base", + f"{self.code_model.get_relative_import_path(serialize_namespace)}_model_base", "_deserialize", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 87aef35991..49eba181ac 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -224,7 +224,7 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: # pyl if self.added_on: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}._validation", + f"{self.code_model.get_relative_import_path(serialize_namespace)}_validation", "api_version_validation", ImportType.LOCAL, ) @@ -400,12 +400,12 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements ) if not async_mode: file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", + f"{self.code_model.get_relative_import_path(serialize_namespace)}_vendor", "prep_if_match", ImportType.LOCAL, ) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", + f"{self.code_model.get_relative_import_path(serialize_namespace)}_vendor", "prep_if_none_match", ImportType.LOCAL, ) @@ -449,23 +449,23 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements file_import.add_submodule_import(relative_path, "_model_base", ImportType.LOCAL) elif xml_serializable(self.parameters.body_parameter.default_content_type): file_import.add_submodule_import( - f"{relative_path}._model_base", + f"{relative_path}_model_base", "_get_element", ImportType.LOCAL, ) elif json_serializable(self.parameters.body_parameter.default_content_type): file_import.add_submodule_import( - f"{relative_path}._model_base", + f"{relative_path}_model_base", "SdkJSONEncoder", ImportType.LOCAL, ) file_import.add_import("json", ImportType.STDLIB) if any(xml_serializable(str(r.default_content_type)) for r in self.responses): - file_import.add_submodule_import(f"{relative_path}._model_base", "_deserialize_xml", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize_xml", ImportType.LOCAL) elif any(r.type for r in self.responses): - file_import.add_submodule_import(f"{relative_path}._model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) if self.default_error_deserialization or self.non_default_errors: - file_import.add_submodule_import(f"{relative_path}._model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) return file_import def get_response_from_status(self, status_code: Optional[Union[str, int]]) -> ResponseType: diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index dbe0c690c4..5eccdb922b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -126,10 +126,10 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") if self.is_mixin: file_import.add_submodule_import( - f"{relative_path}._vendor", f"{self.client.name}MixinABC", ImportType.LOCAL + f"{relative_path}_vendor", f"{self.client.name}MixinABC", ImportType.LOCAL ) if self.has_abstract_operations: - file_import.add_submodule_import(f"{relative_path}._vendor", "raise_if_not_implemented", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}_vendor", "raise_if_not_implemented", ImportType.LOCAL) if all(o.abstract for o in self.operations): return file_import file_import.add_submodule_import("typing", "TypeVar", ImportType.STDLIB, TypingSection.CONDITIONAL) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index 0c232cb9b2..f03b63eac0 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -149,7 +149,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: relative_path = self.code_model.get_relative_import_path(serialize_namespace) file_import.merge(self.item_type.imports(**kwargs)) if self.default_error_deserialization or any(r.type for r in self.responses): - file_import.add_submodule_import(f"{relative_path}._model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter.py b/packages/http-client-python/generator/pygen/codegen/models/parameter.py index 507646256a..1b1b058275 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter.py @@ -166,7 +166,7 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: relative_path = self.code_model.get_relative_import_path(serialize_namespace) if self.added_on and self.implementation != "Client": file_import.add_submodule_import( - f"{relative_path}._validation", + f"{relative_path}_validation", "api_version_validation", ImportType.LOCAL, ) @@ -275,7 +275,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: if self.is_form_data: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}._vendor", + f"{self.code_model.get_relative_import_path(serialize_namespace)}_vendor", "prepare_multipart_form_data", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py index 6986e56d15..3089acc83a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py +++ b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py @@ -621,13 +621,13 @@ def type_annotation(self, **kwargs: Any) -> str: return self.name def docstring_type(self, **kwargs: Any) -> str: - return f"~{self.code_model.namespace}._vendor.{self.name}" + return f"~{self.code_model.namespace}_vendor.{self.name}" def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) relative_path = self.code_model.get_relative_import_path(serialize_namespace) - file_import.add_submodule_import(f"{relative_path}._vendor", self.name, ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}_vendor", self.name, ImportType.LOCAL) return file_import @property diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index 8c21c431dd..575a13e797 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -151,7 +151,7 @@ def imports(self, **kwargs) -> FileImport: if self.code_model.options["models_mode"] == "dpg" and isinstance(self.type, ModelType): serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}._model_base", + f"{self.code_model.get_relative_import_path(serialize_namespace)}_model_base", "rest_discriminator" if self.is_discriminator else "rest_field", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index f39bed0159..1e72f2d8b1 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -296,9 +296,9 @@ def _serialize_and_write_operations_folder( operations_folder_name = self.code_model.operations_folder_name(namespace) # if there was a patch file before, we keep it - self._keep_patch_file(self.exec_path(namespace) / Path("_patch.py"), env) + self._keep_patch_file(self.exec_path(namespace) / operations_folder_name / Path("_patch.py"), env) if self.has_aio_folder: - self._keep_patch_file(self.exec_path(namespace) / Path("aio") / Path("_patch.py"), env) + self._keep_patch_file(self.exec_path(namespace) / Path("aio") / operations_folder_name / Path("_patch.py"), env) # write sync operations init file operations_init_serializer = OperationsInitSerializer( From 32399e8816f55b3f6c5836b16e26c10ce9a5dd10 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 4 Dec 2024 13:15:28 +0800 Subject: [PATCH 13/49] review --- .../generator/pygen/codegen/models/client.py | 2 +- .../pygen/codegen/models/code_model.py | 32 ++++++++++++------- .../generator/pygen/codegen/models/imports.py | 8 ++--- .../pygen/codegen/models/lro_operation.py | 2 +- .../pygen/codegen/models/operation.py | 23 ++++++------- .../pygen/codegen/models/paging_operation.py | 4 +-- .../pygen/codegen/models/parameter.py | 7 ++-- .../pygen/codegen/models/primitive_types.py | 3 +- .../pygen/codegen/models/property.py | 2 +- .../pygen/codegen/models/response.py | 1 - 10 files changed, 45 insertions(+), 39 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index 06161b01e5..0e9a4d04a6 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -390,7 +390,7 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: if self.code_model.options["package_version"]: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}_version", "VERSION", ImportType.LOCAL + f"{self.code_model.get_relative_import_path(serialize_namespace, module_name="_version")}", "VERSION", ImportType.LOCAL ) if self.code_model.options["azure_arm"]: policy = "AsyncARMChallengeAuthenticationPolicy" if async_mode else "ARMChallengeAuthenticationPolicy" diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 4745b01a47..b0fa49a92a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -95,14 +95,15 @@ def __init__( self._operations_folder_name: Dict[str, str] = {} self._relative_import_path: Dict[str, str] = {} - # | serialize_namespace | imported_namespace | relative_import_path | - # |----------------------|--------------------|----------------------| - # |azure.test.operations | azure.test | .. | - # |azure.test.operations | azure.test.subtest | ..subtest | - # |azure.test.operations | azure | ... | - # |azure.test.aio.operations | azure.test | ... | - # |azure.test.subtest.aio.operations|azure.test| .... | - # |azure.test |azure.test.subtest | .subtest | + # | serialize_namespace | imported_namespace | relative_import_path | + # |----------------------|----------------------|----------------------| + # |azure.test.operations | azure.test.operations| .. | + # |azure.test.operations | azure.test | .. | + # |azure.test.operations | azure.test.subtest | ..subtest | + # |azure.test.operations | azure | ... | + # |azure.test.aio.operations | azure.test | ... | + # |azure.test.subtest.aio.operations|azure.test | .... | + # |azure.test |azure.test.subtest | .subtest | def get_relative_import_path( self, serialize_namespace: str, @@ -110,6 +111,7 @@ def get_relative_import_path( *, namespace_type: NamespaceType = NamespaceType.NONE, async_mode: bool = False, + module_name: Optional[str] = None, ) -> str: if imported_namespace is None: imported_namespace = self.namespace @@ -126,12 +128,18 @@ def get_relative_import_path( key = f"{serialize_namespace}-{imported_namespace}" if key not in self._relative_import_path: idx = 0 - while idx < min(len(serialize_namespace), len(imported_namespace)): - if serialize_namespace[idx] != imported_namespace[idx]: + serialize_namespace_split = serialize_namespace.split(".") + imported_namespace_split = imported_namespace.split(".") + while idx < min(len(serialize_namespace_split), len(imported_namespace_split)): + if serialize_namespace_split[idx] != imported_namespace_split[idx]: break idx += 1 - self._relative_import_path[key] = "." * len(serialize_namespace[idx:].strip(".").split(".")) + imported_namespace[idx:].strip(".") - return self._relative_import_path[key] + self._relative_import_path[key] = "." * (len(serialize_namespace_split[idx:]) + 1) + ".".join(imported_namespace_split[idx:]) + result = self._relative_import_path[key] + if module_name is None: + return result + return f"{result}{module_name}" if result.endswith(".") else f"{result}.{module_name}" + @property def need_unique_model_alias(self) -> bool: diff --git a/packages/http-client-python/generator/pygen/codegen/models/imports.py b/packages/http-client-python/generator/pygen/codegen/models/imports.py index ae4a9acb75..6ae60a1f95 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/imports.py +++ b/packages/http-client-python/generator/pygen/codegen/models/imports.py @@ -277,14 +277,14 @@ def add_msrest_import( if self.code_model.options["multiapi"]: # for multiapi, the namespace is azure.mgmt.xxx.v20XX_XX_XX while _serialization.py is in azure.mgmt.xxx imported_namespace = get_parent_namespace(imported_namespace) - relative_path = self.code_model.get_relative_import_path(serialize_namespace, imported_namespace) if msrest_import_type == MsrestImportType.Module: - self.add_submodule_import(relative_path, "_serialization", ImportType.LOCAL, typing_section) + self.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace, imported_namespace), "_serialization", ImportType.LOCAL, typing_section) else: + relative_path = self.code_model.get_relative_import_path(serialize_namespace, imported_namespace, module_name="_serialization") self.add_submodule_import( - f"{relative_path}_serialization", "Serializer", ImportType.LOCAL, typing_section + relative_path, "Serializer", ImportType.LOCAL, typing_section ) if msrest_import_type == MsrestImportType.SerializerDeserializer: self.add_submodule_import( - f"{relative_path}_serialization", "Deserializer", ImportType.LOCAL, typing_section + relative_path, "Deserializer", ImportType.LOCAL, typing_section ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index 3085872c7a..d20a48c16c 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -134,7 +134,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: # but final call returns a model serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}_model_base", + self.code_model.get_relative_import_path(serialize_namespace, module_name='_model_base'), "_deserialize", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 49eba181ac..c0690614cb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -224,7 +224,7 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: # pyl if self.added_on: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}_validation", + self.code_model.get_relative_import_path(serialize_namespace, module_name="_validation"), "api_version_validation", ImportType.LOCAL, ) @@ -308,7 +308,7 @@ def get_request_builder_import( ) if self.code_model.options["builders_visibility"] == "embedded" and async_mode: file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace, namespace_type=NamespaceType.OPERATION)}.{self.filename}", + self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace, namespace_type=NamespaceType.OPERATION, module_name=self.filename), request_builder.name, import_type=ImportType.LOCAL, ) @@ -399,13 +399,14 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements ImportType.SDKCORE, ) if not async_mode: + relative_path = self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor") file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}_vendor", + relative_path, "prep_if_match", ImportType.LOCAL, ) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}_vendor", + relative_path, "prep_if_none_match", ImportType.LOCAL, ) @@ -443,29 +444,29 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements if self.overloads: file_import.add_submodule_import("typing", "overload", ImportType.STDLIB) if self.code_model.options["models_mode"] == "dpg": - relative_path = self.code_model.get_relative_import_path(serialize_namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, module_name="_model_base") if self.parameters.has_body: if self.has_form_data_body: - file_import.add_submodule_import(relative_path, "_model_base", ImportType.LOCAL) + file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace), "_model_base", ImportType.LOCAL) elif xml_serializable(self.parameters.body_parameter.default_content_type): file_import.add_submodule_import( - f"{relative_path}_model_base", + relative_path, "_get_element", ImportType.LOCAL, ) elif json_serializable(self.parameters.body_parameter.default_content_type): file_import.add_submodule_import( - f"{relative_path}_model_base", + relative_path, "SdkJSONEncoder", ImportType.LOCAL, ) file_import.add_import("json", ImportType.STDLIB) if any(xml_serializable(str(r.default_content_type)) for r in self.responses): - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize_xml", ImportType.LOCAL) + file_import.add_submodule_import(relative_path, "_deserialize_xml", ImportType.LOCAL) elif any(r.type for r in self.responses): - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) if self.default_error_deserialization or self.non_default_errors: - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) return file_import def get_response_from_status(self, status_code: Optional[Union[str, int]]) -> ResponseType: diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index f03b63eac0..e663061855 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -146,10 +146,10 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ) if self.code_model.options["models_mode"] == "dpg": serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - relative_path = self.code_model.get_relative_import_path(serialize_namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, module_name="_model_base") file_import.merge(self.item_type.imports(**kwargs)) if self.default_error_deserialization or any(r.type for r in self.responses): - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter.py b/packages/http-client-python/generator/pygen/codegen/models/parameter.py index 1b1b058275..9d7c20b55d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter.py @@ -163,16 +163,15 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: if self.optional and self.client_default_value is None: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - relative_path = self.code_model.get_relative_import_path(serialize_namespace) if self.added_on and self.implementation != "Client": file_import.add_submodule_import( - f"{relative_path}_validation", + self.code_model.get_relative_import_path(serialize_namespace, module_name="_validation"), "api_version_validation", ImportType.LOCAL, ) if isinstance(self.type, CombinedType) and self.type.name: file_import.add_submodule_import( - relative_path, + self.code_model.get_relative_import_path(serialize_namespace), "_types", ImportType.LOCAL, TypingSection.TYPING, @@ -275,7 +274,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: if self.is_form_data: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}_vendor", + self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), "prepare_multipart_form_data", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py index 3089acc83a..ca5dbddd70 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py +++ b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py @@ -626,8 +626,7 @@ def docstring_type(self, **kwargs: Any) -> str: def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - relative_path = self.code_model.get_relative_import_path(serialize_namespace) - file_import.add_submodule_import(f"{relative_path}_vendor", self.name, ImportType.LOCAL) + file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), self.name, ImportType.LOCAL) return file_import @property diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index 575a13e797..064b69f9c3 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -151,7 +151,7 @@ def imports(self, **kwargs) -> FileImport: if self.code_model.options["models_mode"] == "dpg" and isinstance(self.type, ModelType): serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace)}_model_base", + self.code_model.get_relative_import_path(serialize_namespace, module_name="_model_base"), "rest_discriminator" if self.is_discriminator else "rest_field", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/response.py b/packages/http-client-python/generator/pygen/codegen/models/response.py index 6dd4c98601..84ff50b37e 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/response.py +++ b/packages/http-client-python/generator/pygen/codegen/models/response.py @@ -118,7 +118,6 @@ def _imports_shared(self, **kwargs: Any) -> FileImport: if self.nullable: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) if isinstance(self.type, CombinedType) and self.type.name: - async_mode = kwargs.get("async_mode", False) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace), From ab818e241e5aadc3fee7e29d99a50575b6880d82 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 4 Dec 2024 14:32:34 +0800 Subject: [PATCH 14/49] review --- .../generator/pygen/codegen/models/client.py | 2 +- .../pygen/codegen/serializers/__init__.py | 145 +++++++----------- 2 files changed, 60 insertions(+), 87 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index 0e9a4d04a6..9ba9b94427 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -390,7 +390,7 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: if self.code_model.options["package_version"]: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - f"{self.code_model.get_relative_import_path(serialize_namespace, module_name="_version")}", "VERSION", ImportType.LOCAL + self.code_model.get_relative_import_path(serialize_namespace, module_name="_version"), "VERSION", ImportType.LOCAL ) if self.code_model.options["azure_arm"]: policy = "AsyncARMChallengeAuthenticationPolicy" if async_mode else "ARMChallengeAuthenticationPolicy" diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 1e72f2d8b1..b12eef276a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -4,6 +4,7 @@ # license information. # -------------------------------------------------------------------------- import logging +from collections import namedtuple from typing import List, Optional, Any, Union from pathlib import Path from jinja2 import PackageLoader, Environment, FileSystemLoader, StrictUndefined @@ -54,7 +55,7 @@ ] _REGENERATE_FILES = {"setup.py", "MANIFEST.in"} - +AsyncInfo = namedtuple("AsyncInfo", ["async_mode", "async_path"]) # extract sub folders. For example, source_file_path is like: # "xxx/resource-manager/Microsoft.XX/stable/2023-04-01/examples/Compute/createOrUpdate/AKSCompute.json", @@ -86,6 +87,12 @@ def has_aio_folder(self) -> bool: def has_operations_folder(self) -> bool: return self.code_model.options["show_operations"] and bool(self.code_model.has_operations) + @property + def serialize_loop(self) -> List[AsyncInfo]: + sync_loop = AsyncInfo(async_mode=False, async_path="") + async_loop = AsyncInfo(async_mode=True, async_path="aio/") + return [sync_loop, async_loop] if self.has_aio_folder else [sync_loop] + def serialize(self) -> None: env = Environment( loader=PackageLoader("pygen.codegen", "templates"), @@ -97,7 +104,6 @@ def serialize(self) -> None: ) general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) - for client_namespace, client_namespace_type in self.code_model.client_namespace_types.items(): if client_namespace == "": # Write the setup file @@ -118,7 +124,7 @@ def serialize(self) -> None: elif client_namespace_type.clients: # add clients folder if there are clients in this namespace self._serialize_client_and_config_files( - client_namespace, general_serializer, client_namespace_type.clients, env + client_namespace, client_namespace_type.clients, env ) else: # add pkgutil init file if no clients in this namespace @@ -261,76 +267,44 @@ def _serialize_and_write_single_rest_layer( ).serialize_init(), ) - def _serialize_and_write_operations_file( - self, - env: Environment, - operation_groups: List[OperationGroup], - namespace: str, - operation_group: Optional[OperationGroup] = None, - ) -> None: - filename = operation_group.filename if operation_group else "_operations" - # write first sync file - operation_group_serializer = OperationGroupsSerializer( - code_model=self.code_model, - operation_groups=operation_groups if operation_groups else [operation_group], - env=env, - async_mode=False, - ) - operations_folder_name = self.code_model.operations_folder_name(namespace) - self.write_file( - self.exec_path(namespace) / Path(operations_folder_name) / Path(f"{filename}.py"), - operation_group_serializer.serialize(), - ) - - if self.has_aio_folder: - # write async operation group and operation files - operation_group_serializer.async_mode = True - self.write_file( - (self.exec_path(namespace) / Path("aio") / Path(operations_folder_name) / Path(f"{filename}.py")), - operation_group_serializer.serialize(), - ) - def _serialize_and_write_operations_folder( self, operation_groups: List[OperationGroup], env: Environment, namespace: str ) -> None: operations_folder_name = self.code_model.operations_folder_name(namespace) - - # if there was a patch file before, we keep it - self._keep_patch_file(self.exec_path(namespace) / operations_folder_name / Path("_patch.py"), env) - if self.has_aio_folder: - self._keep_patch_file(self.exec_path(namespace) / Path("aio") / operations_folder_name / Path("_patch.py"), env) - - # write sync operations init file - operations_init_serializer = OperationsInitSerializer( - code_model=self.code_model, operation_groups=operation_groups, env=env, async_mode=False - ) - self.write_file( - self.exec_path(namespace) / operations_folder_name / Path("__init__.py"), - operations_init_serializer.serialize(), - ) - - # write async operations init file - if self.has_aio_folder: - operations_init_serializer.async_mode = True + exec_path = self.exec_path(namespace) + for async_mode, async_path in self.serialize_loop: + prefix_path = f"{async_path}{operations_folder_name}" + # write init file + operations_init_serializer = OperationsInitSerializer( + code_model=self.code_model, operation_groups=operation_groups, env=env, async_mode=async_mode + ) self.write_file( - self.exec_path(namespace) / "aio" / operations_folder_name / Path("__init__.py"), + exec_path / Path(f"{prefix_path}/__init__.py"), operations_init_serializer.serialize(), ) - if self.code_model.options["combine_operation_files"]: - self._serialize_and_write_operations_file( - env=env, - namespace=namespace, - operation_groups=operation_groups, - ) - else: - for operation_group in operation_groups: - self._serialize_and_write_operations_file( + # write operations file + OgLoop = namedtuple("OgLoop", ["operation_groups", "filename"]) + if self.code_model.options["combine_operation_files"]: + loops = [OgLoop(operation_groups, "_operations")] + else: + loops = [OgLoop([og], og.filename) for og in operation_groups] + for ogs, filename in loops: + operation_group_serializer = OperationGroupsSerializer( + code_model=self.code_model, + operation_groups=ogs, env=env, - namespace=namespace, - operation_group=operation_group, + async_mode=async_mode, + ) + self.write_file( + exec_path / Path(f"{prefix_path}/{filename}.py"), + operation_group_serializer.serialize(), ) + # if there was a patch file before, we keep it + self._keep_patch_file(exec_path / Path(f"{prefix_path}/_patch.py"), env) + + def _serialize_and_write_version_file( self, namespace: str, @@ -359,36 +333,35 @@ def _write_version_file(original_version_file_name: str) -> None: def _serialize_client_and_config_files( self, namespace: str, - general_serializer: GeneralSerializer, clients: List[Client], env: Environment, ) -> None: - general_serializer.async_mode = False - general_serializer.client_namespace = namespace - self.write_file( - self.exec_path(namespace) / Path(f"{self.code_model.client_filename}.py"), - general_serializer.serialize_service_client_file(clients), - ) - self.write_file( - self.exec_path(namespace) / Path("_configuration.py"), - general_serializer.serialize_config_file(clients), - ) + exec_path = self.exec_path(namespace) + for async_mode, async_path in self.serialize_loop: + general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=async_mode) + # when there is client.py, there must be __init__.py + self.write_file( + exec_path / Path(f"{async_path}__init__.py"), + general_serializer.serialize_init_file(clients), + ) + + # write client file + self.write_file( + exec_path / Path(f"{async_path}{self.code_model.client_filename}.py"), + general_serializer.serialize_service_client_file(clients), + ) + + #write config file + self.write_file( + exec_path / Path(f"{async_path}_configuration.py"), + general_serializer.serialize_config_file(clients), + ) + + # if there was a patch file before, we keep it + self._keep_patch_file(exec_path / Path(f"{async_path}_patch.py"), env) - # if there was a patch file before, we keep it - self._keep_patch_file(self.exec_path(namespace) / Path("_patch.py"), env) - general_serializer.async_mode = True - self.write_file( - self.exec_path(namespace) / Path(f"aio/{self.code_model.client_filename}.py"), - general_serializer.serialize_service_client_file(clients), - ) - self.write_file( - self.exec_path(namespace) / Path("aio/_configuration.py"), - general_serializer.serialize_config_file(clients), - ) - if self.has_aio_folder: - self._keep_patch_file(self.exec_path(namespace) / Path("aio/_patch.py"), env) def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str) -> None: general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) From f3f014081471ec49503cdbf1b2ad1eff4ff359b8 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 4 Dec 2024 15:53:19 +0800 Subject: [PATCH 15/49] review --- .../generator/pygen/codegen/models/client.py | 5 ++- .../pygen/codegen/models/code_model.py | 6 +-- .../pygen/codegen/models/enum_type.py | 3 +- .../pygen/codegen/models/model_type.py | 5 +-- .../pygen/codegen/models/property.py | 2 +- .../pygen/codegen/serializers/__init__.py | 44 +++++++++---------- .../codegen/serializers/general_serializer.py | 2 +- 7 files changed, 32 insertions(+), 35 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index 9ba9b94427..2617e25c45 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -8,7 +8,7 @@ from .base import BaseModel from .parameter_list import ClientGlobalParameterList, ConfigGlobalParameterList from .imports import FileImport, ImportType, TypingSection, MsrestImportType -from .utils import add_to_pylint_disable +from .utils import add_to_pylint_disable, NamespaceType from .operation_group import OperationGroup from .request_builder import ( RequestBuilder, @@ -302,9 +302,10 @@ def imports(self, async_mode: bool, **kwargs) -> FileImport: ImportType.SDKCORE, TypingSection.CONDITIONAL, ) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) for og in self.operation_groups: file_import.add_submodule_import( - f"{self.code_model.operations_folder_name(og.client_namespace)}", + self.code_model.get_relative_import_path(serialize_namespace, og.client_namespace, async_mode=async_mode, namespace_type=NamespaceType.OPERATION), og.class_name, ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index b0fa49a92a..7eb51f3fdb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -97,12 +97,12 @@ def __init__( # | serialize_namespace | imported_namespace | relative_import_path | # |----------------------|----------------------|----------------------| - # |azure.test.operations | azure.test.operations| .. | + # |azure.test.operations | azure.test.operations| . | # |azure.test.operations | azure.test | .. | # |azure.test.operations | azure.test.subtest | ..subtest | # |azure.test.operations | azure | ... | # |azure.test.aio.operations | azure.test | ... | - # |azure.test.subtest.aio.operations|azure.test | .... | + # |azure.test.subtest.aio.operations|azure.test | .... | # |azure.test |azure.test.subtest | .subtest | def get_relative_import_path( self, @@ -148,7 +148,7 @@ def need_unique_model_alias(self) -> bool: def get_unique_models_alias(self, serialize_namespace: str, imported_namespace: str) -> str: if not self.need_unique_model_alias: return "_models" - relative_path = self.get_relative_import_path(serialize_namespace, imported_namespace) + relative_path = self.get_relative_import_path(serialize_namespace, imported_namespace, namespace_type=NamespaceType.MODEL) dot_num = max(relative_path.count(".") - 1, 0) parts = [""] + [p for p in relative_path.split(".") if p] return "_".join(parts) + str(dot_num) diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 973c75dd75..6ce95b819d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -235,8 +235,7 @@ def imports(self, **kwargs: Any) -> FileImport: alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), ) file_import.merge(self.value_type.imports(**kwargs)) - relative_path = kwargs.pop("relative_path", None) - if self.code_model.options["models_mode"] and relative_path: + if self.code_model.options["models_mode"]: # add import for enums in operations file file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 718af0b2f6..e1a1b95cae 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -7,7 +7,7 @@ from collections import OrderedDict from typing import Any, Dict, List, Optional, TYPE_CHECKING, cast import sys -from .utils import add_to_pylint_disable +from .utils import add_to_pylint_disable, NamespaceType from .base import BaseType from .constant_type import ConstantType from .property import Property @@ -301,10 +301,9 @@ def type_description(self) -> str: def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) # add import for models in operations or _types file file_import.add_submodule_import( - relative_path, + self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), "models", ImportType.LOCAL, alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index 064b69f9c3..bcaabef48c 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -148,7 +148,7 @@ def imports(self, **kwargs) -> FileImport: file_import.merge(self.type.imports(**kwargs, model_typing=True)) if self.optional and self.client_default_value is None: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) - if self.code_model.options["models_mode"] == "dpg" and isinstance(self.type, ModelType): + if self.code_model.options["models_mode"] == "dpg": serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace, module_name="_model_base"), diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index b12eef276a..1edb3556fb 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -204,18 +204,17 @@ def _serialize_and_write_models_folder( self, env: Environment, namespace: str, models: List[ModelType], enums: List[EnumType] ) -> None: # Write the models folder - models_namespace = namespace + ".models" - models_path = self.exec_path(models_namespace) + models_path = self.exec_path(namespace + ".models") serializer = DpgModelSerializer if self.code_model.options["models_mode"] == "dpg" else MsrestModelSerializer if models: self.write_file( models_path / Path(f"{self.code_model.models_filename}.py"), - serializer(code_model=self.code_model, env=env, client_namespace=models_namespace).serialize(), + serializer(code_model=self.code_model, env=env, client_namespace=namespace).serialize(), ) if enums: self.write_file( models_path / Path(f"{self.code_model.enums_filename}.py"), - EnumSerializer(code_model=self.code_model, env=env, client_namespace=models_namespace).serialize(), + EnumSerializer(code_model=self.code_model, env=env, client_namespace=namespace).serialize(), ) self.write_file( models_path / Path("__init__.py"), @@ -338,7 +337,7 @@ def _serialize_client_and_config_files( ) -> None: exec_path = self.exec_path(namespace) for async_mode, async_path in self.serialize_loop: - general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=async_mode) + general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=async_mode, client_namespace=namespace) # when there is client.py, there must be __init__.py self.write_file( exec_path / Path(f"{async_path}__init__.py"), @@ -364,58 +363,57 @@ def _serialize_client_and_config_files( def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str) -> None: - general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) - + exec_path = self.exec_path(namespace) # write _vendor.py - if self.code_model.need_vendored_code(async_mode=False): - self.write_file( - self.exec_path(namespace) / Path("_vendor.py"), - general_serializer.serialize_vendor_file(self.code_model.clients), - ) - if self.code_model.need_vendored_code(async_mode=True): - self.write_file( - self.exec_path(namespace) / Path("aio/_vendor.py"), - general_serializer.serialize_vendor_file(self.code_model.clients), - ) + for async_mode, async_path in self.serialize_loop: + if self.code_model.need_vendored_code(async_mode=async_mode): + self.write_file( + exec_path / Path(f"{async_path}_vendor.py"), + GeneralSerializer(code_model=self.code_model, env=env, async_mode=async_mode).serialize_vendor_file( + self.code_model.clients + ) + ) + + general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) # write _version.py self._serialize_and_write_version_file(namespace, general_serializer) # write the empty py.typed file - self.write_file(self.exec_path(namespace) / Path("py.typed"), "# Marker file for PEP 561.") + self.write_file(exec_path / Path("py.typed"), "# Marker file for PEP 561.") # write _serialization.py if not self.code_model.options["client_side_validation"] and not self.code_model.options["multiapi"]: self.write_file( - self.exec_path(namespace) / Path("_serialization.py"), + exec_path / Path("_serialization.py"), general_serializer.serialize_serialization_file(), ) # write _model_base.py if self.code_model.options["models_mode"] == "dpg": self.write_file( - self.exec_path(namespace) / Path("_model_base.py"), + exec_path / Path("_model_base.py"), general_serializer.serialize_model_base_file(), ) # write _validation.py if any(og for client in self.code_model.clients for og in client.operation_groups if og.need_validation): self.write_file( - self.exec_path(namespace) / Path("_validation.py"), + exec_path / Path("_validation.py"), general_serializer.serialize_validation_file(), ) # write apiview_mapping_python.json if self.code_model.options.get("emit_cross_language_definition_file"): self.write_file( - self.exec_path(namespace) / Path("apiview_mapping_python.json"), + exec_path / Path("apiview_mapping_python.json"), general_serializer.serialize_cross_language_definition_file(), ) # write _types.py if self.code_model.named_unions: self.write_file( - self.exec_path(namespace) / Path("_types.py"), + exec_path / Path("_types.py"), TypesSerializer(code_model=self.code_model, env=env).serialize(), ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index d46a6f6211..83c34ea0fe 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -68,7 +68,7 @@ def serialize_service_client_file(self, clients: List[Client]) -> str: imports = FileImport(self.code_model) for client in clients: - imports.merge(client.imports(self.async_mode, serliaze_namespace=self.serialize_namespace)) + imports.merge(client.imports(self.async_mode, serialize_namespace=self.serialize_namespace)) return template.render( code_model=self.code_model, From ea246a9e03c37b4e442dc6b369a5c9596930c255 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 4 Dec 2024 18:26:18 +0800 Subject: [PATCH 16/49] review --- .../pygen/codegen/models/model_type.py | 29 ++++++++++--------- .../pygen/codegen/models/operation_group.py | 9 +++--- .../pygen/codegen/models/primitive_types.py | 2 +- .../pygen/codegen/models/property.py | 3 +- .../pygen/codegen/serializers/__init__.py | 11 +++---- .../codegen/serializers/test_serializer.py | 4 +-- 6 files changed, 30 insertions(+), 28 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index e1a1b95cae..b9c998bb7b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -302,20 +302,21 @@ def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) # add import for models in operations or _types file - file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), - "models", - ImportType.LOCAL, - alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), - ) - if self.is_form_data: - file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace), - "_model_base", - ImportType.LOCAL, - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), - ) + if kwargs.get("need_import_models", False): + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), + "models", + ImportType.LOCAL, + alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), + typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), + ) + if self.is_form_data: + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace), + "_model_base", + ImportType.LOCAL, + typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), + ) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 5eccdb922b..0117867247 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -102,9 +102,8 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - relative_path = self.code_model.get_relative_import_path(serialize_namespace) for operation in self.operations: - file_import.merge(operation.imports(async_mode, **kwargs)) + file_import.merge(operation.imports(async_mode, need_import_models=True, **kwargs)) if not self.code_model.options["combine_operation_files"]: for og in self.operation_groups: file_import.add_submodule_import( @@ -123,13 +122,13 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: and self.code_model.options["models_mode"] == "msrest" and not self.is_mixin ): - file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") + file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace), "models", ImportType.LOCAL, alias="_models") if self.is_mixin: file_import.add_submodule_import( - f"{relative_path}_vendor", f"{self.client.name}MixinABC", ImportType.LOCAL + self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), f"{self.client.name}MixinABC", ImportType.LOCAL ) if self.has_abstract_operations: - file_import.add_submodule_import(f"{relative_path}_vendor", "raise_if_not_implemented", ImportType.LOCAL) + file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), "raise_if_not_implemented", ImportType.LOCAL) if all(o.abstract for o in self.operations): return file_import file_import.add_submodule_import("typing", "TypeVar", ImportType.STDLIB, TypingSection.CONDITIONAL) diff --git a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py index ca5dbddd70..fbd9b9317e 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py +++ b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py @@ -621,7 +621,7 @@ def type_annotation(self, **kwargs: Any) -> str: return self.name def docstring_type(self, **kwargs: Any) -> str: - return f"~{self.code_model.namespace}_vendor.{self.name}" + return f"~{self.code_model.namespace}._vendor.{self.name}" def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index bcaabef48c..126fb61e35 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -145,7 +145,8 @@ def imports(self, **kwargs) -> FileImport: file_import = FileImport(self.code_model) if self.is_discriminator and isinstance(self.type, EnumType): return file_import - file_import.merge(self.type.imports(**kwargs, model_typing=True)) + need_import_models = isinstance(self.type, ModelType) + file_import.merge(self.type.imports(**kwargs, model_typing=True, need_import_models=need_import_models)) if self.optional and self.client_default_value is None: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) if self.code_model.options["models_mode"] == "dpg": diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 1edb3556fb..30c4ee6bea 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -116,11 +116,12 @@ def serialize(self) -> None: self._serialize_and_write_package_files(client_namespace) # add generated samples and generated tests - if self.code_model.options["show_operations"] and self.code_model.has_operations: - if self.code_model.options["generate_sample"]: - self._serialize_and_write_sample(env, namespace=client_namespace) - if self.code_model.options["generate_test"]: - self._serialize_and_write_test(env, namespace=client_namespace) + # TODO + # if self.code_model.options["show_operations"] and self.code_model.has_operations: + # if self.code_model.options["generate_sample"]: + # self._serialize_and_write_sample(env, namespace=client_namespace) + # if self.code_model.options["generate_test"]: + # self._serialize_and_write_test(env, namespace=client_namespace) elif client_namespace_type.clients: # add clients folder if there are clients in this namespace self._serialize_client_and_config_files( diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index 35b43fe8aa..fd75a3fb93 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -143,9 +143,9 @@ def __init__( env: Environment, *, is_async: bool = False, - serialize_namespace: Optional[str] = None, + client_namespace: Optional[str] = None, ) -> None: - super().__init__(code_model, env, is_async, serialize_namespace=serialize_namespace) + super().__init__(code_model, env, is_async, client_namespace=client_namespace) self.is_async = is_async @property From 08a68ff675784337d7cb10d8812cc36ae8515176 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 5 Dec 2024 16:16:31 +0800 Subject: [PATCH 17/49] review --- .../generator/pygen/codegen/models/client.py | 4 +- .../pygen/codegen/models/code_model.py | 6 +-- .../pygen/codegen/models/enum_type.py | 20 +++------ .../pygen/codegen/models/model_type.py | 42 ++++++++++++------- .../pygen/codegen/models/operation.py | 2 +- .../pygen/codegen/models/operation_group.py | 6 +-- .../pygen/codegen/models/primitive_types.py | 3 -- .../pygen/codegen/models/property.py | 5 +-- .../generator/pygen/codegen/models/utils.py | 1 + .../pygen/codegen/serializers/__init__.py | 6 +-- .../codegen/serializers/client_serializer.py | 2 +- .../codegen/serializers/enum_serializer.py | 20 +++++++-- .../codegen/serializers/general_serializer.py | 7 ++-- .../serializers/model_init_serializer.py | 11 +++-- .../codegen/serializers/model_serializer.py | 15 ++++--- .../operation_groups_serializer.py | 1 + .../codegen/serializers/types_serializer.py | 2 +- .../templates/enum_container.py.jinja2 | 2 +- .../templates/model_container.py.jinja2 | 2 +- .../templates/operation_group.py.jinja2 | 2 +- 20 files changed, 90 insertions(+), 69 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index 2617e25c45..b611283d99 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -15,6 +15,8 @@ OverloadedRequestBuilder, get_request_builder, ) +from .model_type import ModelType +from .enum_type import EnumType from .parameter import Parameter, ParameterMethodLocation from .lro_operation import LROOperation from .lro_paging_operation import LROPagingOperation @@ -408,7 +410,6 @@ def imports(self, async_mode: bool, **kwargs) -> FileImport: file_import.merge( gp.imports( async_mode=async_mode, - in_operation_file=True, **kwargs, ) ) @@ -426,7 +427,6 @@ def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import.merge( gp.imports_for_multiapi( async_mode=async_mode, - in_operation_file=True, **kwargs, ) ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 7eb51f3fdb..85257798cb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -291,9 +291,9 @@ def model_types(self) -> List[ModelType]: def model_types(self, val: List[ModelType]) -> None: self._model_types = val - @property - def public_model_types(self) -> List[ModelType]: - return [m for m in self.model_types if not m.internal and not m.base == "json"] + def public_model_types(self, models: Optional[ModelType] = None) -> List[ModelType]: + models = models or self.model_types + return [m for m in models if not m.internal and not m.base == "json"] @property def enums(self) -> List[EnumType]: diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 6ce95b819d..fe3be54773 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -7,6 +7,7 @@ from .base import BaseType from .imports import FileImport, ImportType, TypingSection +from .utils import NamespaceType if TYPE_CHECKING: @@ -81,7 +82,6 @@ def imports(self, **kwargs: Any) -> FileImport: self.enum_type.name, ImportType.LOCAL, TypingSection.REGULAR, - client_namespace=self.enum_type.client_namespace, ) return file_import @@ -221,27 +221,17 @@ def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "EnumT ) def imports(self, **kwargs: Any) -> FileImport: - in_operation_file = kwargs.get("in_operation_file", False) file_import = FileImport(self.code_model) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - if self.code_model.options["models_mode"]: + if self.code_model.options["models_mode"] and kwargs.get("namespace_type") in [NamespaceType.OPERATION, NamespaceType.CLIENT]: file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) - if not in_operation_file: - file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), - "models", - ImportType.LOCAL, - TypingSection.TYPING, - alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), - ) - file_import.merge(self.value_type.imports(**kwargs)) - if self.code_model.options["models_mode"]: - # add import for enums in operations file file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), "models", ImportType.LOCAL, alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), + typing_section=TypingSection.REGULAR, ) + + file_import.merge(self.value_type.imports(**kwargs)) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index b9c998bb7b..6dd06f5386 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -11,6 +11,7 @@ from .base import BaseType from .constant_type import ConstantType from .property import Property +from .enum_type import EnumType from .imports import FileImport, ImportType, TypingSection from ...utils import NAME_LENGTH_LIMIT @@ -301,22 +302,33 @@ def type_description(self) -> str: def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) + alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) + namespace_type = kwargs.get("namespace_type") # add import for models in operations or _types file - if kwargs.get("need_import_models", False): - file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), - "models", - ImportType.LOCAL, - alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), - ) - if self.is_form_data: - file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace), - "_model_base", - ImportType.LOCAL, - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), - ) + if namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias=alias, + typing_section=TypingSection.REGULAR, + ) + if self.is_form_data: + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace), + "_model_base", + ImportType.LOCAL, + typing_section=TypingSection.REGULAR, + ) + elif namespace_type == NamespaceType.MODEL: + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias=alias, + typing_section=TypingSection.TYPING, + ) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index c0690614cb..d7248c52ae 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -341,7 +341,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements if self.parameters.has_body and self.parameters.body_parameter.flattened: file_import.merge( - self.parameters.body_parameter.type.imports(need_import_iobase=self.need_import_iobase, **kwargs) + self.parameters.body_parameter.type.imports(need_import_iobase=self.need_import_iobase,**kwargs) ) if not async_mode: for param in self.parameters.headers: diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 0117867247..4636f4ab9b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -103,7 +103,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) for operation in self.operations: - file_import.merge(operation.imports(async_mode, need_import_models=True, **kwargs)) + file_import.merge(operation.imports(async_mode, **kwargs)) if not self.code_model.options["combine_operation_files"]: for og in self.operation_groups: file_import.add_submodule_import( @@ -118,14 +118,14 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ) # for multiapi if ( - (self.code_model.public_model_types) + (self.code_model.public_model_types()) and self.code_model.options["models_mode"] == "msrest" and not self.is_mixin ): file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace), "models", ImportType.LOCAL, alias="_models") if self.is_mixin: file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), f"{self.client.name}MixinABC", ImportType.LOCAL + self.code_model.get_relative_import_path(serialize_namespace, self.code_model.namespace, module_name="_vendor", async_mode=async_mode), f"{self.client.name}MixinABC", ImportType.LOCAL ) if self.has_abstract_operations: file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), "raise_if_not_implemented", ImportType.LOCAL) diff --git a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py index fbd9b9317e..320094fd80 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py +++ b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py @@ -82,9 +82,6 @@ def default_template_representation_declaration(self) -> str: return self.get_declaration(b"bytes") def imports(self, **kwargs: Any) -> FileImport: - from .combined_type import CombinedType - from .operation import OperationBase - file_import = FileImport(self.code_model) file_import.add_submodule_import("typing", "IO", ImportType.STDLIB) if kwargs.get("need_import_iobase", False): diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index 126fb61e35..78c8902668 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -140,13 +140,10 @@ def validation(self) -> Optional[Dict[str, Any]]: return retval or None def imports(self, **kwargs) -> FileImport: - from .model_type import ModelType - file_import = FileImport(self.code_model) if self.is_discriminator and isinstance(self.type, EnumType): return file_import - need_import_models = isinstance(self.type, ModelType) - file_import.merge(self.type.imports(**kwargs, model_typing=True, need_import_models=need_import_models)) + file_import.merge(self.type.imports(**kwargs)) if self.optional and self.client_default_value is None: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) if self.code_model.options["models_mode"] == "dpg": diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index 082bfa1296..213b9abbb7 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -29,4 +29,5 @@ class NamespaceType(str, Enum): NONE = "none" MODEL = "model" OPERATION = "operation" + CLIENT = "client" diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 30c4ee6bea..c257319384 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -210,16 +210,16 @@ def _serialize_and_write_models_folder( if models: self.write_file( models_path / Path(f"{self.code_model.models_filename}.py"), - serializer(code_model=self.code_model, env=env, client_namespace=namespace).serialize(), + serializer(code_model=self.code_model, env=env, client_namespace=namespace, models=models).serialize(), ) if enums: self.write_file( models_path / Path(f"{self.code_model.enums_filename}.py"), - EnumSerializer(code_model=self.code_model, env=env, client_namespace=namespace).serialize(), + EnumSerializer(code_model=self.code_model, env=env, client_namespace=namespace, enums=enums).serialize(), ) self.write_file( models_path / Path("__init__.py"), - ModelInitSerializer(code_model=self.code_model, env=env).serialize(), + ModelInitSerializer(code_model=self.code_model, env=env, models=models, enums=enums).serialize(), ) self._keep_patch_file(models_path / Path("_patch.py"), env) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py index 22540a1a85..f7f7232935 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py @@ -136,7 +136,7 @@ def _get_client_models_value(models_dict_name: str) -> str: is_msrest_model = self.client.code_model.options["models_mode"] == "msrest" if is_msrest_model: add_private_models = len(self.client.code_model.model_types) != len( - self.client.code_model.public_model_types + self.client.code_model.public_model_types() ) model_dict_name = f"_models.{self.client.code_model.models_filename}" if add_private_models else "_models" retval.append( diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/enum_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/enum_serializer.py index 4b9ce87e3f..8207b29db3 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/enum_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/enum_serializer.py @@ -3,13 +3,27 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- - +from typing import Optional, List +from jinja2 import Environment from .base_serializer import BaseSerializer -from ..models import FileImport +from ..models import FileImport, CodeModel, EnumType class EnumSerializer(BaseSerializer): + + def __init__( + self, + code_model: CodeModel, + env: Environment, + async_mode: bool = False, + *, + enums: List[EnumType], + client_namespace: Optional[str] = None + ): + super().__init__(code_model, env, async_mode=async_mode, client_namespace=client_namespace) + self.enums = enums + def serialize(self) -> str: # Generate the enum file template = self.env.get_template("enum_container.py.jinja2") - return template.render(code_model=self.code_model, file_import=FileImport(self.code_model)) + return template.render(code_model=self.code_model, file_import=FileImport(self.code_model), enums=self.enums) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index 83c34ea0fe..5e5bcedb11 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -14,6 +14,7 @@ TokenCredentialType, Client, ) +from ..models.utils import NamespaceType from .client_serializer import ClientSerializer, ConfigSerializer from .base_serializer import BaseSerializer @@ -68,7 +69,7 @@ def serialize_service_client_file(self, clients: List[Client]) -> str: imports = FileImport(self.code_model) for client in clients: - imports.merge(client.imports(self.async_mode, serialize_namespace=self.serialize_namespace)) + imports.merge(client.imports(self.async_mode, serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.CLIENT)) return template.render( code_model=self.code_model, @@ -149,7 +150,7 @@ def serialize_config_file(self, clients: List[Client]) -> str: template = self.env.get_template("config_container.py.jinja2") imports = FileImport(self.code_model) for client in self.code_model.clients: - imports.merge(client.config.imports(self.async_mode, serialize_namespace=self.serialize_namespace)) + imports.merge(client.config.imports(self.async_mode, serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.CLIENT)) return template.render( code_model=self.code_model, async_mode=self.async_mode, @@ -180,7 +181,7 @@ def serialize_validation_file(self) -> str: def serialize_cross_language_definition_file(self) -> str: cross_langauge_def_dict = { f"{self.code_model.namespace}.models.{model.name}": model.cross_language_definition_id - for model in self.code_model.public_model_types + for model in self.code_model.public_model_types() } cross_langauge_def_dict.update( { diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py index 5df688adbb..6a346e1f06 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py @@ -3,19 +3,22 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +from typing import List from jinja2 import Environment -from ..models import CodeModel +from ..models import CodeModel, ModelType, EnumType class ModelInitSerializer: - def __init__(self, code_model: CodeModel, env: Environment) -> None: + def __init__(self, code_model: CodeModel, env: Environment, *, models: List[ModelType], enums: List[EnumType]) -> None: self.code_model = code_model self.env = env + self.models = models + self.enums = enums def serialize(self) -> str: - schemas = [s.name for s in self.code_model.public_model_types] + schemas = [s.name for s in self.code_model.public_model_types(self.models)] schemas.sort() - enums = [e.name for e in self.code_model.enums if not e.internal] if self.code_model.enums else None + enums = [e.name for e in self.enums if not e.internal] if self.enums else None if enums: enums.sort() diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 9d271c19b9..269d1df424 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import List +from typing import List, Optional from abc import ABC, abstractmethod from ..models import ModelType, Property, ConstantType, EnumValue @@ -23,6 +23,10 @@ def _documentation_string(prop: Property, description_keyword: str, docstring_ty class _ModelSerializer(BaseSerializer, ABC): + def __init__(self, code_model, env, async_mode = False, *, models: List[ModelType], client_namespace: Optional[str] = None): + super().__init__(code_model, env, async_mode, client_namespace=client_namespace) + self.models = models + @abstractmethod def imports(self) -> FileImport: ... @@ -34,6 +38,7 @@ def serialize(self) -> str: imports=FileImportSerializer(self.imports()), str=str, serializer=self, + models=self.models, ) @abstractmethod @@ -141,7 +146,7 @@ def imports(self) -> FileImport: msrest_import_type=MsrestImportType.Module, typing_section=TypingSection.REGULAR, ) - for model in self.code_model.model_types: + for model in self.models: file_import.merge(model.imports(is_operation_file=False)) for param in self._init_line_parameters(model): file_import.merge(param.imports()) @@ -221,12 +226,12 @@ def imports(self) -> FileImport: TypingSection.REGULAR, ) - for model in self.code_model.model_types: + for model in self.models: if model.base == "json": continue file_import.merge(model.imports(is_operation_file=False, serialize_namespace=self.serialize_namespace)) for prop in model.properties: - file_import.merge(prop.imports(serialize_namespace=self.serialize_namespace)) + file_import.merge(prop.imports(serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.MODEL)) if model.is_polymorphic: file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) if not model.internal and self.init_line(model): @@ -329,7 +334,7 @@ def properties_to_pass_to_super(model: ModelType) -> str: def global_pylint_disables(self) -> str: result = [] - for model in self.code_model.model_types: + for model in self.models: if self.need_init(model): for item in self.pylint_disable_items(model): if item: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py index 9fcbe41f61..78ef76c7cf 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -63,6 +63,7 @@ def serialize(self) -> str: operation_group.imports( async_mode=self.async_mode, serialize_namespace=self.serialize_namespace, + namespace_type=NamespaceType.OPERATION, ) ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index f628e8f023..b0575d0f53 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -19,7 +19,7 @@ def imports(self) -> FileImport: ) for nu in self.code_model.named_unions: file_import.merge( - nu.imports(model_typing=True, is_types_file=True, serialize_namespace=self.serialize_namespace) + nu.imports(is_types_file=True, serialize_namespace=self.serialize_namespace) ) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/templates/enum_container.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/enum_container.py.jinja2 index 099fbb072a..55439c7ed5 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/enum_container.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/enum_container.py.jinja2 @@ -5,6 +5,6 @@ from enum import Enum from {{ code_model.core_library }}{{ "" if code_model.is_azure_flavor else ".utils" }} import CaseInsensitiveEnumMeta -{% for enum in code_model.enums | sort %} +{% for enum in enums | sort %} {% include "enum.py.jinja2" %} {% endfor %} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 index b2ec433efc..fb0337e360 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 @@ -6,7 +6,7 @@ {% endif %} {{ imports }} -{% for model in code_model.model_types %} +{% for model in models %} {% if model.base == "dpg" %} {% include "model_dpg.py.jinja2" %} {% elif model.base == "msrest" %} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/operation_group.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/operation_group.py.jinja2 index 1ee2846535..a894b95e75 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/operation_group.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/operation_group.py.jinja2 @@ -25,7 +25,7 @@ class {{ operation_group.class_name }}: {{ operation_group.pylint_disable() }} :attr:`{{ operation_group.property_name }}` attribute. """ -{% if code_model.public_model_types and code_model.options["models_mode"] == "msrest" %} +{% if code_model.public_model_types() and code_model.options["models_mode"] == "msrest" %} models = _models {% endif %} From a8dc1d4ba354e08fed4d74716bc78cb5ef22f85a Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 5 Dec 2024 16:47:05 +0800 Subject: [PATCH 18/49] review --- .../generator/pygen/codegen/models/enum_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index fe3be54773..4c8eacd524 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -78,7 +78,7 @@ def imports(self, **kwargs: Any) -> FileImport: file_import.add_submodule_import("typing", "Literal", ImportType.STDLIB, TypingSection.REGULAR) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, self.enum_type.client_namespace), + self.code_model.get_relative_import_path(serialize_namespace, self.enum_type.client_namespace, namespace_type=NamespaceType.MODEL, module_name=self.code_model.enums_filename), self.enum_type.name, ImportType.LOCAL, TypingSection.REGULAR, From 089ebcc0a5743d5aa17890b0323361bb5f7d0364 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 5 Dec 2024 17:27:13 +0800 Subject: [PATCH 19/49] review --- .../emitter/src/external-process.ts | 5 ++- .../pygen/codegen/models/enum_type.py | 32 +++++++++++++------ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/packages/http-client-python/emitter/src/external-process.ts b/packages/http-client-python/emitter/src/external-process.ts index 23ddfe2eed..22905d8d24 100644 --- a/packages/http-client-python/emitter/src/external-process.ts +++ b/packages/http-client-python/emitter/src/external-process.ts @@ -22,7 +22,10 @@ export async function saveCodeModelAsYaml(name: string, codemodel: unknown): Pro const filename = createTempPath(".yaml", name); const yamlStr = jsyaml.dump(codemodel); await writeFile(filename, yamlStr); - await writeFile(joinPaths("C:/dev/typespec/packages/http-client-python", "alpha", "output.yaml"), yamlStr); + await writeFile( + joinPaths("C:/dev/typespec/packages/http-client-python", "alpha", "output.yaml"), + yamlStr, + ); return filename; } diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 4c8eacd524..b54e0e6897 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -222,16 +222,30 @@ def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "EnumT def imports(self, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) - serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - if self.code_model.options["models_mode"] and kwargs.get("namespace_type") in [NamespaceType.OPERATION, NamespaceType.CLIENT]: + file_import.merge(self.value_type.imports(**kwargs)) + if self.code_model.options["models_mode"]: file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) - file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace), - "models", - ImportType.LOCAL, - alias=self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace), - typing_section=TypingSection.REGULAR, - ) + + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) + alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) + namespace_type = kwargs.get("namespace_type") + if namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias=alias, + typing_section=TypingSection.REGULAR, + ) + elif namespace_type == NamespaceType.MODEL: + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias=alias, + typing_section=TypingSection.TYPING, + ) file_import.merge(self.value_type.imports(**kwargs)) return file_import From 59d97c2afd96bb0551a4683c3ee07c86c7287ca0 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 11:27:26 +0800 Subject: [PATCH 20/49] review --- .../http-client-python/emitter/src/external-process.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/emitter/src/external-process.ts b/packages/http-client-python/emitter/src/external-process.ts index 22905d8d24..eb2471285a 100644 --- a/packages/http-client-python/emitter/src/external-process.ts +++ b/packages/http-client-python/emitter/src/external-process.ts @@ -22,10 +22,10 @@ export async function saveCodeModelAsYaml(name: string, codemodel: unknown): Pro const filename = createTempPath(".yaml", name); const yamlStr = jsyaml.dump(codemodel); await writeFile(filename, yamlStr); - await writeFile( - joinPaths("C:/dev/typespec/packages/http-client-python", "alpha", "output.yaml"), - yamlStr, - ); + // await writeFile( + // joinPaths("C:/dev/typespec/packages/http-client-python", "alpha", "output.yaml"), + // yamlStr, + // ); return filename; } From 9c5f209843e3b59b7f44f0e2c1382beac8a905da Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 11:39:44 +0800 Subject: [PATCH 21/49] review --- packages/http-client-python/package-lock.json | 8 ++++---- packages/http-client-python/package.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/http-client-python/package-lock.json b/packages/http-client-python/package-lock.json index d815e4c4bd..3e087f7ce9 100644 --- a/packages/http-client-python/package-lock.json +++ b/packages/http-client-python/package-lock.json @@ -1,12 +1,12 @@ { "name": "@typespec/http-client-python", - "version": "0.3.11", + "version": "0.3.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@typespec/http-client-python", - "version": "0.3.11", + "version": "0.3.12", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -21,7 +21,7 @@ "@azure-tools/typespec-azure-core": "~0.48.0", "@azure-tools/typespec-azure-resource-manager": "~0.48.0", "@azure-tools/typespec-azure-rulesets": "~0.48.0", - "@azure-tools/typespec-client-generator-core": "~0.48.1", + "@azure-tools/typespec-client-generator-core": "~0.48.0", "@types/js-yaml": "~4.0.5", "@types/node": "~22.5.4", "@types/semver": "7.5.8", @@ -45,7 +45,7 @@ "@azure-tools/typespec-azure-core": ">=0.48.0 <1.0.0", "@azure-tools/typespec-azure-resource-manager": ">=0.48.0 <1.0.0", "@azure-tools/typespec-azure-rulesets": ">=0.48.0 <3.0.0", - "@azure-tools/typespec-client-generator-core": ">=0.48.1 <1.0.0", + "@azure-tools/typespec-client-generator-core": ">=0.48.0 <1.0.0", "@typespec/compiler": ">=0.62.0 <1.0.0", "@typespec/http": ">=0.62.0 <1.0.0", "@typespec/openapi": ">=0.62.0 <1.0.0", diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 083042eb75..34627e8a8d 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -60,7 +60,7 @@ "@azure-tools/typespec-azure-resource-manager": ">=0.48.0 <1.0.0", "@azure-tools/typespec-autorest": ">=0.48.0 <1.0.0", "@azure-tools/typespec-azure-rulesets": ">=0.48.0 <3.0.0", - "@azure-tools/typespec-client-generator-core": ">=0.48.1 <1.0.0" + "@azure-tools/typespec-client-generator-core": ">=0.48.0 <1.0.0" }, "dependencies": { "js-yaml": "~4.1.0", @@ -77,7 +77,7 @@ "@azure-tools/typespec-azure-core": "~0.48.0", "@azure-tools/typespec-azure-rulesets": "~0.48.0", "@azure-tools/typespec-azure-resource-manager": "~0.48.0", - "@azure-tools/typespec-client-generator-core": "~0.48.1", + "@azure-tools/typespec-client-generator-core": "~0.48.0", "@azure-tools/cadl-ranch-specs": "~0.39.1", "@azure-tools/cadl-ranch-expect": "~0.15.6", "@types/js-yaml": "~4.0.5", From d8881d17c0f65c3c2c6e26651091538a68b275ee Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 13:14:33 +0800 Subject: [PATCH 22/49] review --- packages/http-client-python/emitter/src/utils.ts | 7 ++++--- .../generator/pygen/codegen/serializers/__init__.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/http-client-python/emitter/src/utils.ts b/packages/http-client-python/emitter/src/utils.ts index f8c54ed9b5..c6920746fb 100644 --- a/packages/http-client-python/emitter/src/utils.ts +++ b/packages/http-client-python/emitter/src/utils.ts @@ -204,16 +204,17 @@ export function getClientNamespace, clientNamespace: string, ) { + const rootNamespace = removeUnderscoresFromNamespace(context.sdkPackage.rootNamespace).toLowerCase(); const options = context.emitContext.options; if ([undefined, false].includes(options["enable-typespec-namespace"])) { - return context.sdkPackage.rootNamespace.toLowerCase(); + return rootNamespace; } if ( ["azure.core", "azure.resourcemanager"].some((item) => clientNamespace.toLowerCase().startsWith(item), ) ) { - return context.sdkPackage.rootNamespace.toLowerCase(); + return rootNamespace; } - return clientNamespace.toLowerCase(); + return removeUnderscoresFromNamespace(clientNamespace).toLowerCase(); } diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index c257319384..06bb35287f 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -310,12 +310,13 @@ def _serialize_and_write_version_file( namespace: str, general_serializer: GeneralSerializer, ): + exec_path = self.exec_path(namespace) def _read_version_file(original_version_file_name: str) -> str: - return self.read_file(self.exec_path(namespace) / original_version_file_name) + return self.read_file(exec_path / original_version_file_name) def _write_version_file(original_version_file_name: str) -> None: self.write_file( - self.exec_path(namespace) / Path("_version.py"), + exec_path / Path("_version.py"), _read_version_file(original_version_file_name), ) @@ -326,7 +327,7 @@ def _write_version_file(original_version_file_name: str) -> None: _write_version_file(original_version_file_name="version.py") elif self.code_model.options["package_version"]: self.write_file( - self.exec_path(namespace) / Path("_version.py"), + exec_path / Path("_version.py"), general_serializer.serialize_version_file(), ) From 5aeb9cf7faa3e6ef36045f338ad64f57950151e4 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 13:48:12 +0800 Subject: [PATCH 23/49] review --- .../http-client-python/emitter/src/utils.ts | 4 +- .../generator/pygen/codegen/_utils.py | 3 -- .../generator/pygen/codegen/models/client.py | 11 +++++- .../pygen/codegen/models/code_model.py | 31 +++++++++------- .../pygen/codegen/models/enum_type.py | 37 +++++++++++-------- .../generator/pygen/codegen/models/imports.py | 17 +++++---- .../pygen/codegen/models/lro_operation.py | 2 +- .../pygen/codegen/models/operation.py | 13 +++++-- .../pygen/codegen/models/operation_group.py | 19 ++++++++-- .../pygen/codegen/models/primitive_types.py | 6 ++- .../generator/pygen/codegen/models/utils.py | 1 - .../pygen/codegen/serializers/__init__.py | 29 ++++++++------- .../codegen/serializers/enum_serializer.py | 2 +- .../codegen/serializers/general_serializer.py | 12 +++++- .../serializers/model_init_serializer.py | 4 +- .../codegen/serializers/model_serializer.py | 8 +++- .../serializers/parameter_serializer.py | 3 +- .../codegen/serializers/types_serializer.py | 4 +- .../pygen/codegen/serializers/utils.py | 2 - 19 files changed, 131 insertions(+), 77 deletions(-) diff --git a/packages/http-client-python/emitter/src/utils.ts b/packages/http-client-python/emitter/src/utils.ts index c6920746fb..d451a0163c 100644 --- a/packages/http-client-python/emitter/src/utils.ts +++ b/packages/http-client-python/emitter/src/utils.ts @@ -204,7 +204,9 @@ export function getClientNamespace, clientNamespace: string, ) { - const rootNamespace = removeUnderscoresFromNamespace(context.sdkPackage.rootNamespace).toLowerCase(); + const rootNamespace = removeUnderscoresFromNamespace( + context.sdkPackage.rootNamespace, + ).toLowerCase(); const options = context.emitContext.options; if ([undefined, false].includes(options["enable-typespec-namespace"])) { return rootNamespace; diff --git a/packages/http-client-python/generator/pygen/codegen/_utils.py b/packages/http-client-python/generator/pygen/codegen/_utils.py index b7133e3aa0..3611b55b60 100644 --- a/packages/http-client-python/generator/pygen/codegen/_utils.py +++ b/packages/http-client-python/generator/pygen/codegen/_utils.py @@ -20,6 +20,3 @@ def get_parent_namespace(namespace: str) -> str: return namespace.rsplit(".", 1)[0] - - - diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index b611283d99..c10e27825d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -307,7 +307,12 @@ def imports(self, async_mode: bool, **kwargs) -> FileImport: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) for og in self.operation_groups: file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, og.client_namespace, async_mode=async_mode, namespace_type=NamespaceType.OPERATION), + self.code_model.get_relative_import_path( + serialize_namespace, + og.client_namespace, + async_mode=async_mode, + namespace_type=NamespaceType.OPERATION, + ), og.class_name, ImportType.LOCAL, ) @@ -393,7 +398,9 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: if self.code_model.options["package_version"]: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, module_name="_version"), "VERSION", ImportType.LOCAL + self.code_model.get_relative_import_path(serialize_namespace, module_name="_version"), + "VERSION", + ImportType.LOCAL, ) if self.code_model.options["azure_arm"]: policy = "AsyncARMChallengeAuthenticationPolicy" if async_mode else "ARMChallengeAuthenticationPolicy" diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 85257798cb..b78203cfe7 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -18,6 +18,7 @@ def _is_legacy(options) -> bool: return not (options.get("version_tolerant") or options.get("low_level_client")) + def get_all_operation_groups_recursively(clients: List[Client]) -> List[OperationGroup]: operation_groups = [] queue = [] @@ -29,6 +30,7 @@ def get_all_operation_groups_recursively(clients: List[Client]) -> List[Operatio queue.extend(operation_groups[-1].operation_groups) return operation_groups + class ClientNamespaceType: def __init__( self, @@ -124,31 +126,34 @@ def get_relative_import_path( else: module_namespace = "" imported_namespace = imported_namespace + async_namespace + module_namespace - + key = f"{serialize_namespace}-{imported_namespace}" if key not in self._relative_import_path: - idx = 0 - serialize_namespace_split = serialize_namespace.split(".") - imported_namespace_split = imported_namespace.split(".") - while idx < min(len(serialize_namespace_split), len(imported_namespace_split)): - if serialize_namespace_split[idx] != imported_namespace_split[idx]: - break - idx += 1 - self._relative_import_path[key] = "." * (len(serialize_namespace_split[idx:]) + 1) + ".".join(imported_namespace_split[idx:]) + idx = 0 + serialize_namespace_split = serialize_namespace.split(".") + imported_namespace_split = imported_namespace.split(".") + while idx < min(len(serialize_namespace_split), len(imported_namespace_split)): + if serialize_namespace_split[idx] != imported_namespace_split[idx]: + break + idx += 1 + self._relative_import_path[key] = "." * (len(serialize_namespace_split[idx:]) + 1) + ".".join( + imported_namespace_split[idx:] + ) result = self._relative_import_path[key] if module_name is None: - return result + return result return f"{result}{module_name}" if result.endswith(".") else f"{result}.{module_name}" - @property def need_unique_model_alias(self) -> bool: return self.has_subnamespace and self.options["enable_typespec_namespace"] - + def get_unique_models_alias(self, serialize_namespace: str, imported_namespace: str) -> str: if not self.need_unique_model_alias: return "_models" - relative_path = self.get_relative_import_path(serialize_namespace, imported_namespace, namespace_type=NamespaceType.MODEL) + relative_path = self.get_relative_import_path( + serialize_namespace, imported_namespace, namespace_type=NamespaceType.MODEL + ) dot_num = max(relative_path.count(".") - 1, 0) parts = [""] + [p for p in relative_path.split(".") if p] return "_".join(parts) + str(dot_num) diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index b54e0e6897..826f7d51ed 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -78,7 +78,12 @@ def imports(self, **kwargs: Any) -> FileImport: file_import.add_submodule_import("typing", "Literal", ImportType.STDLIB, TypingSection.REGULAR) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, self.enum_type.client_namespace, namespace_type=NamespaceType.MODEL, module_name=self.code_model.enums_filename), + self.code_model.get_relative_import_path( + serialize_namespace, + self.enum_type.client_namespace, + namespace_type=NamespaceType.MODEL, + module_name=self.code_model.enums_filename, + ), self.enum_type.name, ImportType.LOCAL, TypingSection.REGULAR, @@ -225,27 +230,27 @@ def imports(self, **kwargs: Any) -> FileImport: file_import.merge(self.value_type.imports(**kwargs)) if self.code_model.options["models_mode"]: file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) - + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) namespace_type = kwargs.get("namespace_type") if namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: - file_import.add_submodule_import( - relative_path, - "models", - ImportType.LOCAL, - alias=alias, - typing_section=TypingSection.REGULAR, - ) + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias=alias, + typing_section=TypingSection.REGULAR, + ) elif namespace_type == NamespaceType.MODEL: - file_import.add_submodule_import( - relative_path, - "models", - ImportType.LOCAL, - alias=alias, - typing_section=TypingSection.TYPING, - ) + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias=alias, + typing_section=TypingSection.TYPING, + ) file_import.merge(self.value_type.imports(**kwargs)) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/imports.py b/packages/http-client-python/generator/pygen/codegen/models/imports.py index 6ae60a1f95..ed98052a7b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/imports.py +++ b/packages/http-client-python/generator/pygen/codegen/models/imports.py @@ -278,13 +278,16 @@ def add_msrest_import( # for multiapi, the namespace is azure.mgmt.xxx.v20XX_XX_XX while _serialization.py is in azure.mgmt.xxx imported_namespace = get_parent_namespace(imported_namespace) if msrest_import_type == MsrestImportType.Module: - self.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace, imported_namespace), "_serialization", ImportType.LOCAL, typing_section) - else: - relative_path = self.code_model.get_relative_import_path(serialize_namespace, imported_namespace, module_name="_serialization") self.add_submodule_import( - relative_path, "Serializer", ImportType.LOCAL, typing_section + self.code_model.get_relative_import_path(serialize_namespace, imported_namespace), + "_serialization", + ImportType.LOCAL, + typing_section, + ) + else: + relative_path = self.code_model.get_relative_import_path( + serialize_namespace, imported_namespace, module_name="_serialization" ) + self.add_submodule_import(relative_path, "Serializer", ImportType.LOCAL, typing_section) if msrest_import_type == MsrestImportType.SerializerDeserializer: - self.add_submodule_import( - relative_path, "Deserializer", ImportType.LOCAL, typing_section - ) + self.add_submodule_import(relative_path, "Deserializer", ImportType.LOCAL, typing_section) diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index d20a48c16c..0904c65cd9 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -134,7 +134,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: # but final call returns a model serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, module_name='_model_base'), + self.code_model.get_relative_import_path(serialize_namespace, module_name="_model_base"), "_deserialize", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index d7248c52ae..fb43e2d03a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -308,7 +308,12 @@ def get_request_builder_import( ) if self.code_model.options["builders_visibility"] == "embedded" and async_mode: file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace, namespace_type=NamespaceType.OPERATION, module_name=self.filename), + self.code_model.get_relative_import_path( + serialize_namespace, + self.client_namespace, + namespace_type=NamespaceType.OPERATION, + module_name=self.filename, + ), request_builder.name, import_type=ImportType.LOCAL, ) @@ -341,7 +346,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements if self.parameters.has_body and self.parameters.body_parameter.flattened: file_import.merge( - self.parameters.body_parameter.type.imports(need_import_iobase=self.need_import_iobase,**kwargs) + self.parameters.body_parameter.type.imports(need_import_iobase=self.need_import_iobase, **kwargs) ) if not async_mode: for param in self.parameters.headers: @@ -447,7 +452,9 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements relative_path = self.code_model.get_relative_import_path(serialize_namespace, module_name="_model_base") if self.parameters.has_body: if self.has_form_data_body: - file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace), "_model_base", ImportType.LOCAL) + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace), "_model_base", ImportType.LOCAL + ) elif xml_serializable(self.parameters.body_parameter.default_content_type): file_import.add_submodule_import( relative_path, diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 4636f4ab9b..4e5204cd04 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -122,13 +122,26 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: and self.code_model.options["models_mode"] == "msrest" and not self.is_mixin ): - file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace), "models", ImportType.LOCAL, alias="_models") + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace), + "models", + ImportType.LOCAL, + alias="_models", + ) if self.is_mixin: file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, self.code_model.namespace, module_name="_vendor", async_mode=async_mode), f"{self.client.name}MixinABC", ImportType.LOCAL + self.code_model.get_relative_import_path( + serialize_namespace, self.code_model.namespace, module_name="_vendor", async_mode=async_mode + ), + f"{self.client.name}MixinABC", + ImportType.LOCAL, ) if self.has_abstract_operations: - file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), "raise_if_not_implemented", ImportType.LOCAL) + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), + "raise_if_not_implemented", + ImportType.LOCAL, + ) if all(o.abstract for o in self.operations): return file_import file_import.add_submodule_import("typing", "TypeVar", ImportType.STDLIB, TypingSection.CONDITIONAL) diff --git a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py index 6d4bdafd83..08eb8de03d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py +++ b/packages/http-client-python/generator/pygen/codegen/models/primitive_types.py @@ -623,7 +623,11 @@ def docstring_type(self, **kwargs: Any) -> str: def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - file_import.add_submodule_import(self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), self.name, ImportType.LOCAL) + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), + self.name, + ImportType.LOCAL, + ) return file_import @property diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index 213b9abbb7..c7b44fbda9 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -30,4 +30,3 @@ class NamespaceType(str, Enum): MODEL = "model" OPERATION = "operation" CLIENT = "client" - diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 06bb35287f..98dc950312 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -57,6 +57,7 @@ _REGENERATE_FILES = {"setup.py", "MANIFEST.in"} AsyncInfo = namedtuple("AsyncInfo", ["async_mode", "async_path"]) + # extract sub folders. For example, source_file_path is like: # "xxx/resource-manager/Microsoft.XX/stable/2023-04-01/examples/Compute/createOrUpdate/AKSCompute.json", # and we want to extract the sub folders after "examples/", which is "compute/create_or_update" @@ -113,7 +114,8 @@ def serialize(self) -> None: ) # add packaging files in root namespace (e.g. setup.py, README.md, etc.) - self._serialize_and_write_package_files(client_namespace) + if self.code_model.options["package_mode"]: + self._serialize_and_write_package_files(client_namespace) # add generated samples and generated tests # TODO @@ -124,9 +126,7 @@ def serialize(self) -> None: # self._serialize_and_write_test(env, namespace=client_namespace) elif client_namespace_type.clients: # add clients folder if there are clients in this namespace - self._serialize_client_and_config_files( - client_namespace, client_namespace_type.clients, env - ) + self._serialize_client_and_config_files(client_namespace, client_namespace_type.clients, env) else: # add pkgutil init file if no clients in this namespace self.write_file( @@ -215,7 +215,9 @@ def _serialize_and_write_models_folder( if enums: self.write_file( models_path / Path(f"{self.code_model.enums_filename}.py"), - EnumSerializer(code_model=self.code_model, env=env, client_namespace=namespace, enums=enums).serialize(), + EnumSerializer( + code_model=self.code_model, env=env, client_namespace=namespace, enums=enums + ).serialize(), ) self.write_file( models_path / Path("__init__.py"), @@ -297,20 +299,20 @@ def _serialize_and_write_operations_folder( async_mode=async_mode, ) self.write_file( - exec_path / Path(f"{prefix_path}/{filename}.py"), - operation_group_serializer.serialize(), + exec_path / Path(f"{prefix_path}/{filename}.py"), + operation_group_serializer.serialize(), ) # if there was a patch file before, we keep it self._keep_patch_file(exec_path / Path(f"{prefix_path}/_patch.py"), env) - def _serialize_and_write_version_file( self, namespace: str, general_serializer: GeneralSerializer, ): exec_path = self.exec_path(namespace) + def _read_version_file(original_version_file_name: str) -> str: return self.read_file(exec_path / original_version_file_name) @@ -339,7 +341,9 @@ def _serialize_client_and_config_files( ) -> None: exec_path = self.exec_path(namespace) for async_mode, async_path in self.serialize_loop: - general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=async_mode, client_namespace=namespace) + general_serializer = GeneralSerializer( + code_model=self.code_model, env=env, async_mode=async_mode, client_namespace=namespace + ) # when there is client.py, there must be __init__.py self.write_file( exec_path / Path(f"{async_path}__init__.py"), @@ -352,7 +356,7 @@ def _serialize_client_and_config_files( general_serializer.serialize_service_client_file(clients), ) - #write config file + # write config file self.write_file( exec_path / Path(f"{async_path}_configuration.py"), general_serializer.serialize_config_file(clients), @@ -361,9 +365,6 @@ def _serialize_client_and_config_files( # if there was a patch file before, we keep it self._keep_patch_file(exec_path / Path(f"{async_path}_patch.py"), env) - - - def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str) -> None: exec_path = self.exec_path(namespace) # write _vendor.py @@ -373,7 +374,7 @@ def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str exec_path / Path(f"{async_path}_vendor.py"), GeneralSerializer(code_model=self.code_model, env=env, async_mode=async_mode).serialize_vendor_file( self.code_model.clients - ) + ), ) general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/enum_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/enum_serializer.py index 8207b29db3..7b2c719136 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/enum_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/enum_serializer.py @@ -10,7 +10,7 @@ class EnumSerializer(BaseSerializer): - + def __init__( self, code_model: CodeModel, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index 5e5bcedb11..9796178d46 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -69,7 +69,11 @@ def serialize_service_client_file(self, clients: List[Client]) -> str: imports = FileImport(self.code_model) for client in clients: - imports.merge(client.imports(self.async_mode, serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.CLIENT)) + imports.merge( + client.imports( + self.async_mode, serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.CLIENT + ) + ) return template.render( code_model=self.code_model, @@ -150,7 +154,11 @@ def serialize_config_file(self, clients: List[Client]) -> str: template = self.env.get_template("config_container.py.jinja2") imports = FileImport(self.code_model) for client in self.code_model.clients: - imports.merge(client.config.imports(self.async_mode, serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.CLIENT)) + imports.merge( + client.config.imports( + self.async_mode, serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.CLIENT + ) + ) return template.render( code_model=self.code_model, async_mode=self.async_mode, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py index 6a346e1f06..1da2ce7e27 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py @@ -9,7 +9,9 @@ class ModelInitSerializer: - def __init__(self, code_model: CodeModel, env: Environment, *, models: List[ModelType], enums: List[EnumType]) -> None: + def __init__( + self, code_model: CodeModel, env: Environment, *, models: List[ModelType], enums: List[EnumType] + ) -> None: self.code_model = code_model self.env = env self.models = models diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 269d1df424..98a4e4c6c3 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -23,7 +23,9 @@ def _documentation_string(prop: Property, description_keyword: str, docstring_ty class _ModelSerializer(BaseSerializer, ABC): - def __init__(self, code_model, env, async_mode = False, *, models: List[ModelType], client_namespace: Optional[str] = None): + def __init__( + self, code_model, env, async_mode=False, *, models: List[ModelType], client_namespace: Optional[str] = None + ): super().__init__(code_model, env, async_mode, client_namespace=client_namespace) self.models = models @@ -231,7 +233,9 @@ def imports(self) -> FileImport: continue file_import.merge(model.imports(is_operation_file=False, serialize_namespace=self.serialize_namespace)) for prop in model.properties: - file_import.merge(prop.imports(serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.MODEL)) + file_import.merge( + prop.imports(serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.MODEL) + ) if model.is_polymorphic: file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) if not model.internal and self.init_line(model): diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py index 1a27d30400..1a94530a0f 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/parameter_serializer.py @@ -109,7 +109,8 @@ def serialize_parameter(self, parameter: ParameterType, serializer_name: str) -> return f"[{serialize_line} if q is not None else '' for q in {origin_name}]" return serialize_line - def serialize_path(self, + def serialize_path( + self, parameters: Union[ List[Parameter], List[RequestBuilderParameter], diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index b0575d0f53..fae40d73bf 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -18,9 +18,7 @@ def imports(self) -> FileImport: ImportType.STDLIB, ) for nu in self.code_model.named_unions: - file_import.merge( - nu.imports(is_types_file=True, serialize_namespace=self.serialize_namespace) - ) + file_import.merge(nu.imports(is_types_file=True, serialize_namespace=self.serialize_namespace)) return file_import def serialize(self) -> str: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py index bd10b31577..e3defdd349 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py @@ -8,7 +8,6 @@ from pathlib import Path - def method_signature_and_response_type_annotation_template( *, method_signature: str, @@ -34,7 +33,6 @@ def get_namespace_from_package_name(package_name: Optional[str]) -> str: return (package_name or "").replace("-", ".") - def _improve_json_string(template_representation: str) -> Any: origin = template_representation.split("\n") final = [] From c8cac2c54f0559bdf56513133c2bfbae8d3c4a35 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 15:50:05 +0800 Subject: [PATCH 24/49] review --- .../pygen/codegen/models/code_model.py | 6 +-- .../generator/pygen/codegen/models/utils.py | 1 - .../pygen/codegen/serializers/__init__.py | 2 +- .../serializers/metadata_serializer.py | 39 +++++++++++++++---- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index b78203cfe7..efb4caa8cb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -111,7 +111,7 @@ def get_relative_import_path( serialize_namespace: str, imported_namespace: Optional[str] = None, *, - namespace_type: NamespaceType = NamespaceType.NONE, + namespace_type: NamespaceType = NamespaceType.CLIENT, async_mode: bool = False, module_name: Optional[str] = None, ) -> str: @@ -256,9 +256,9 @@ def operations_folder_name(self, client_namespace: str) -> str: return self._operations_folder_name[client_namespace] def get_serialize_namespace( - self, client_namespace: str, async_mode: bool = False, namespace_type: NamespaceType = NamespaceType.NONE + self, client_namespace: str, async_mode: bool = False, namespace_type: NamespaceType = NamespaceType.CLIENT ) -> str: - if namespace_type == NamespaceType.NONE: + if namespace_type == NamespaceType.CLIENT: return client_namespace + (".aio" if async_mode else "") if namespace_type == NamespaceType.MODEL: return client_namespace + ".models" diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index c7b44fbda9..c0ee5a950b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -26,7 +26,6 @@ def add_to_pylint_disable(curr_str: str, entry: str) -> str: class NamespaceType(str, Enum): """Special signal for impports""" - NONE = "none" MODEL = "model" OPERATION = "operation" CLIENT = "client" diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 98dc950312..c538efa5c1 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -421,7 +421,7 @@ def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str ) def _serialize_and_write_metadata(self, env: Environment, namespace: str) -> None: - metadata_serializer = MetadataSerializer(self.code_model, env) + metadata_serializer = MetadataSerializer(self.code_model, env, client_namespace=namespace) self.write_file(self.exec_path(namespace) / Path("_metadata.json"), metadata_serializer.serialize()) @property diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py index 35d374983f..0ae74b2db1 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py @@ -15,7 +15,9 @@ ImportType, CodeModel, ) +from ..models.utils import NamespaceType from .builder_serializer import get_operation_serializer +from .base_serializer import BaseSerializer from .import_serializer import FileImportSerializer @@ -104,11 +106,10 @@ def _mixin_typing_definitions( return sync_mixin_typing_definitions, async_mixin_typing_definitions -class MetadataSerializer: - def __init__(self, code_model: CodeModel, env: Environment) -> None: - self.code_model = code_model +class MetadataSerializer(BaseSerializer): + def __init__(self, code_model: CodeModel, env: Environment, *, client_namespace: Optional[str] = None): + super().__init__(code_model, env, client_namespace=client_namespace) self.client = self.code_model.clients[0] # we only do one client for multiapi - self.env = env def _choose_api_version(self) -> Tuple[str, List[str]]: chosen_version = "" @@ -161,6 +162,10 @@ def _is_paging(operation): self.code_model.options["package_version"] = "0.1.0" template = self.env.get_template("metadata.json.jinja2") + client_serialize_namespace = self.code_model.get_serialize_namespace(self.client_namespace, async_mode=False) + client_serialize_namespace_async = self.code_model.get_serialize_namespace( + self.client_namespace, async_mode=True + ) return template.render( code_model=self.code_model, chosen_version=chosen_version, @@ -176,23 +181,41 @@ def _is_paging(operation): async_mixin_imports=async_mixin_imports, sync_mixin_typing_definitions=sync_mixin_typing_definitions, async_mixin_typing_definitions=async_mixin_typing_definitions, - sync_client_imports=_json_serialize_imports(self.client.imports_for_multiapi(async_mode=False).to_dict()), - async_client_imports=_json_serialize_imports(self.client.imports_for_multiapi(async_mode=True).to_dict()), + sync_client_imports=_json_serialize_imports( + self.client.imports_for_multiapi( + async_mode=False, serialize_namespace=client_serialize_namespace + ).to_dict() + ), + async_client_imports=_json_serialize_imports( + self.client.imports_for_multiapi( + async_mode=True, serialize_namespace=client_serialize_namespace_async + ).to_dict() + ), sync_config_imports=_json_serialize_imports( - self.client.config.imports_for_multiapi(async_mode=False).to_dict() + self.client.config.imports_for_multiapi( + async_mode=False, serialize_namespace=client_serialize_namespace + ).to_dict() ), async_config_imports=_json_serialize_imports( - self.client.config.imports_for_multiapi(async_mode=True).to_dict() + self.client.config.imports_for_multiapi( + async_mode=True, serialize_namespace=client_serialize_namespace_async + ).to_dict() ), get_async_operation_serializer=functools.partial( get_operation_serializer, code_model=self.client.code_model, async_mode=True, + serialize_namespace=self.code_model.get_serialize_namespace( + self.client_namespace, async_mode=True, namespace_type=NamespaceType.OPERATION + ), ), get_sync_operation_serializer=functools.partial( get_operation_serializer, code_model=self.client.code_model, async_mode=False, + serialize_namespace=self.code_model.get_serialize_namespace( + self.client_namespace, async_mode=False, namespace_type=NamespaceType.OPERATION + ), ), has_credential=bool(self.client.credential), ) From b0b1b431aa87be9b9cfe5aa82e425987fca5c751 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 16:19:45 +0800 Subject: [PATCH 25/49] review --- .../generator/pygen/codegen/models/paging_operation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index e663061855..9d13522633 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -129,6 +129,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: return FileImport(self.code_model) file_import = self._imports_shared(async_mode, **kwargs) file_import.merge(super().imports(async_mode, **kwargs)) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) if self.code_model.options["tracing"] and self.want_tracing: file_import.add_submodule_import( "azure.core.tracing.decorator", @@ -136,7 +137,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ImportType.SDKCORE, ) if self.next_request_builder: - file_import.merge(self.get_request_builder_import(self.next_request_builder, async_mode)) + file_import.merge(self.get_request_builder_import(self.next_request_builder, async_mode, serialize_namespace)) elif any(p.is_api_version for p in self.client.parameters): file_import.add_import("urllib.parse", ImportType.STDLIB) file_import.add_submodule_import( @@ -145,7 +146,6 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ImportType.SDKCORE, ) if self.code_model.options["models_mode"] == "dpg": - serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) relative_path = self.code_model.get_relative_import_path(serialize_namespace, module_name="_model_base") file_import.merge(self.item_type.imports(**kwargs)) if self.default_error_deserialization or any(r.type for r in self.responses): From a21d7ad79265da762f81bc613efd3841c7eee13d Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 16:20:01 +0800 Subject: [PATCH 26/49] review --- .../generator/pygen/codegen/models/paging_operation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index 9d13522633..28d0368a56 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -137,7 +137,9 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ImportType.SDKCORE, ) if self.next_request_builder: - file_import.merge(self.get_request_builder_import(self.next_request_builder, async_mode, serialize_namespace)) + file_import.merge( + self.get_request_builder_import(self.next_request_builder, async_mode, serialize_namespace) + ) elif any(p.is_api_version for p in self.client.parameters): file_import.add_import("urllib.parse", ImportType.STDLIB) file_import.add_submodule_import( From 6c37c0cc0b575551c6582e471f6cf86e2e5a35b9 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 18:33:38 +0800 Subject: [PATCH 27/49] review --- .../generator/pygen/codegen/_utils.py | 2 -- .../pygen/codegen/serializers/__init__.py | 21 ++++++++++--------- .../codegen/serializers/model_serializer.py | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/_utils.py b/packages/http-client-python/generator/pygen/codegen/_utils.py index 3611b55b60..a89af22c47 100644 --- a/packages/http-client-python/generator/pygen/codegen/_utils.py +++ b/packages/http-client-python/generator/pygen/codegen/_utils.py @@ -3,7 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import List DEFAULT_HEADER_TEXT = ( "Copyright (c) {company_name} Corporation. All rights reserved.\n" @@ -17,6 +16,5 @@ VALID_PACKAGE_MODE = SWAGGER_PACKAGE_MODE + TYPESPEC_PACKAGE_MODE NAME_LENGTH_LIMIT = 40 - def get_parent_namespace(namespace: str) -> str: return namespace.rsplit(".", 1)[0] diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index c538efa5c1..6796bcdc05 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -106,17 +106,25 @@ def serialize(self) -> None: general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) for client_namespace, client_namespace_type in self.code_model.client_namespace_types.items(): + exec_path = self.exec_path(client_namespace) if client_namespace == "": # Write the setup file if self.code_model.options["basic_setup_py"]: self.write_file( - self.exec_path(client_namespace) / Path("setup.py"), general_serializer.serialize_setup_file() + exec_path / Path("setup.py"), general_serializer.serialize_setup_file() ) # add packaging files in root namespace (e.g. setup.py, README.md, etc.) if self.code_model.options["package_mode"]: self._serialize_and_write_package_files(client_namespace) + # write apiview_mapping_python.json + if self.code_model.options.get("emit_cross_language_definition_file"): + self.write_file( + exec_path / Path("apiview_mapping_python.json"), + general_serializer.serialize_cross_language_definition_file(), + ) + # add generated samples and generated tests # TODO # if self.code_model.options["show_operations"] and self.code_model.has_operations: @@ -130,7 +138,7 @@ def serialize(self) -> None: else: # add pkgutil init file if no clients in this namespace self.write_file( - self.exec_path(client_namespace) / Path("__init__.py"), + exec_path / Path("__init__.py"), general_serializer.serialize_pkgutil_init_file(), ) @@ -149,7 +157,7 @@ def serialize(self) -> None: if not self.code_model.options["models_mode"]: # keep models file if users ended up just writing a models file - model_path = self.exec_path(client_namespace) / Path("models.py") + model_path = exec_path / Path("models.py") if self.read_file(model_path): self.write_file(model_path, self.read_file(model_path)) @@ -406,13 +414,6 @@ def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str general_serializer.serialize_validation_file(), ) - # write apiview_mapping_python.json - if self.code_model.options.get("emit_cross_language_definition_file"): - self.write_file( - exec_path / Path("apiview_mapping_python.json"), - general_serializer.serialize_cross_language_definition_file(), - ) - # write _types.py if self.code_model.named_unions: self.write_file( diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 98a4e4c6c3..120e164551 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -151,7 +151,7 @@ def imports(self) -> FileImport: for model in self.models: file_import.merge(model.imports(is_operation_file=False)) for param in self._init_line_parameters(model): - file_import.merge(param.imports()) + file_import.merge(param.imports(serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.MODEL)) return file_import From d0dcc6880b2f83fb81a97b6c4962c52f8ce05eb4 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 18:52:10 +0800 Subject: [PATCH 28/49] review --- .../pygen/codegen/serializers/__init__.py | 11 +++++------ .../codegen/serializers/sample_serializer.py | 16 ++++------------ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 6796bcdc05..593696a746 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -126,12 +126,11 @@ def serialize(self) -> None: ) # add generated samples and generated tests - # TODO - # if self.code_model.options["show_operations"] and self.code_model.has_operations: - # if self.code_model.options["generate_sample"]: - # self._serialize_and_write_sample(env, namespace=client_namespace) - # if self.code_model.options["generate_test"]: - # self._serialize_and_write_test(env, namespace=client_namespace) + if self.code_model.options["show_operations"] and self.code_model.has_operations: + if self.code_model.options["generate_sample"]: + self._serialize_and_write_sample(env, namespace=client_namespace) + # if self.code_model.options["generate_test"]: + # self._serialize_and_write_test(env, namespace=client_namespace) elif client_namespace_type.clients: # add clients folder if there are clients in this namespace self._serialize_client_and_config_files(client_namespace, client_namespace_type.clients, env) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py index a5c69713ed..30e05da652 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -21,6 +21,7 @@ FileImport, ) from .utils import get_namespace_config, get_namespace_from_package_name +from .._utils import get_parent_namespace _LOGGER = logging.getLogger(__name__) @@ -34,10 +35,8 @@ def __init__( operation: OperationBase[Any], sample: Dict[str, Any], file_name: str, - *, - serialize_namespace: Optional[str] = None, ) -> None: - super().__init__(code_model, env, serialize_namespace=serialize_namespace) + super().__init__(code_model, env) self.operation_group = operation_group self.operation = operation self.sample = sample @@ -46,15 +45,8 @@ def __init__( def _imports(self) -> FileImportSerializer: imports = FileImport(self.code_model) - namespace_from_package_name = get_namespace_from_package_name(self.code_model.options["package_name"]) - namespace_config = get_namespace_config(self.code_model.namespace, self.code_model.options["multiapi"]) - namespace = namespace_from_package_name or namespace_config - # mainly for "azure-mgmt-rdbms" - if not self.code_model.options["multiapi"] and namespace_config.count(".") > namespace_from_package_name.count( - "." - ): - namespace = namespace_config - client = self.code_model.clients[0] + client = self.operation_group.client + namespace = get_parent_namespace(client.client_namespace) if self.code_model.options["multiapi"] else client.client_namespace imports.add_submodule_import(namespace, client.name, ImportType.LOCAL) credential_type = getattr(client.credential, "type", None) if isinstance(credential_type, TokenCredentialType): From 05309a8ce3dc094e8cab647ef3b71e9e70282694 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 19:18:48 +0800 Subject: [PATCH 29/49] for sample and test --- .../generator/pygen/codegen/serializers/__init__.py | 4 ++-- .../generator/pygen/codegen/serializers/test_serializer.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 593696a746..71d89d0e64 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -129,8 +129,8 @@ def serialize(self) -> None: if self.code_model.options["show_operations"] and self.code_model.has_operations: if self.code_model.options["generate_sample"]: self._serialize_and_write_sample(env, namespace=client_namespace) - # if self.code_model.options["generate_test"]: - # self._serialize_and_write_test(env, namespace=client_namespace) + if self.code_model.options["generate_test"]: + self._serialize_and_write_test(env, namespace=client_namespace) elif client_namespace_type.clients: # add clients folder if there are clients in this namespace self._serialize_client_and_config_files(client_namespace, client_namespace_type.clients, env) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index fd75a3fb93..bfbfd82d71 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -143,9 +143,8 @@ def __init__( env: Environment, *, is_async: bool = False, - client_namespace: Optional[str] = None, ) -> None: - super().__init__(code_model, env, is_async, client_namespace=client_namespace) + super().__init__(code_model, env, is_async) self.is_async = is_async @property @@ -157,9 +156,8 @@ def test_names(self) -> List[TestName]: return [TestName(self.code_model, c.name, is_async=self.is_async) for c in self.code_model.clients] def add_import_client(self, imports: FileImport) -> None: - namespace = get_namespace_from_package_name(self.code_model.options["package_name"]) for client in self.code_model.clients: - imports.add_submodule_import(namespace + self.aio_str, client.name, ImportType.STDLIB) + imports.add_submodule_import(client.client_namespace + self.aio_str, client.name, ImportType.STDLIB) @property def import_clients(self) -> FileImportSerializer: From f3d25fff3ffa5bcc21e48d9dd95937f079300da4 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 20:13:28 +0800 Subject: [PATCH 30/49] for sample and test --- packages/http-client-python/generator/pygen/black.py | 2 +- .../generator/pygen/codegen/models/operation_group.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/generator/pygen/black.py b/packages/http-client-python/generator/pygen/black.py index f8fd58b223..839692c06f 100644 --- a/packages/http-client-python/generator/pygen/black.py +++ b/packages/http-client-python/generator/pygen/black.py @@ -43,7 +43,7 @@ def process(self) -> bool: "venv", "env", ) - and not Path(f).parts[0].startswith(".") + and (not Path(f).parts[0].startswith(".") or Path(f).parts[0].startswith("..")) and Path(f).suffix == ".py" ], ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 4e5204cd04..d1637f10e2 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -138,7 +138,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ) if self.has_abstract_operations: file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor"), + self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor", async_mode=async_mode), "raise_if_not_implemented", ImportType.LOCAL, ) From 42f29c9edbb715b817b5ab7af355dd47ab7a0c1a Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 6 Dec 2024 20:27:09 +0800 Subject: [PATCH 31/49] review --- packages/http-client-python/generator/pygen/black.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/pygen/black.py b/packages/http-client-python/generator/pygen/black.py index 839692c06f..bdc1619d24 100644 --- a/packages/http-client-python/generator/pygen/black.py +++ b/packages/http-client-python/generator/pygen/black.py @@ -43,7 +43,7 @@ def process(self) -> bool: "venv", "env", ) - and (not Path(f).parts[0].startswith(".") or Path(f).parts[0].startswith("..")) + and (not Path(f).parts[0].startswith(".") or Path(f).parts[0] == "..") and Path(f).suffix == ".py" ], ) From ab88321a84dcd70ba6c88a621e00d938a2500c0c Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 9 Dec 2024 20:02:39 +0800 Subject: [PATCH 32/49] debug --- .../generator/pygen/codegen/models/client.py | 1 + .../generator/pygen/codegen/models/model_type.py | 2 +- .../generator/pygen/codegen/models/operation.py | 6 +++--- .../generator/pygen/codegen/models/property.py | 4 ++-- .../generator/pygen/codegen/serializers/__init__.py | 1 + .../pygen/codegen/serializers/builder_serializer.py | 8 ++++---- .../pygen/codegen/serializers/general_serializer.py | 3 ++- .../pygen/codegen/serializers/model_serializer.py | 12 +++++------- .../generator/pygen/codegen/templates/init.py.jinja2 | 2 +- 9 files changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index c10e27825d..77f9d1f800 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -397,6 +397,7 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) if self.code_model.options["package_version"]: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + x = self.code_model.get_relative_import_path(serialize_namespace, module_name="_version") file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace, module_name="_version"), "VERSION", diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 398ce38ca8..3c967ee0cf 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -290,7 +290,7 @@ def type_annotation(self, **kwargs: Any) -> str: return retval if is_operation_file or skip_quote else f'"{retval}"' def docstring_type(self, **kwargs: Any) -> str: - return f"~{self.code_model.namespace}.models.{self.type_annotation(need_model_alias=False, skip_quote=True)}" + return f"~{self.code_model.namespace}.models.{self.type_annotation(need_model_alias=False, skip_quote=True, **kwargs)}" def docstring_text(self, **kwargs: Any) -> str: return self.name diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index b755cf3411..a32c963214 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -148,10 +148,10 @@ def pylint_disable(self, async_mode: bool) -> str: retval = add_to_pylint_disable(retval, "name-too-long") return retval - def cls_type_annotation(self, *, async_mode: bool) -> str: + def cls_type_annotation(self, *, async_mode: bool, **kwargs: Any) -> str: if self.request_builder.method.lower() == "head" and self.code_model.options["head_as_boolean"]: return "ClsType[None]" - return f"ClsType[{self.response_type_annotation(async_mode=async_mode)}]" + return f"ClsType[{self.response_type_annotation(async_mode=async_mode, **kwargs)}]" def _response_docstring_helper(self, attr_name: str, **kwargs: Any) -> str: responses_with_body = [r for r in self.responses if r.type] @@ -213,7 +213,7 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: # pyl file_import = FileImport(self.code_model) file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) - response_types = [r.type_annotation(async_mode=async_mode) for r in self.responses if r.type] + response_types = [r.type_annotation(async_mode=async_mode, **kwargs) for r in self.responses if r.type] if len(set(response_types)) > 1: file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) if self.added_on: diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index 78c8902668..f7adafd351 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -98,10 +98,10 @@ def is_base_discriminator(self) -> bool: def xml_metadata(self) -> Optional[Dict[str, Union[str, bool]]]: return self.yaml_data.get("xmlMetadata") - def type_annotation(self, *, is_operation_file: bool = False) -> str: + def type_annotation(self, *, is_operation_file: bool = False, **kwargs: Any) -> str: if self.is_base_discriminator: return "str" - types_type_annotation = self.type.type_annotation(is_operation_file=is_operation_file) + types_type_annotation = self.type.type_annotation(is_operation_file=is_operation_file, **kwargs) if self.optional and self.client_default_value is None: return f"Optional[{types_type_annotation}]" return types_type_annotation diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 71d89d0e64..f8e42d7bb5 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -304,6 +304,7 @@ def _serialize_and_write_operations_folder( operation_groups=ogs, env=env, async_mode=async_mode, + client_namespace=namespace, ) self.write_file( exec_path / Path(f"{prefix_path}/{filename}.py"), diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 823f9d947e..181383b075 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -238,7 +238,7 @@ def _method_signature(self, builder: BuilderType) -> str: def method_signature_and_response_type_annotation( self, builder: BuilderType, *, want_decorators: Optional[bool] = True ) -> str: - response_type_annotation = builder.response_type_annotation(async_mode=self.async_mode) + response_type_annotation = builder.response_type_annotation(async_mode=self.async_mode, serialize_namespace=self.serialize_namespace) method_signature = self._method_signature(builder) decorators = self.decorators(builder) decorators_str = "" @@ -617,13 +617,13 @@ def pop_kwargs_from_signature(self, builder: OperationType) -> List[str]: for p in builder.parameters.parameters: if p.hide_in_operation_signature: kwargs.append(f'{p.client_name} = kwargs.pop("{p.client_name}", None)') - cls_annotation = builder.cls_type_annotation(async_mode=self.async_mode) + cls_annotation = builder.cls_type_annotation(async_mode=self.async_mode, serialize_namespace=self.serialize_namespace) kwargs.append(f"cls: {cls_annotation} = kwargs.pop(\n 'cls', None\n)") return kwargs def response_docstring(self, builder: OperationType) -> List[str]: response_str = f":return: {builder.response_docstring_text(async_mode=self.async_mode)}" - rtype_str = f":rtype: {builder.response_docstring_type(async_mode=self.async_mode)}" + rtype_str = f":rtype: {builder.response_docstring_type(async_mode=self.async_mode, serialize_namespace=self.serialize_namespace)}" return [ response_str, rtype_str, @@ -959,7 +959,7 @@ def response_deserialization( deserialize_func = "_deserialize_xml" deserialize_code.append(f"deserialized = {deserialize_func}(") deserialize_code.append( - f" {response.type.type_annotation(is_operation_file=True)},{pylint_disable}" + f" {response.type.type_annotation(is_operation_file=True, serialize_namespace=self.serialize_namespace)},{pylint_disable}" ) deserialize_code.append(f" response.{response_attr}(){response.result_property}{format_filed}") deserialize_code.append(")") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index 9796178d46..69a34be467 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -62,6 +62,7 @@ def serialize_init_file(self, clients: List[Client]) -> str: code_model=self.code_model, clients=clients, async_mode=self.async_mode, + serialize_namespace=self.serialize_namespace, ) def serialize_service_client_file(self, clients: List[Client]) -> str: @@ -109,7 +110,7 @@ def serialize_vendor_file(self, clients: List[Client]) -> str: for client in clients: if client.has_mixin: file_import.add_submodule_import( - "._configuration", + self.code_model.get_relative_import_path(self.serialize_namespace, client.client_namespace, async_mode=self.async_mode, module_name="_configuration"), f"{client.name}Configuration", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 120e164551..36ff9c20c1 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -71,12 +71,11 @@ def initialize_discriminator_property(model: ModelType, prop: Property) -> str: typing = "str" return f"self.{prop.client_name}: {typing} = {discriminator_value}" - @staticmethod - def initialize_standard_property(prop: Property): + def initialize_standard_property(self, prop: Property): if not (prop.optional or prop.client_default_value is not None): - return f"{prop.client_name}: {prop.type_annotation()},{prop.pylint_disable()}" + return f"{prop.client_name}: {prop.type_annotation(serialize_namespace=self.serialize_namespace)},{prop.pylint_disable()}" return ( - f"{prop.client_name}: {prop.type_annotation()} = " + f"{prop.client_name}: {prop.type_annotation(serialize_namespace=self.serialize_namespace)} = " f"{prop.client_default_value_declaration},{prop.pylint_disable()}" ) @@ -272,8 +271,7 @@ def get_properties_to_declare(model: ModelType) -> List[Property]: raise ValueError("We do not generate anonymous properties") return properties_to_declare - @staticmethod - def declare_property(prop: Property) -> str: + def declare_property(self, prop: Property) -> str: args = [] if prop.client_name != prop.wire_name or prop.is_discriminator: args.append(f'name="{prop.wire_name}"') @@ -297,7 +295,7 @@ def declare_property(prop: Property) -> str: if prop.is_discriminator and isinstance(prop.type, (ConstantType, EnumValue)) and prop.type.value else "" ) - generated_code = f'{prop.client_name}: {prop.type_annotation()} = {field}({", ".join(args)})' + generated_code = f'{prop.client_name}: {prop.type_annotation(serialize_namespace=self.serialize_namespace)} = {field}({", ".join(args)})' # there is 4 spaces indentation so original line length limit 120 - 4 = 116 pylint_disable = ( " # pylint: disable=line-too-long" diff --git a/packages/http-client-python/generator/pygen/codegen/templates/init.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/init.py.jinja2 index ce16adcbcf..3d05188843 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/init.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/init.py.jinja2 @@ -8,7 +8,7 @@ from .{{ client.filename }} import {{ client.name }} # type: ignore {% endfor %} {% endif %} {% if not async_mode and code_model.options['package_version']%} -from ._version import VERSION +from {{ code_model.get_relative_import_path(serialize_namespace, module_name="_version") }} import VERSION __version__ = VERSION {% endif %} From 7f88e3e6c22969b5a6932b39809069c5bfd55f4f Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 9 Dec 2024 20:07:49 +0800 Subject: [PATCH 33/49] review --- .../http-client-python/generator/pygen/codegen/models/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index 77f9d1f800..c10e27825d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -397,7 +397,6 @@ def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) if self.code_model.options["package_version"]: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - x = self.code_model.get_relative_import_path(serialize_namespace, module_name="_version") file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace, module_name="_version"), "VERSION", From adf95fe98747c8f80b7341bf8b70aabd7d984e88 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 10 Dec 2024 10:43:36 +0800 Subject: [PATCH 34/49] fix --- .../generator/pygen/codegen/models/lro_operation.py | 4 ++-- .../generator/pygen/codegen/models/lro_paging_operation.py | 4 ++-- .../generator/pygen/codegen/models/paging_operation.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index 0904c65cd9..8717954aa5 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -92,9 +92,9 @@ def response_type_annotation(self, **kwargs) -> str: return lro_response.type_annotation(**kwargs) return "None" - def cls_type_annotation(self, *, async_mode: bool) -> str: + def cls_type_annotation(self, *, async_mode: bool, **kwargs: Any) -> str: """We don't want the poller to show up in ClsType, so we call super() on response type annotation""" - return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]" + return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode, **kwargs)}]" def get_poller_with_response_type(self, async_mode: bool) -> str: return self.response_type_annotation(async_mode=async_mode) diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_paging_operation.py index 326ed2aac9..7e4ad4e3ac 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_paging_operation.py @@ -20,8 +20,8 @@ def success_status_codes(self): def operation_type(self) -> str: return "lropaging" - def cls_type_annotation(self, *, async_mode: bool) -> str: - return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]" + def cls_type_annotation(self, *, async_mode: bool, **kwargs: Any) -> str: + return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode, **kwargs)}]" def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: lro_imports = LROOperationBase.imports(self, async_mode, **kwargs) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index 28d0368a56..c81eb870d0 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -103,8 +103,8 @@ def item_type(self) -> ModelType: def operation_type(self) -> str: return "paging" - def cls_type_annotation(self, *, async_mode: bool) -> str: - return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]" + def cls_type_annotation(self, *, async_mode: bool, **kwargs: Any) -> str: + return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode, **kwargs)}]" def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import = super()._imports_shared(async_mode, **kwargs) From 7c5532c7d680c0bd3b5b69d44889df796d645ce5 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 10 Dec 2024 13:53:52 +0800 Subject: [PATCH 35/49] fix --- .../emitter/src/external-process.ts | 8 ++-- .../eng/scripts/ci/regenerate.ts | 3 ++ .../pygen/codegen/models/code_model.py | 37 +++++++++++++------ .../pygen/codegen/serializers/__init__.py | 16 +++++--- .../codegen/serializers/general_serializer.py | 12 +++--- .../pygen/codegen/templates/vendor.py.jinja2 | 8 ++-- .../generator/test/azure/requirements.txt | 1 + 7 files changed, 55 insertions(+), 30 deletions(-) diff --git a/packages/http-client-python/emitter/src/external-process.ts b/packages/http-client-python/emitter/src/external-process.ts index eb2471285a..22905d8d24 100644 --- a/packages/http-client-python/emitter/src/external-process.ts +++ b/packages/http-client-python/emitter/src/external-process.ts @@ -22,10 +22,10 @@ export async function saveCodeModelAsYaml(name: string, codemodel: unknown): Pro const filename = createTempPath(".yaml", name); const yamlStr = jsyaml.dump(codemodel); await writeFile(filename, yamlStr); - // await writeFile( - // joinPaths("C:/dev/typespec/packages/http-client-python", "alpha", "output.yaml"), - // yamlStr, - // ); + await writeFile( + joinPaths("C:/dev/typespec/packages/http-client-python", "alpha", "output.yaml"), + yamlStr, + ); return filename; } diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index 629c29c411..80cf60899b 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -121,6 +121,9 @@ const EMITTER_OPTIONS: Record | Record str: def client_filename(self) -> str: return self.clients[0].filename - def need_vendored_code(self, async_mode: bool) -> bool: - """Whether we need to vendor code in the _vendor.py file for this SDK""" - if self.has_abstract_operations: - return True - if async_mode: - return self.need_mixin_abc - return self.need_mixin_abc or self.has_etag or self.has_form_data - - @property - def need_mixin_abc(self) -> bool: - return any(c for c in self.clients if c.has_mixin) + def get_clients(self, client_namespace: str) -> List[Client]: + return self.client_namespace_types.get(client_namespace, ClientNamespaceType()).clients + + def is_top_namespace(self, client_namespace: str) -> bool: + return client_namespace == self.namespace + + def need_vendored_code(self, async_mode: bool, client_namespace: str) -> bool: + """Whether we need to vendor code in the _vendor.py in specific namespace""" + self.need_vendored_form_data(async_mode, client_namespace) or self.need_vendored_etag(client_namespace) or self.need_vendored_abstract(client_namespace) or self.need_vendored_mixin(client_namespace) + + def need_vendored_form_data(self, async_mode: bool, client_namespace: str) -> bool: + return (not async_mode) and self.is_top_namespace(client_namespace) and self.has_form_data and self.options["models_mode"] == "dpg" + + def need_vendored_etag(self, client_namespace: str) -> bool: + return self.is_top_namespace(client_namespace) and self.has_etag + + def need_vendored_abstract(self, client_namespace: str) -> bool: + return self.is_top_namespace(client_namespace) and self.has_abstract_operations + + def need_vendored_mixin(self, client_namespace: str) -> bool: + return self.has_mixin_abc(client_namespace) + + def has_mixin_abc(self, client_namespace: str) -> bool: + return any(c for c in self.get_clients(client_namespace) if c.has_mixin) @property def has_abstract_operations(self) -> bool: @@ -297,7 +310,7 @@ def model_types(self, val: List[ModelType]) -> None: self._model_types = val def public_model_types(self, models: Optional[ModelType] = None) -> List[ModelType]: - models = models or self.model_types + models = self.model_types if models is None else models return [m for m in models if not m.internal and not m.base == "json"] @property diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index f8e42d7bb5..8324840545 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -373,18 +373,24 @@ def _serialize_client_and_config_files( # if there was a patch file before, we keep it self._keep_patch_file(exec_path / Path(f"{async_path}_patch.py"), env) - def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str) -> None: + # sometimes we need define additional Mixin class for client in _vendor.py + self._serialize_and_write_vendor_file(env, namespace) + + def _serialize_and_write_vendor_file(self, env: Environment, namespace: str) -> None: exec_path = self.exec_path(namespace) # write _vendor.py for async_mode, async_path in self.serialize_loop: - if self.code_model.need_vendored_code(async_mode=async_mode): + if self.code_model.need_vendored_code(async_mode=async_mode, client_namespace=namespace): self.write_file( exec_path / Path(f"{async_path}_vendor.py"), - GeneralSerializer(code_model=self.code_model, env=env, async_mode=async_mode).serialize_vendor_file( - self.code_model.clients - ), + GeneralSerializer(code_model=self.code_model, env=env, async_mode=async_mode, client_namespace=namespace).serialize_vendor_file(), ) + def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str) -> None: + exec_path = self.exec_path(namespace) + # write _vendor.py + self._serialize_and_write_vendor_file(env, namespace) + general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) # write _version.py diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index 69a34be467..fc43ceb5e6 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -85,12 +85,13 @@ def serialize_service_client_file(self, clients: List[Client]) -> str: serialize_namespace=self.serialize_namespace, ) - def serialize_vendor_file(self, clients: List[Client]) -> str: + def serialize_vendor_file(self) -> str: template = self.env.get_template("vendor.py.jinja2") + clients = self.code_model.get_clients(self.client_namespace) # configure imports file_import = FileImport(self.code_model) - if self.code_model.need_mixin_abc: + if self.code_model.need_vendored_mixin(self.client_namespace): file_import.add_submodule_import( "abc", "ABC", @@ -110,18 +111,18 @@ def serialize_vendor_file(self, clients: List[Client]) -> str: for client in clients: if client.has_mixin: file_import.add_submodule_import( - self.code_model.get_relative_import_path(self.serialize_namespace, client.client_namespace, async_mode=self.async_mode, module_name="_configuration"), + "._configuration", f"{client.name}Configuration", ImportType.LOCAL, ) - if self.code_model.has_etag: + if self.code_model.need_vendored_etag(self.client_namespace): file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) file_import.add_submodule_import( "", "MatchConditions", ImportType.SDKCORE, ) - if self.code_model.has_form_data and self.code_model.options["models_mode"] == "dpg" and not self.async_mode: + if self.code_model.need_vendored_form_data(self.async_mode, self.client_namespace): file_import.add_submodule_import("typing", "IO", ImportType.STDLIB) file_import.add_submodule_import("typing", "Tuple", ImportType.STDLIB) file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) @@ -149,6 +150,7 @@ def serialize_vendor_file(self, clients: List[Client]) -> str: ), async_mode=self.async_mode, clients=clients, + client_namespace=self.client_namespace, ) def serialize_config_file(self, clients: List[Client]) -> str: diff --git a/packages/http-client-python/generator/pygen/codegen/templates/vendor.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/vendor.py.jinja2 index 512e3f2a17..b92ef3b5a2 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/vendor.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/vendor.py.jinja2 @@ -3,7 +3,7 @@ {{ imports }} -{% if code_model.need_mixin_abc %} +{% if code_model.need_vendored_mixin(client_namespace) %} {% for client in clients | selectattr("has_mixin") %} {% set pylint_disable = "# pylint: disable=name-too-long" if (client.name | length) + ("MixinABC" | length) > 40 else "" %} class {{ client.name }}MixinABC( {{ pylint_disable }} @@ -16,7 +16,7 @@ class {{ client.name }}MixinABC( {{ pylint_disable }} _deserialize: "Deserializer" {% endfor %} {% endif %} -{% if code_model.has_abstract_operations %} +{% if code_model.need_vendored_abstract(client_namespace) %} def raise_if_not_implemented(cls, abstract_methods): not_implemented = [f for f in abstract_methods if not callable(getattr(cls, f, None))] @@ -27,7 +27,7 @@ def raise_if_not_implemented(cls, abstract_methods): ) {% endif %} -{% if code_model.has_etag %} +{% if code_model.need_vendored_etag(client_namespace) %} def quote_etag(etag: Optional[str]) -> Optional[str]: if not etag or etag == "*": return etag @@ -57,7 +57,7 @@ def prep_if_none_match(etag: Optional[str], match_condition: Optional[MatchCondi return "*" return None {% endif %} -{% if code_model.has_form_data and code_model.options["models_mode"] == "dpg" and not async_mode %} +{% if code_model.need_vendored_form_data(async_mode, client_namespace) %} # file-like tuple could be `(filename, IO (or bytes))` or `(filename, IO (or bytes), content_type)` FileContent = Union[str, bytes, IO[str], IO[bytes]] diff --git a/packages/http-client-python/generator/test/azure/requirements.txt b/packages/http-client-python/generator/test/azure/requirements.txt index 56fb69745d..22b8c430b5 100644 --- a/packages/http-client-python/generator/test/azure/requirements.txt +++ b/packages/http-client-python/generator/test/azure/requirements.txt @@ -24,6 +24,7 @@ mypy==1.13.0 -e ./generated/azure-example-basic -e ./generated/azure-resource-manager-common-properties -e ./generated/azure-resource-manager-resources +-e ./generated/client-namespace # common test case -e ./generated/authentication-api-key From 2781b09ce518174fc4341ee25e15c4d6885660c9 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 10 Dec 2024 14:14:18 +0800 Subject: [PATCH 36/49] new test --- .../pygen/codegen/models/code_model.py | 2 +- .../pygen/codegen/models/operation_group.py | 2 +- .../asynctests/test_client_namespace_async.py | 32 +++++++++++++++++++ .../mock_api_tests/test_client_namespace.py | 30 +++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py create mode 100644 packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index a16823f0b7..0d715cb91d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -236,7 +236,7 @@ def is_top_namespace(self, client_namespace: str) -> bool: def need_vendored_code(self, async_mode: bool, client_namespace: str) -> bool: """Whether we need to vendor code in the _vendor.py in specific namespace""" - self.need_vendored_form_data(async_mode, client_namespace) or self.need_vendored_etag(client_namespace) or self.need_vendored_abstract(client_namespace) or self.need_vendored_mixin(client_namespace) + return self.need_vendored_form_data(async_mode, client_namespace) or self.need_vendored_etag(client_namespace) or self.need_vendored_abstract(client_namespace) or self.need_vendored_mixin(client_namespace) def need_vendored_form_data(self, async_mode: bool, client_namespace: str) -> bool: return (not async_mode) and self.is_top_namespace(client_namespace) and self.has_form_data and self.options["models_mode"] == "dpg" diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index d1637f10e2..12a4b8908a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -131,7 +131,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: if self.is_mixin: file_import.add_submodule_import( self.code_model.get_relative_import_path( - serialize_namespace, self.code_model.namespace, module_name="_vendor", async_mode=async_mode + serialize_namespace, self.client.client_namespace, module_name="_vendor", async_mode=async_mode ), f"{self.client.name}MixinABC", ImportType.LOCAL, diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py b/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py new file mode 100644 index 0000000000..3a70b81ba8 --- /dev/null +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py @@ -0,0 +1,32 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from client.clientnamespace.aio import ClientNamespaceFirstClient +from client.clientnamespace.first.models import FirstClientResult + +from client.clientnamespace.second.aio import ClientNamespaceSecondClient +from client.clientnamespace.second.models import SecondClientResult +from client.clientnamespace.second.sub.models import SecondClientEnumType + + + +@pytest.fixture +async def first_client(): + async with ClientNamespaceFirstClient() as client: + yield client + +@pytest.fixture +async def second_client(): + async with ClientNamespaceSecondClient() as client: + yield client + +@pytest.mark.asyncio +async def test_get_first(first_client: ClientNamespaceFirstClient): + assert await first_client.get_first() == FirstClientResult(name="first") + +@pytest.mark.asyncio +async def test_get_second(second_client: ClientNamespaceSecondClient): + assert await second_client.get_second() == SecondClientResult(type=SecondClientEnumType.SECOND) diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py b/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py new file mode 100644 index 0000000000..b629c26240 --- /dev/null +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py @@ -0,0 +1,30 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from client.clientnamespace import ClientNamespaceFirstClient +from client.clientnamespace.first.models import FirstClientResult + +from client.clientnamespace.second import ClientNamespaceSecondClient +from client.clientnamespace.second.models import SecondClientResult +from client.clientnamespace.second.sub.models import SecondClientEnumType + + + +@pytest.fixture +def first_client(): + with ClientNamespaceFirstClient() as client: + yield client + +@pytest.fixture +def second_client(): + with ClientNamespaceSecondClient() as client: + yield client + +def test_get_first(first_client: ClientNamespaceFirstClient): + assert first_client.get_first() == FirstClientResult(name="first") + +def test_get_second(second_client: ClientNamespaceSecondClient): + assert second_client.get_second() == SecondClientResult(type=SecondClientEnumType.SECOND) From 657176dc3e7b07a347decc9c656aec2fb277798c Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 10 Dec 2024 14:28:30 +0800 Subject: [PATCH 37/49] review --- .../generator/pygen/codegen/serializers/__init__.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 8324840545..5aa2845593 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -442,7 +442,7 @@ def _name_space(self) -> str: return self._namespace_from_package_name @property - def _exec_path_implimentation(self) -> Path: + def exec_path_compensation(self) -> Path: """Assume the process is running in the root folder of the package. If not, we need the path implementation.""" return ( Path("../" * (self._name_space().count(".") + 1)) @@ -451,11 +451,7 @@ def _exec_path_implimentation(self) -> Path: ) def exec_path(self, namespace: str) -> Path: - return self._exec_path_implimentation / Path(*namespace.split(".")) - - # find root folder where "setup.py" is - def _package_root_folder(self, namespace_path: Path) -> Path: - return namespace_path / Path("../" * (self._name_space().count(".") + 1)) + return self.exec_path_compensation / Path(*namespace.split(".")) @property def _additional_folder(self) -> Path: From 29c30fd62c66faacf7df5cf39e5efcaae6e3cd25 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 10 Dec 2024 14:29:56 +0800 Subject: [PATCH 38/49] review --- packages/http-client-python/emitter/src/external-process.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/http-client-python/emitter/src/external-process.ts b/packages/http-client-python/emitter/src/external-process.ts index 22905d8d24..1e3ca74fb6 100644 --- a/packages/http-client-python/emitter/src/external-process.ts +++ b/packages/http-client-python/emitter/src/external-process.ts @@ -22,10 +22,6 @@ export async function saveCodeModelAsYaml(name: string, codemodel: unknown): Pro const filename = createTempPath(".yaml", name); const yamlStr = jsyaml.dump(codemodel); await writeFile(filename, yamlStr); - await writeFile( - joinPaths("C:/dev/typespec/packages/http-client-python", "alpha", "output.yaml"), - yamlStr, - ); return filename; } From af457caa7d403da71f4f507b3f125d5b483f9b9e Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 10 Dec 2024 14:30:43 +0800 Subject: [PATCH 39/49] review --- .../asynctests/test_client_namespace_async.py | 40 +++++++++---------- .../mock_api_tests/test_client_namespace.py | 36 ++++++++--------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py b/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py index 3a70b81ba8..9274387d32 100644 --- a/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py @@ -3,30 +3,30 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import pytest -from client.clientnamespace.aio import ClientNamespaceFirstClient -from client.clientnamespace.first.models import FirstClientResult +# import pytest +# from client.clientnamespace.aio import ClientNamespaceFirstClient +# from client.clientnamespace.first.models import FirstClientResult -from client.clientnamespace.second.aio import ClientNamespaceSecondClient -from client.clientnamespace.second.models import SecondClientResult -from client.clientnamespace.second.sub.models import SecondClientEnumType +# from client.clientnamespace.second.aio import ClientNamespaceSecondClient +# from client.clientnamespace.second.models import SecondClientResult +# from client.clientnamespace.second.sub.models import SecondClientEnumType -@pytest.fixture -async def first_client(): - async with ClientNamespaceFirstClient() as client: - yield client +# @pytest.fixture +# async def first_client(): +# async with ClientNamespaceFirstClient() as client: +# yield client -@pytest.fixture -async def second_client(): - async with ClientNamespaceSecondClient() as client: - yield client +# @pytest.fixture +# async def second_client(): +# async with ClientNamespaceSecondClient() as client: +# yield client -@pytest.mark.asyncio -async def test_get_first(first_client: ClientNamespaceFirstClient): - assert await first_client.get_first() == FirstClientResult(name="first") +# @pytest.mark.asyncio +# async def test_get_first(first_client: ClientNamespaceFirstClient): +# assert await first_client.get_first() == FirstClientResult(name="first") -@pytest.mark.asyncio -async def test_get_second(second_client: ClientNamespaceSecondClient): - assert await second_client.get_second() == SecondClientResult(type=SecondClientEnumType.SECOND) +# @pytest.mark.asyncio +# async def test_get_second(second_client: ClientNamespaceSecondClient): +# assert await second_client.get_second() == SecondClientResult(type=SecondClientEnumType.SECOND) diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py b/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py index b629c26240..265c6cf2b6 100644 --- a/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py @@ -3,28 +3,28 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import pytest -from client.clientnamespace import ClientNamespaceFirstClient -from client.clientnamespace.first.models import FirstClientResult +# import pytest +# from client.clientnamespace import ClientNamespaceFirstClient +# from client.clientnamespace.first.models import FirstClientResult -from client.clientnamespace.second import ClientNamespaceSecondClient -from client.clientnamespace.second.models import SecondClientResult -from client.clientnamespace.second.sub.models import SecondClientEnumType +# from client.clientnamespace.second import ClientNamespaceSecondClient +# from client.clientnamespace.second.models import SecondClientResult +# from client.clientnamespace.second.sub.models import SecondClientEnumType -@pytest.fixture -def first_client(): - with ClientNamespaceFirstClient() as client: - yield client +# @pytest.fixture +# def first_client(): +# with ClientNamespaceFirstClient() as client: +# yield client -@pytest.fixture -def second_client(): - with ClientNamespaceSecondClient() as client: - yield client +# @pytest.fixture +# def second_client(): +# with ClientNamespaceSecondClient() as client: +# yield client -def test_get_first(first_client: ClientNamespaceFirstClient): - assert first_client.get_first() == FirstClientResult(name="first") +# def test_get_first(first_client: ClientNamespaceFirstClient): +# assert first_client.get_first() == FirstClientResult(name="first") -def test_get_second(second_client: ClientNamespaceSecondClient): - assert second_client.get_second() == SecondClientResult(type=SecondClientEnumType.SECOND) +# def test_get_second(second_client: ClientNamespaceSecondClient): +# assert second_client.get_second() == SecondClientResult(type=SecondClientEnumType.SECOND) From b58ef98db7a1d3ae9a887f596aa1a13d70a09a0d Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 10 Dec 2024 16:18:26 +0800 Subject: [PATCH 40/49] review --- .../eng/scripts/ci/regenerate.ts | 2 +- .../generator/pygen/codegen/models/client.py | 2 +- .../pygen/codegen/models/code_model.py | 31 +++++++---- .../pygen/codegen/models/enum_type.py | 8 +-- .../pygen/codegen/models/model_type.py | 6 +- .../pygen/codegen/models/operation.py | 2 +- .../pygen/codegen/models/operation_group.py | 4 +- .../pygen/codegen/serializers/__init__.py | 10 ++-- .../codegen/serializers/builder_serializer.py | 27 ++++++--- .../codegen/serializers/client_serializer.py | 2 +- .../codegen/serializers/general_serializer.py | 8 +-- .../serializers/metadata_serializer.py | 8 +-- .../serializers/model_init_serializer.py | 2 +- .../codegen/serializers/model_serializer.py | 6 +- .../operation_groups_serializer.py | 7 ++- .../request_builders_serializer.py | 2 +- .../codegen/serializers/sample_serializer.py | 1 - .../codegen/serializers/test_serializer.py | 55 ++++++++----------- .../templates/operation_group.py.jinja2 | 2 +- .../generator/test/azure/requirements.txt | 2 +- 20 files changed, 96 insertions(+), 91 deletions(-) diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index 80cf60899b..8a9e9b8dbd 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -123,7 +123,7 @@ const EMITTER_OPTIONS: Record | Record FileImport: serialize_namespace, og.client_namespace, async_mode=async_mode, - namespace_type=NamespaceType.OPERATION, + imported_namespace_type=NamespaceType.OPERATION, ), og.class_name, ImportType.LOCAL, diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 0d715cb91d..329d6177cc 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -111,7 +111,7 @@ def get_relative_import_path( serialize_namespace: str, imported_namespace: Optional[str] = None, *, - namespace_type: NamespaceType = NamespaceType.CLIENT, + imported_namespace_type: NamespaceType = NamespaceType.CLIENT, async_mode: bool = False, module_name: Optional[str] = None, ) -> str: @@ -119,9 +119,9 @@ def get_relative_import_path( imported_namespace = self.namespace else: async_namespace = ".aio" if async_mode else "" - if namespace_type == NamespaceType.MODEL: + if imported_namespace_type == NamespaceType.MODEL: module_namespace = ".models" - elif namespace_type == NamespaceType.OPERATION: + elif imported_namespace_type == NamespaceType.OPERATION: module_namespace = f".{self.operations_folder_name(imported_namespace)}" else: module_namespace = "" @@ -152,7 +152,7 @@ def get_unique_models_alias(self, serialize_namespace: str, imported_namespace: if not self.need_unique_model_alias: return "_models" relative_path = self.get_relative_import_path( - serialize_namespace, imported_namespace, namespace_type=NamespaceType.MODEL + serialize_namespace, imported_namespace, imported_namespace_type=NamespaceType.MODEL ) dot_num = max(relative_path.count(".") - 1, 0) parts = [""] + [p for p in relative_path.split(".") if p] @@ -161,6 +161,7 @@ def get_unique_models_alias(self, serialize_namespace: str, imported_namespace: @property def client_namespace_types(self) -> Dict[str, ClientNamespaceType]: if not self._client_namespace_types: + # calculate client namespace types for each kind of client namespace for client in self.clients: if client.client_namespace not in self._client_namespace_types: self._client_namespace_types[client.client_namespace] = ClientNamespaceType() @@ -178,6 +179,7 @@ def client_namespace_types(self) -> Dict[str, ClientNamespaceType]: self._client_namespace_types[operation_group.client_namespace] = ClientNamespaceType() self._client_namespace_types[operation_group.client_namespace].operation_groups.append(operation_group) + # here we can check and record whether there are multi kinds of client namespace if len(self._client_namespace_types.keys()) > 1: self.has_subnamespace = True @@ -229,9 +231,11 @@ def client_filename(self) -> str: return self.clients[0].filename def get_clients(self, client_namespace: str) -> List[Client]: + """ Get all clients in specific namespace """ return self.client_namespace_types.get(client_namespace, ClientNamespaceType()).clients def is_top_namespace(self, client_namespace: str) -> bool: + """Whether the namespace is the top namespace. For a package named 'azure-mgmt-service', 'azure.mgmt.service' is the top namespace""" return client_namespace == self.namespace def need_vendored_code(self, async_mode: bool, client_namespace: str) -> bool: @@ -248,9 +252,9 @@ def need_vendored_abstract(self, client_namespace: str) -> bool: return self.is_top_namespace(client_namespace) and self.has_abstract_operations def need_vendored_mixin(self, client_namespace: str) -> bool: - return self.has_mixin_abc(client_namespace) + return self.has_mixin(client_namespace) - def has_mixin_abc(self, client_namespace: str) -> bool: + def has_mixin(self, client_namespace: str) -> bool: return any(c for c in self.get_clients(client_namespace) if c.has_mixin) @property @@ -269,11 +273,12 @@ def operations_folder_name(self, client_namespace: str) -> str: return self._operations_folder_name[client_namespace] def get_serialize_namespace( - self, client_namespace: str, async_mode: bool = False, namespace_type: NamespaceType = NamespaceType.CLIENT + self, client_namespace: str, async_mode: bool = False, client_namespace_type: NamespaceType = NamespaceType.CLIENT ) -> str: - if namespace_type == NamespaceType.CLIENT: + """calculate the namespace for serialization from client namespace""" + if client_namespace_type == NamespaceType.CLIENT: return client_namespace + (".aio" if async_mode else "") - if namespace_type == NamespaceType.MODEL: + if client_namespace_type == NamespaceType.MODEL: return client_namespace + ".models" operations_folder_name = self.operations_folder_name(client_namespace) @@ -309,10 +314,14 @@ def model_types(self) -> List[ModelType]: def model_types(self, val: List[ModelType]) -> None: self._model_types = val - def public_model_types(self, models: Optional[ModelType] = None) -> List[ModelType]: - models = self.model_types if models is None else models + @staticmethod + def get_public_model_types(models: List[ModelType]) -> List[ModelType]: return [m for m in models if not m.internal and not m.base == "json"] + @property + def public_model_types(self) -> List[ModelType]: + return self.get_public_model_types(self.model_types) + @property def enums(self) -> List[EnumType]: """All of the enums""" diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 826f7d51ed..f3879a003e 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -81,7 +81,7 @@ def imports(self, **kwargs: Any) -> FileImport: self.code_model.get_relative_import_path( serialize_namespace, self.enum_type.client_namespace, - namespace_type=NamespaceType.MODEL, + imported_namespace_type=NamespaceType.MODEL, module_name=self.code_model.enums_filename, ), self.enum_type.name, @@ -234,8 +234,8 @@ def imports(self, **kwargs: Any) -> FileImport: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) - namespace_type = kwargs.get("namespace_type") - if namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: + serialize_namespace_type = kwargs.get("serialize_namespace_type") + if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: file_import.add_submodule_import( relative_path, "models", @@ -243,7 +243,7 @@ def imports(self, **kwargs: Any) -> FileImport: alias=alias, typing_section=TypingSection.REGULAR, ) - elif namespace_type == NamespaceType.MODEL: + elif serialize_namespace_type == NamespaceType.MODEL: file_import.add_submodule_import( relative_path, "models", diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 3c967ee0cf..7e52c87a9a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -304,9 +304,9 @@ def imports(self, **kwargs: Any) -> FileImport: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) - namespace_type = kwargs.get("namespace_type") + serialize_namespace_type = kwargs.get("serialize_namespace_type") # add import for models in operations or _types file - if namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: + if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: file_import.add_submodule_import( relative_path, "models", @@ -321,7 +321,7 @@ def imports(self, **kwargs: Any) -> FileImport: ImportType.LOCAL, typing_section=TypingSection.REGULAR, ) - elif namespace_type == NamespaceType.MODEL: + elif serialize_namespace_type == NamespaceType.MODEL: file_import.add_submodule_import( relative_path, "models", diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index a32c963214..6316346f8f 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -306,7 +306,7 @@ def get_request_builder_import( self.code_model.get_relative_import_path( serialize_namespace, self.client_namespace, - namespace_type=NamespaceType.OPERATION, + imported_namespace_type=NamespaceType.OPERATION, module_name=self.filename, ), request_builder.name, diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 12a4b8908a..790e1f04d3 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -110,7 +110,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: self.code_model.get_relative_import_path( serialize_namespace, self.client_namespace, - namespace_type=NamespaceType.OPERATION, + imported_namespace_type=NamespaceType.OPERATION, async_mode=async_mode, ), og.class_name, @@ -118,7 +118,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ) # for multiapi if ( - (self.code_model.public_model_types()) + (self.code_model.public_model_types) and self.code_model.options["models_mode"] == "msrest" and not self.is_mixin ): diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 5aa2845593..2e508e1ef4 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -501,9 +501,9 @@ def _serialize_and_write_test(self, env: Environment, namespace: Path): general_serializer = TestGeneralSerializer(code_model=self.code_model, env=env) self.write_file(out_path / "conftest.py", general_serializer.serialize_conftest()) if not self.code_model.options["azure_arm"]: - for is_async in (True, False): - async_suffix = "_async" if is_async else "" - general_serializer.is_async = is_async + for async_mode in (True, False): + async_suffix = "_async" if async_mode else "" + general_serializer.async_mode = async_mode self.write_file( out_path / f"testpreparer{async_suffix}.py", general_serializer.serialize_testpreparer(), @@ -516,9 +516,9 @@ def _serialize_and_write_test(self, env: Environment, namespace: Path): ): continue test_serializer = TestSerializer(self.code_model, env, client=client, operation_group=og) - for is_async in (True, False): + for async_mode in (True, False): try: - test_serializer.is_async = is_async + test_serializer.async_mode = async_mode self.write_file( out_path / f"{to_snake_case(test_serializer.test_class_name)}.py", test_serializer.serialize_test(), diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 181383b075..b30e808109 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -32,6 +32,7 @@ ParameterListType, ByteArraySchema, ) +from ..models.utils import NamespaceType from .parameter_serializer import ParameterSerializer, PopKwargType from ..models.parameter_list import ParameterType from . import utils @@ -188,12 +189,20 @@ def is_json_model_type(parameters: ParameterListType) -> bool: class _BuilderBaseSerializer(Generic[BuilderType]): - def __init__(self, code_model: CodeModel, async_mode: bool, serialize_namespace: Optional[str] = None) -> None: + def __init__(self, code_model: CodeModel, async_mode: bool, client_namespace: str) -> None: self.code_model = code_model self.async_mode = async_mode - self.serialize_namespace = code_model.namespace if serialize_namespace is None else serialize_namespace + self.client_namespace = client_namespace self.parameter_serializer = ParameterSerializer(self.serialize_namespace) + @property + def serialize_namespace(self) -> str: + return self.code_model.get_serialize_namespace( + self.client_namespace, + async_mode=self.async_mode, + client_namespace_type=NamespaceType.OPERATION, + ) + @property @abstractmethod def _need_self_param(self) -> bool: ... @@ -362,6 +371,7 @@ def pipeline_name(self) -> str: class RequestBuilderSerializer(_BuilderBaseSerializer[RequestBuilderType]): + def description_and_summary(self, builder: RequestBuilderType) -> List[str]: retval = super().description_and_summary(builder) retval += [ @@ -1196,10 +1206,10 @@ class OperationSerializer(_OperationSerializer[Operation]): ... class _PagingOperationSerializer(_OperationSerializer[PagingOperationType]): - def __init__(self, code_model: CodeModel, async_mode: bool, serialize_namespace: Optional[str] = None) -> None: + def __init__(self, code_model: CodeModel, async_mode: bool, client_namespace: str) -> None: # for pylint reasons need to redefine init # probably because inheritance is going too deep - super().__init__(code_model, async_mode, serialize_namespace) + super().__init__(code_model, async_mode, client_namespace) # self.code_model = code_model # self.async_mode = async_mode # self.parameter_serializer = ParameterSerializer(self.serialize_namespace) @@ -1361,10 +1371,10 @@ class PagingOperationSerializer(_PagingOperationSerializer[PagingOperation]): .. class _LROOperationSerializer(_OperationSerializer[LROOperationType]): - def __init__(self, code_model: CodeModel, async_mode: bool, serialize_namespace: Optional[str] = None) -> None: + def __init__(self, code_model: CodeModel, async_mode: bool, client_namespace: str) -> None: # for pylint reasons need to redefine init # probably because inheritance is going too deep - super().__init__(code_model, async_mode, serialize_namespace) + super().__init__(code_model, async_mode, client_namespace) # self.code_model = code_model # self.async_mode = async_mode # self.parameter_serializer = ParameterSerializer() @@ -1510,7 +1520,7 @@ def get_operation_serializer( builder: Operation, code_model, async_mode: bool, - serialize_namespace: Optional[str] = None, + client_namespace: str, ) -> Union[ OperationSerializer, PagingOperationSerializer, @@ -1529,5 +1539,4 @@ def get_operation_serializer( ret_cls = LROOperationSerializer elif builder.operation_type == "paging": ret_cls = PagingOperationSerializer - serialize_namespace = code_model.namespace if serialize_namespace is None else serialize_namespace - return ret_cls(code_model, async_mode, serialize_namespace) + return ret_cls(code_model, async_mode, client_namespace) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py index f7f7232935..22540a1a85 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py @@ -136,7 +136,7 @@ def _get_client_models_value(models_dict_name: str) -> str: is_msrest_model = self.client.code_model.options["models_mode"] == "msrest" if is_msrest_model: add_private_models = len(self.client.code_model.model_types) != len( - self.client.code_model.public_model_types() + self.client.code_model.public_model_types ) model_dict_name = f"_models.{self.client.code_model.models_filename}" if add_private_models else "_models" retval.append( diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index fc43ceb5e6..2e3f2e5032 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -72,7 +72,7 @@ def serialize_service_client_file(self, clients: List[Client]) -> str: for client in clients: imports.merge( client.imports( - self.async_mode, serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.CLIENT + self.async_mode, serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.CLIENT ) ) @@ -159,7 +159,7 @@ def serialize_config_file(self, clients: List[Client]) -> str: for client in self.code_model.clients: imports.merge( client.config.imports( - self.async_mode, serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.CLIENT + self.async_mode, serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.CLIENT ) ) return template.render( @@ -191,8 +191,8 @@ def serialize_validation_file(self) -> str: def serialize_cross_language_definition_file(self) -> str: cross_langauge_def_dict = { - f"{self.code_model.namespace}.models.{model.name}": model.cross_language_definition_id - for model in self.code_model.public_model_types() + f"{model.client_namespace}.models.{model.name}": model.cross_language_definition_id + for model in self.code_model.public_model_types } cross_langauge_def_dict.update( { diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py index 0ae74b2db1..19fd28e2c0 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py @@ -205,17 +205,13 @@ def _is_paging(operation): get_operation_serializer, code_model=self.client.code_model, async_mode=True, - serialize_namespace=self.code_model.get_serialize_namespace( - self.client_namespace, async_mode=True, namespace_type=NamespaceType.OPERATION - ), + client_namespace=self.client_namespace, ), get_sync_operation_serializer=functools.partial( get_operation_serializer, code_model=self.client.code_model, async_mode=False, - serialize_namespace=self.code_model.get_serialize_namespace( - self.client_namespace, async_mode=False, namespace_type=NamespaceType.OPERATION - ), + client_namespace=self.client_namespace, ), has_credential=bool(self.client.credential), ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py index 1da2ce7e27..2b6a664798 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_init_serializer.py @@ -18,7 +18,7 @@ def __init__( self.enums = enums def serialize(self) -> str: - schemas = [s.name for s in self.code_model.public_model_types(self.models)] + schemas = [s.name for s in self.code_model.get_public_model_types(self.models)] schemas.sort() enums = [e.name for e in self.enums if not e.internal] if self.enums else None diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 36ff9c20c1..d147b1c369 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -136,7 +136,7 @@ def global_pylint_disables(self) -> str: @property def serialize_namespace(self) -> str: - return self.code_model.get_serialize_namespace(self.client_namespace, namespace_type=NamespaceType.MODEL) + return self.code_model.get_serialize_namespace(self.client_namespace, client_namespace_type=NamespaceType.MODEL) class MsrestModelSerializer(_ModelSerializer): @@ -150,7 +150,7 @@ def imports(self) -> FileImport: for model in self.models: file_import.merge(model.imports(is_operation_file=False)) for param in self._init_line_parameters(model): - file_import.merge(param.imports(serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.MODEL)) + file_import.merge(param.imports(serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL)) return file_import @@ -233,7 +233,7 @@ def imports(self) -> FileImport: file_import.merge(model.imports(is_operation_file=False, serialize_namespace=self.serialize_namespace)) for prop in model.properties: file_import.merge( - prop.imports(serialize_namespace=self.serialize_namespace, namespace_type=NamespaceType.MODEL) + prop.imports(serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL) ) if model.is_polymorphic: file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py index 78ef76c7cf..c95236178f 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -53,7 +53,7 @@ def _get_request_builders( @property def serialize_namespace(self) -> str: return self.code_model.get_serialize_namespace( - self.client_namespace, async_mode=self.async_mode, namespace_type=NamespaceType.OPERATION + self.client_namespace, async_mode=self.async_mode, client_namespace_type=NamespaceType.OPERATION ) def serialize(self) -> str: @@ -63,7 +63,7 @@ def serialize(self) -> str: operation_group.imports( async_mode=self.async_mode, serialize_namespace=self.serialize_namespace, - namespace_type=NamespaceType.OPERATION, + serialize_namespace_type=NamespaceType.OPERATION, ) ) @@ -81,11 +81,12 @@ def serialize(self) -> str: get_operation_serializer, code_model=self.code_model, async_mode=self.async_mode, - serialize_namespace=self.serialize_namespace, + client_namespace=self.client_namespace, ), request_builder_serializer=RequestBuilderSerializer( self.code_model, async_mode=False, + client_namespace=self.client_namespace, ), get_request_builders=self._get_request_builders, ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py index 2a9f56f3e5..aef7c68614 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py @@ -44,7 +44,7 @@ def serialize_init(self) -> str: @property def serialize_namespace(self) -> str: - return self.code_model.get_serialize_namespace(self.client_namespace, namespace_type=NamespaceType.OPERATION) + return self.code_model.get_serialize_namespace(self.client_namespace, client_namespace_type=NamespaceType.OPERATION) def serialize_request_builders(self) -> str: template = self.env.get_template("request_builders.py.jinja2") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py index 30e05da652..112a0a2e1a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -20,7 +20,6 @@ BodyParameter, FileImport, ) -from .utils import get_namespace_config, get_namespace_from_package_name from .._utils import get_parent_namespace _LOGGER = logging.getLogger(__name__) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index bfbfd82d71..3925f34117 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -35,18 +35,18 @@ def is_common_operation(operation_type: str) -> bool: class TestName: - def __init__(self, code_model: CodeModel, client_name: str, *, is_async: bool = False) -> None: + def __init__(self, code_model: CodeModel, client_name: str, *, async_mode: bool = False) -> None: self.code_model = code_model self.client_name = client_name - self.is_async = is_async + self.async_mode = async_mode @property def async_suffix_capt(self) -> str: - return "Async" if self.is_async else "" + return "Async" if self.async_mode else "" @property def create_client_name(self) -> str: - return "create_async_client" if self.is_async else "create_client" + return "create_async_client" if self.async_mode else "create_client" @property def prefix(self) -> str: @@ -72,12 +72,12 @@ def __init__( params: Dict[str, Any], operation: OperationType, *, - is_async: bool = False, + async_mode: bool = False, ) -> None: self.operation_groups = operation_groups self.params = params self.operation = operation - self.is_async = is_async + self.async_mode = async_mode @property def name(self) -> str: @@ -93,7 +93,7 @@ def operation_group_prefix(self) -> str: @property def response(self) -> str: - if self.is_async: + if self.async_mode: if is_lro(self.operation.operation_type): return "response = await (await " if is_common_operation(self.operation.operation_type): @@ -107,14 +107,14 @@ def lro_comment(self) -> str: @property def operation_suffix(self) -> str: if is_lro(self.operation.operation_type): - extra = ")" if self.is_async else "" + extra = ")" if self.async_mode else "" return f"{extra}.result(){self.lro_comment}" return "" @property def extra_operation(self) -> str: if is_paging(self.operation.operation_type): - async_str = "async " if self.is_async else "" + async_str = "async " if self.async_mode else "" return f"result = [r {async_str}for r in response]" return "" @@ -128,32 +128,23 @@ def __init__( testcases: List[TestCase], test_class_name: str, *, - is_async: bool = False, + async_mode: bool = False, ) -> None: - super().__init__(code_model, client_name, is_async=is_async) + super().__init__(code_model, client_name, async_mode=async_mode) self.operation_group = operation_group self.testcases = testcases self.test_class_name = test_class_name class TestGeneralSerializer(BaseSerializer): - def __init__( - self, - code_model: CodeModel, - env: Environment, - *, - is_async: bool = False, - ) -> None: - super().__init__(code_model, env, is_async) - self.is_async = is_async @property def aio_str(self) -> str: - return ".aio" if self.is_async else "" + return ".aio" if self.async_mode else "" @property def test_names(self) -> List[TestName]: - return [TestName(self.code_model, c.name, is_async=self.is_async) for c in self.code_model.clients] + return [TestName(self.code_model, c.name, async_mode=self.async_mode) for c in self.code_model.clients] def add_import_client(self, imports: FileImport) -> None: for client in self.code_model.clients: @@ -164,12 +155,12 @@ def import_clients(self) -> FileImportSerializer: imports = self.init_file_import() imports.add_submodule_import("devtools_testutils", "AzureRecordedTestCase", ImportType.STDLIB) - if not self.is_async: + if not self.async_mode: imports.add_import("functools", ImportType.STDLIB) imports.add_submodule_import("devtools_testutils", "PowerShellPreparer", ImportType.STDLIB) self.add_import_client(imports) - return FileImportSerializer(imports, self.is_async) + return FileImportSerializer(imports, self.async_mode) def serialize_conftest(self) -> str: return self.env.get_template("conftest.py.jinja2").render( @@ -193,17 +184,17 @@ def __init__( *, client: Client, operation_group: OperationGroup, - is_async: bool = False, + async_mode: bool = False, ) -> None: - super().__init__(code_model, env, is_async=is_async) + super().__init__(code_model, env, async_mode=async_mode) self.client = client self.operation_group = operation_group @property def import_test(self) -> FileImportSerializer: imports = self.init_file_import() - test_name = TestName(self.code_model, self.client.name, is_async=self.is_async) - async_suffix = "_async" if self.is_async else "" + test_name = TestName(self.code_model, self.client.name, async_mode=self.async_mode) + async_suffix = "_async" if self.async_mode else "" imports.add_submodule_import( "devtools_testutils" if self.code_model.options["azure_arm"] else "testpreparer" + async_suffix, test_name.base_test_class_name, @@ -221,7 +212,7 @@ def import_test(self) -> FileImportSerializer: ) if self.code_model.options["azure_arm"]: self.add_import_client(imports) - return FileImportSerializer(imports, self.is_async) + return FileImportSerializer(imports, self.async_mode) @property def breadth_search_operation_group(self) -> List[List[OperationGroup]]: @@ -268,7 +259,7 @@ def get_test(self) -> Test: operation_groups=operation_groups, params=operation_params, operation=operation, - is_async=self.is_async, + async_mode=self.async_mode, ) testcases.append(testcase) if not testcases: @@ -280,12 +271,12 @@ def get_test(self) -> Test: operation_group=self.operation_group, testcases=testcases, test_class_name=self.test_class_name, - is_async=self.is_async, + async_mode=self.async_mode, ) @property def test_class_name(self) -> str: - test_name = TestName(self.code_model, self.client.name, is_async=self.is_async) + test_name = TestName(self.code_model, self.client.name, async_mode=self.async_mode) class_name = "" if self.operation_group.is_mixin else self.operation_group.class_name return f"Test{test_name.prefix}{class_name}{test_name.async_suffix_capt}" diff --git a/packages/http-client-python/generator/pygen/codegen/templates/operation_group.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/operation_group.py.jinja2 index a894b95e75..1ee2846535 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/operation_group.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/operation_group.py.jinja2 @@ -25,7 +25,7 @@ class {{ operation_group.class_name }}: {{ operation_group.pylint_disable() }} :attr:`{{ operation_group.property_name }}` attribute. """ -{% if code_model.public_model_types() and code_model.options["models_mode"] == "msrest" %} +{% if code_model.public_model_types and code_model.options["models_mode"] == "msrest" %} models = _models {% endif %} diff --git a/packages/http-client-python/generator/test/azure/requirements.txt b/packages/http-client-python/generator/test/azure/requirements.txt index 22b8c430b5..d78a160aca 100644 --- a/packages/http-client-python/generator/test/azure/requirements.txt +++ b/packages/http-client-python/generator/test/azure/requirements.txt @@ -24,7 +24,7 @@ mypy==1.13.0 -e ./generated/azure-example-basic -e ./generated/azure-resource-manager-common-properties -e ./generated/azure-resource-manager-resources --e ./generated/client-namespace +# -e ./generated/client-namespace # common test case -e ./generated/authentication-api-key From d5b473aab33def2b1cfe1da5f245c9dc58a000bb Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 11 Dec 2024 10:03:41 +0800 Subject: [PATCH 41/49] fix --- .../generator/pygen/codegen/templates/test.py.jinja2 | 6 +++--- .../pygen/codegen/templates/testpreparer.py.jinja2 | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 index 25f8da7b53..7d5dded167 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 @@ -1,7 +1,7 @@ {% set prefix_lower = test.prefix|lower %} {% set client_var = "self.client" if code_model.options["azure_arm"] else "client" %} -{% set async = "async " if test.is_async else "" %} -{% set async_suffix = "_async" if test.is_async else "" %} +{% set async = "async " if test.async_mode else "" %} +{% set async_suffix = "_async" if test.async_mode else "" %} # coding=utf-8 {{ code_model.options['license_header'] }} import pytest @@ -15,7 +15,7 @@ AZURE_LOCATION = "eastus" class {{ test.test_class_name }}({{ test.base_test_class_name }}): {% if code_model.options["azure_arm"] %} def setup_method(self, method): - {% if test.is_async %} + {% if test.async_mode %} self.client = self.create_mgmt_client({{ test.client_name }}, is_async=True) {% else %} self.client = self.create_mgmt_client({{ test.client_name }}) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/testpreparer.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/testpreparer.py.jinja2 index b3b15f3727..930d9825fd 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/testpreparer.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/testpreparer.py.jinja2 @@ -3,7 +3,7 @@ {{ imports }} {% for test_name in test_names %} -{% set extra_async = ", is_async=True" if test_name.is_async else ""%} +{% set extra_async = ", is_async=True" if test_name.async_mode else ""%} {% set prefix_lower = test_name.prefix|lower %} class {{ test_name.base_test_class_name }}(AzureRecordedTestCase): @@ -15,7 +15,7 @@ class {{ test_name.base_test_class_name }}(AzureRecordedTestCase): endpoint=endpoint, ) -{% if not test_name.is_async %} +{% if not test_name.async_mode %} {{ test_name.preparer_name }} = functools.partial( PowerShellPreparer, "{{ prefix_lower }}", From 689a06b23d51bf1d213724bfd0299f513c2a16ba Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 11 Dec 2024 10:40:32 +0800 Subject: [PATCH 42/49] fix --- .../generator/pygen/codegen/_utils.py | 1 + .../generator/pygen/codegen/models/client.py | 2 - .../pygen/codegen/models/code_model.py | 39 +++++++++++----- .../pygen/codegen/models/model_type.py | 4 +- .../pygen/codegen/models/operation.py | 2 +- .../pygen/codegen/models/operation_group.py | 4 +- .../pygen/codegen/models/parameter.py | 2 +- .../pygen/codegen/serializers/__init__.py | 23 +++++----- .../codegen/serializers/builder_serializer.py | 45 ++++++++----------- .../codegen/serializers/general_serializer.py | 12 ++--- .../serializers/metadata_serializer.py | 1 - .../codegen/serializers/model_serializer.py | 16 +++++-- .../serializers/operations_init_serializer.py | 1 - .../request_builders_serializer.py | 14 +++--- .../codegen/serializers/sample_serializer.py | 8 +++- .../asynctests/test_client_namespace_async.py | 1 - .../mock_api_tests/test_client_namespace.py | 1 - 17 files changed, 99 insertions(+), 77 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/_utils.py b/packages/http-client-python/generator/pygen/codegen/_utils.py index a89af22c47..6ea066031f 100644 --- a/packages/http-client-python/generator/pygen/codegen/_utils.py +++ b/packages/http-client-python/generator/pygen/codegen/_utils.py @@ -16,5 +16,6 @@ VALID_PACKAGE_MODE = SWAGGER_PACKAGE_MODE + TYPESPEC_PACKAGE_MODE NAME_LENGTH_LIMIT = 40 + def get_parent_namespace(namespace: str) -> str: return namespace.rsplit(".", 1)[0] diff --git a/packages/http-client-python/generator/pygen/codegen/models/client.py b/packages/http-client-python/generator/pygen/codegen/models/client.py index dcdf92e0ba..f6a4fc0cd2 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/client.py +++ b/packages/http-client-python/generator/pygen/codegen/models/client.py @@ -15,8 +15,6 @@ OverloadedRequestBuilder, get_request_builder, ) -from .model_type import ModelType -from .enum_type import EnumType from .parameter import Parameter, ParameterMethodLocation from .lro_operation import LROOperation from .lro_paging_operation import LROPagingOperation diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 329d6177cc..31f26baf20 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import List, Dict, Any, Set, Union, Literal, Optional +from typing import List, Dict, Any, Set, Union, Literal, Optional, cast from .base import BaseType from .enum_type import EnumType @@ -131,7 +131,7 @@ def get_relative_import_path( if key not in self._relative_import_path: idx = 0 serialize_namespace_split = serialize_namespace.split(".") - imported_namespace_split = imported_namespace.split(".") + imported_namespace_split = cast(str, imported_namespace).split(".") while idx < min(len(serialize_namespace_split), len(imported_namespace_split)): if serialize_namespace_split[idx] != imported_namespace_split[idx]: break @@ -183,7 +183,7 @@ def client_namespace_types(self) -> Dict[str, ClientNamespaceType]: if len(self._client_namespace_types.keys()) > 1: self.has_subnamespace = True - # insert namespace to make sure it is continuous(e.g. ("", "azure", "azure.mgmt", "azure.mgmt.service", ...)) + # insert namespace to make sure it is continuous(e.g. ("", "azure", "azure.mgmt", "azure.mgmt.service")) longest_namespace = sorted(self._client_namespace_types.keys())[-1] namespace_parts = longest_namespace.split(".") for idx in range(len(namespace_parts) + 1): @@ -231,26 +231,38 @@ def client_filename(self) -> str: return self.clients[0].filename def get_clients(self, client_namespace: str) -> List[Client]: - """ Get all clients in specific namespace """ + """Get all clients in specific namespace""" return self.client_namespace_types.get(client_namespace, ClientNamespaceType()).clients def is_top_namespace(self, client_namespace: str) -> bool: - """Whether the namespace is the top namespace. For a package named 'azure-mgmt-service', 'azure.mgmt.service' is the top namespace""" + """Whether the namespace is the top namespace. For example, a package named 'azure-mgmt-service', + 'azure.mgmt.service' is the top namespace. + """ return client_namespace == self.namespace def need_vendored_code(self, async_mode: bool, client_namespace: str) -> bool: """Whether we need to vendor code in the _vendor.py in specific namespace""" - return self.need_vendored_form_data(async_mode, client_namespace) or self.need_vendored_etag(client_namespace) or self.need_vendored_abstract(client_namespace) or self.need_vendored_mixin(client_namespace) - + return ( + self.need_vendored_form_data(async_mode, client_namespace) + or self.need_vendored_etag(client_namespace) + or self.need_vendored_abstract(client_namespace) + or self.need_vendored_mixin(client_namespace) + ) + def need_vendored_form_data(self, async_mode: bool, client_namespace: str) -> bool: - return (not async_mode) and self.is_top_namespace(client_namespace) and self.has_form_data and self.options["models_mode"] == "dpg" - + return ( + (not async_mode) + and self.is_top_namespace(client_namespace) + and self.has_form_data + and self.options["models_mode"] == "dpg" + ) + def need_vendored_etag(self, client_namespace: str) -> bool: return self.is_top_namespace(client_namespace) and self.has_etag - + def need_vendored_abstract(self, client_namespace: str) -> bool: return self.is_top_namespace(client_namespace) and self.has_abstract_operations - + def need_vendored_mixin(self, client_namespace: str) -> bool: return self.has_mixin(client_namespace) @@ -273,7 +285,10 @@ def operations_folder_name(self, client_namespace: str) -> str: return self._operations_folder_name[client_namespace] def get_serialize_namespace( - self, client_namespace: str, async_mode: bool = False, client_namespace_type: NamespaceType = NamespaceType.CLIENT + self, + client_namespace: str, + async_mode: bool = False, + client_namespace_type: NamespaceType = NamespaceType.CLIENT, ) -> str: """calculate the namespace for serialization from client namespace""" if client_namespace_type == NamespaceType.CLIENT: diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 7e52c87a9a..f999abefbd 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -11,7 +11,6 @@ from .base import BaseType from .constant_type import ConstantType from .property import Property -from .enum_type import EnumType from .imports import FileImport, ImportType, TypingSection from ...utils import NAME_LENGTH_LIMIT @@ -290,7 +289,8 @@ def type_annotation(self, **kwargs: Any) -> str: return retval if is_operation_file or skip_quote else f'"{retval}"' def docstring_type(self, **kwargs: Any) -> str: - return f"~{self.code_model.namespace}.models.{self.type_annotation(need_model_alias=False, skip_quote=True, **kwargs)}" + type_annotation = self.type_annotation(need_model_alias=False, skip_quote=True, **kwargs) + return f"~{self.code_model.namespace}.models.{type_annotation}" def docstring_text(self, **kwargs: Any) -> str: return self.name diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 160ccd4a6b..c5e7815ca7 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -208,7 +208,7 @@ def non_default_errors(self) -> List[Response]: e for e in self.exceptions if "default" not in e.status_codes and e.type and isinstance(e.type, ModelType) ] - def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: # pylint: disable=unused-argument + def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 790e1f04d3..7e0bba7f53 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -138,7 +138,9 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ) if self.has_abstract_operations: file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, module_name="_vendor", async_mode=async_mode), + self.code_model.get_relative_import_path( + serialize_namespace, module_name="_vendor", async_mode=async_mode + ), "raise_if_not_implemented", ImportType.LOCAL, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter.py b/packages/http-client-python/generator/pygen/codegen/models/parameter.py index 9d7c20b55d..19ec0c380d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter.py @@ -158,7 +158,7 @@ def docstring_type(self, **kwargs: Any) -> str: def serialization_type(self, **kwargs: Any) -> str: return self.type.serialization_type(**kwargs) - def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: + def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: # pylint: disable=unused-argument file_import = FileImport(self.code_model) if self.optional and self.client_default_value is None: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 2e508e1ef4..2a76ac086c 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import logging from collections import namedtuple -from typing import List, Optional, Any, Union +from typing import List, Any, Union from pathlib import Path from jinja2 import PackageLoader, Environment, FileSystemLoader, StrictUndefined @@ -110,9 +110,7 @@ def serialize(self) -> None: if client_namespace == "": # Write the setup file if self.code_model.options["basic_setup_py"]: - self.write_file( - exec_path / Path("setup.py"), general_serializer.serialize_setup_file() - ) + self.write_file(exec_path / Path("setup.py"), general_serializer.serialize_setup_file()) # add packaging files in root namespace (e.g. setup.py, README.md, etc.) if self.code_model.options["package_mode"]: @@ -141,8 +139,9 @@ def serialize(self) -> None: general_serializer.serialize_pkgutil_init_file(), ) - # _model_base.py/_serialization.py/_vendor.py/py.typed/_types.py/_validation.py is always put in top level namespace - if client_namespace == self.code_model.namespace: + # _model_base.py/_serialization.py/_vendor.py/py.typed/_types.py/_validation.py + # is always put in top level namespace + if self.code_model.is_top_namespace(client_namespace): self._serialize_and_write_top_level_folder(env=env, namespace=client_namespace) # add models folder if there are models in this namespace @@ -168,8 +167,8 @@ def serialize(self) -> None: if self.code_model.options["multiapi"]: self._serialize_and_write_metadata(env=env, namespace=client_namespace) - def _serialize_and_write_package_files(self, namespace: str) -> None: - root_of_sdk = self.exec_path(namespace) + def _serialize_and_write_package_files(self, client_namespace: str) -> None: + root_of_sdk = self.exec_path(client_namespace) if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: env = Environment( loader=PackageLoader("pygen.codegen", "templates/packaging_templates"), @@ -383,7 +382,9 @@ def _serialize_and_write_vendor_file(self, env: Environment, namespace: str) -> if self.code_model.need_vendored_code(async_mode=async_mode, client_namespace=namespace): self.write_file( exec_path / Path(f"{async_path}_vendor.py"), - GeneralSerializer(code_model=self.code_model, env=env, async_mode=async_mode, client_namespace=namespace).serialize_vendor_file(), + GeneralSerializer( + code_model=self.code_model, env=env, async_mode=async_mode, client_namespace=namespace + ).serialize_vendor_file(), ) def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str) -> None: @@ -443,7 +444,7 @@ def _name_space(self) -> str: @property def exec_path_compensation(self) -> Path: - """Assume the process is running in the root folder of the package. If not, we need the path implementation.""" + """Assume the process is running in the root folder of the package. If not, we need the path compensation.""" return ( Path("../" * (self._name_space().count(".") + 1)) if self.code_model.options["no_namespace_folders"] @@ -495,7 +496,7 @@ def _serialize_and_write_sample(self, env: Environment, namespace: str): log_error = f"error happens in sample {file}: {e}" _LOGGER.error(log_error) - def _serialize_and_write_test(self, env: Environment, namespace: Path): + def _serialize_and_write_test(self, env: Environment, namespace: str): self.code_model.for_test = True out_path = self.exec_path(namespace) / Path("generated_tests") general_serializer = TestGeneralSerializer(code_model=self.code_model, env=env) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index dc83f6f127..520c6dbf11 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -247,7 +247,9 @@ def _method_signature(self, builder: BuilderType) -> str: def method_signature_and_response_type_annotation( self, builder: BuilderType, *, want_decorators: Optional[bool] = True ) -> str: - response_type_annotation = builder.response_type_annotation(async_mode=self.async_mode, serialize_namespace=self.serialize_namespace) + response_type_annotation = builder.response_type_annotation( + async_mode=self.async_mode, serialize_namespace=self.serialize_namespace + ) method_signature = self._method_signature(builder) decorators = self.decorators(builder) decorators_str = "" @@ -627,13 +629,18 @@ def pop_kwargs_from_signature(self, builder: OperationType) -> List[str]: for p in builder.parameters.parameters: if p.hide_in_operation_signature: kwargs.append(f'{p.client_name} = kwargs.pop("{p.client_name}", None)') - cls_annotation = builder.cls_type_annotation(async_mode=self.async_mode, serialize_namespace=self.serialize_namespace) + cls_annotation = builder.cls_type_annotation( + async_mode=self.async_mode, serialize_namespace=self.serialize_namespace + ) kwargs.append(f"cls: {cls_annotation} = kwargs.pop(\n 'cls', None\n)") return kwargs def response_docstring(self, builder: OperationType) -> List[str]: response_str = f":return: {builder.response_docstring_text(async_mode=self.async_mode)}" - rtype_str = f":rtype: {builder.response_docstring_type(async_mode=self.async_mode, serialize_namespace=self.serialize_namespace)}" + response_docstring_type = builder.response_docstring_type( + async_mode=self.async_mode, serialize_namespace=self.serialize_namespace + ) + rtype_str = f":rtype: {response_docstring_type}" return [ response_str, rtype_str, @@ -681,9 +688,10 @@ def _serialize_body_parameter(self, builder: OperationType) -> List[str]: if self.code_model.options["models_mode"] == "msrest": is_xml_cmd = _xml_config(send_xml, builder.parameters.body_parameter.content_types) serialization_ctxt_cmd = f", {ser_ctxt_name}={ser_ctxt_name}" if xml_serialization_ctxt else "" + serialization_type = body_param.type.serialization_type(serialize_namespace=self.serialize_namespace) create_body_call = ( f"_{body_kwarg_name} = self._serialize.body({body_param.client_name}, " - f"'{body_param.type.serialization_type(serialize_namespace=self.serialize_namespace)}'{is_xml_cmd}{serialization_ctxt_cmd})" + f"'{serialization_type}'{is_xml_cmd}{serialization_ctxt_cmd})" ) elif self.code_model.options["models_mode"] == "dpg": if json_serializable(body_param.default_content_type): @@ -914,7 +922,7 @@ def response_headers(self, response: Response) -> List[str]: retval: List[str] = [ ( f"response_headers['{response_header.wire_name}']=self._deserialize(" - f"'{response_header.serialization_type(serialize_namespace=self.serialize_namespace)}', response.headers.get('{response_header.wire_name}'))" + f"'{response_header.serialization_type(serialize_namespace=self.serialize_namespace)}', response.headers.get('{response_header.wire_name}'))" # pylint: disable=line-too-long ) for response_header in response.headers ] @@ -922,7 +930,7 @@ def response_headers(self, response: Response) -> List[str]: retval.append("") return retval - def response_deserialization( + def response_deserialization( # pylint: disable=too-many-statements self, builder: OperationType, response: Response, @@ -948,9 +956,8 @@ def response_deserialization( pylint_disable = " # pylint: disable=protected-access" if self.code_model.options["models_mode"] == "msrest": deserialize_code.append("deserialized = self._deserialize(") - deserialize_code.append( - f" '{response.serialization_type(serialize_namespace=self.serialize_namespace)}',{pylint_disable}" - ) + serialization_type = response.serialization_type(serialize_namespace=self.serialize_namespace) + deserialize_code.append(f" '{serialization_type}',{pylint_disable}") deserialize_code.append(" pipeline_response.http_response") deserialize_code.append(")") elif self.code_model.options["models_mode"] == "dpg": @@ -968,9 +975,10 @@ def response_deserialization( if xml_serializable(str(response.default_content_type)): deserialize_func = "_deserialize_xml" deserialize_code.append(f"deserialized = {deserialize_func}(") - deserialize_code.append( - f" {response.type.type_annotation(is_operation_file=True, serialize_namespace=self.serialize_namespace)},{pylint_disable}" + type_annotation = response.type.type_annotation( + is_operation_file=True, serialize_namespace=self.serialize_namespace ) + deserialize_code.append(f" {type_annotation},{pylint_disable}") deserialize_code.append(f" response.{response_attr}(){response.result_property}{format_filed}") deserialize_code.append(")") @@ -1211,13 +1219,6 @@ class OperationSerializer(_OperationSerializer[Operation]): ... class _PagingOperationSerializer(_OperationSerializer[PagingOperationType]): - def __init__(self, code_model: CodeModel, async_mode: bool, client_namespace: str) -> None: - # for pylint reasons need to redefine init - # probably because inheritance is going too deep - super().__init__(code_model, async_mode, client_namespace) - # self.code_model = code_model - # self.async_mode = async_mode - # self.parameter_serializer = ParameterSerializer(self.serialize_namespace) def serialize_path(self, builder: PagingOperationType) -> List[str]: return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) @@ -1376,14 +1377,6 @@ class PagingOperationSerializer(_PagingOperationSerializer[PagingOperation]): .. class _LROOperationSerializer(_OperationSerializer[LROOperationType]): - def __init__(self, code_model: CodeModel, async_mode: bool, client_namespace: str) -> None: - # for pylint reasons need to redefine init - # probably because inheritance is going too deep - super().__init__(code_model, async_mode, client_namespace) - # self.code_model = code_model - # self.async_mode = async_mode - # self.parameter_serializer = ParameterSerializer() - def serialize_path(self, builder: LROOperationType) -> List[str]: return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index 2e3f2e5032..57e774d99b 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -4,13 +4,11 @@ # license information. # -------------------------------------------------------------------------- import json -from typing import Any, List, Optional -from jinja2 import Environment +from typing import Any, List from .import_serializer import FileImportSerializer, TypingSection from ..models.imports import MsrestImportType, FileImport from ..models import ( ImportType, - CodeModel, TokenCredentialType, Client, ) @@ -72,7 +70,9 @@ def serialize_service_client_file(self, clients: List[Client]) -> str: for client in clients: imports.merge( client.imports( - self.async_mode, serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.CLIENT + self.async_mode, + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.CLIENT, ) ) @@ -159,7 +159,9 @@ def serialize_config_file(self, clients: List[Client]) -> str: for client in self.code_model.clients: imports.merge( client.config.imports( - self.async_mode, serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.CLIENT + self.async_mode, + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.CLIENT, ) ) return template.render( diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py index 19fd28e2c0..d02fbeae60 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/metadata_serializer.py @@ -15,7 +15,6 @@ ImportType, CodeModel, ) -from ..models.utils import NamespaceType from .builder_serializer import get_operation_serializer from .base_serializer import BaseSerializer from .import_serializer import FileImportSerializer diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index d147b1c369..66447ff587 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -73,7 +73,8 @@ def initialize_discriminator_property(model: ModelType, prop: Property) -> str: def initialize_standard_property(self, prop: Property): if not (prop.optional or prop.client_default_value is not None): - return f"{prop.client_name}: {prop.type_annotation(serialize_namespace=self.serialize_namespace)},{prop.pylint_disable()}" + type_annotation = prop.type_annotation(serialize_namespace=self.serialize_namespace) + return f"{prop.client_name}: {type_annotation},{prop.pylint_disable()}" return ( f"{prop.client_name}: {prop.type_annotation(serialize_namespace=self.serialize_namespace)} = " f"{prop.client_default_value_declaration},{prop.pylint_disable()}" @@ -150,7 +151,11 @@ def imports(self) -> FileImport: for model in self.models: file_import.merge(model.imports(is_operation_file=False)) for param in self._init_line_parameters(model): - file_import.merge(param.imports(serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL)) + file_import.merge( + param.imports( + serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL + ) + ) return file_import @@ -233,7 +238,9 @@ def imports(self) -> FileImport: file_import.merge(model.imports(is_operation_file=False, serialize_namespace=self.serialize_namespace)) for prop in model.properties: file_import.merge( - prop.imports(serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL) + prop.imports( + serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL + ) ) if model.is_polymorphic: file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) @@ -295,7 +302,8 @@ def declare_property(self, prop: Property) -> str: if prop.is_discriminator and isinstance(prop.type, (ConstantType, EnumValue)) and prop.type.value else "" ) - generated_code = f'{prop.client_name}: {prop.type_annotation(serialize_namespace=self.serialize_namespace)} = {field}({", ".join(args)})' + type_annotation = prop.type_annotation(serialize_namespace=self.serialize_namespace) + generated_code = f'{prop.client_name}: {type_annotation} = {field}({", ".join(args)})' # there is 4 spaces indentation so original line length limit 120 - 4 = 116 pylint_disable = ( " # pylint: disable=line-too-long" diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py index 4f49832f36..0b2a0685fd 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py @@ -6,7 +6,6 @@ from typing import List from jinja2 import Environment -from ..models.operation_group import OperationGroup from ..models import CodeModel, OperationGroup diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py index aef7c68614..2f7d4a95c4 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/request_builders_serializer.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import List, Optional +from typing import List from jinja2 import Environment from ..models import FileImport @@ -20,10 +20,8 @@ def __init__( code_model: CodeModel, env: Environment, request_builders: List[RequestBuilderType], - *, - serialize_namespace: Optional[str] = None, ) -> None: - super().__init__(code_model, env, serialize_namespace=serialize_namespace) + super().__init__(code_model, env) self.request_builders = request_builders self.group_name = request_builders[0].group_name @@ -44,7 +42,9 @@ def serialize_init(self) -> str: @property def serialize_namespace(self) -> str: - return self.code_model.get_serialize_namespace(self.client_namespace, client_namespace_type=NamespaceType.OPERATION) + return self.code_model.get_serialize_namespace( + self.client_namespace, client_namespace_type=NamespaceType.OPERATION + ) def serialize_request_builders(self) -> str: template = self.env.get_template("request_builders.py.jinja2") @@ -55,5 +55,7 @@ def serialize_request_builders(self) -> str: imports=FileImportSerializer( self.imports, ), - request_builder_serializer=RequestBuilderSerializer(self.code_model, async_mode=False), + request_builder_serializer=RequestBuilderSerializer( + self.code_model, async_mode=False, client_namespace=self.client_namespace + ), ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py index 112a0a2e1a..cc587e5179 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- import logging -from typing import Dict, Any, Union, Tuple, Optional +from typing import Dict, Any, Union, Tuple from jinja2 import Environment from ..models.operation import OperationBase @@ -45,7 +45,11 @@ def __init__( def _imports(self) -> FileImportSerializer: imports = FileImport(self.code_model) client = self.operation_group.client - namespace = get_parent_namespace(client.client_namespace) if self.code_model.options["multiapi"] else client.client_namespace + namespace = ( + get_parent_namespace(client.client_namespace) + if self.code_model.options["multiapi"] + else client.client_namespace + ) imports.add_submodule_import(namespace, client.name, ImportType.LOCAL) credential_type = getattr(client.credential, "type", None) if isinstance(credential_type, TokenCredentialType): diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py b/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py index 9274387d32..206cac35a9 100644 --- a/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/asynctests/test_client_namespace_async.py @@ -12,7 +12,6 @@ # from client.clientnamespace.second.sub.models import SecondClientEnumType - # @pytest.fixture # async def first_client(): # async with ClientNamespaceFirstClient() as client: diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py b/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py index 265c6cf2b6..201b6107d9 100644 --- a/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/test_client_namespace.py @@ -12,7 +12,6 @@ # from client.clientnamespace.second.sub.models import SecondClientEnumType - # @pytest.fixture # def first_client(): # with ClientNamespaceFirstClient() as client: From 4128461de1b38aca5486f46686b9f4a166123563 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 11 Dec 2024 13:02:27 +0800 Subject: [PATCH 43/49] fix --- .../pygen/codegen/models/code_model.py | 9 ++++-- .../pygen/codegen/models/operation_group.py | 4 ++- .../pygen/codegen/serializers/__init__.py | 32 ++++++++++--------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 31f26baf20..aefa1871dd 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -200,9 +200,13 @@ def has_form_data(self) -> bool: def has_etag(self) -> bool: return any(client.has_etag for client in self.clients) + @staticmethod + def clients_has_operations(clients: List[Client]) -> bool: + return any(c for c in clients if c.has_operations) + @property def has_operations(self) -> bool: - return any(c for c in self.clients if c.has_operations) + return self.clients_has_operations(self.clients) @property def has_non_abstract_operations(self) -> bool: @@ -277,8 +281,7 @@ def operations_folder_name(self, client_namespace: str) -> str: """Get the name of the operations folder that holds operations.""" if client_namespace not in self._operations_folder_name: name = "operations" - client_namespace_type = self.client_namespace_types.get(client_namespace) - operation_groups = client_namespace_type.operation_groups if client_namespace_type else [] + operation_groups = self.client_namespace_types.get(client_namespace, ClientNamespaceType()).operation_groups if self.options["version_tolerant"] and all(og.is_mixin for og in operation_groups): name = f"_{name}" self._operations_folder_name[client_namespace] = name diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 7e0bba7f53..3f1988ce0c 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -130,6 +130,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ) if self.is_mixin: file_import.add_submodule_import( + # XxxMixinABC is always defined in _vendor of client namespace self.code_model.get_relative_import_path( serialize_namespace, self.client.client_namespace, module_name="_vendor", async_mode=async_mode ), @@ -138,8 +139,9 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ) if self.has_abstract_operations: file_import.add_submodule_import( + # raise_if_not_implemented is always defined in _vendor of top namespace self.code_model.get_relative_import_path( - serialize_namespace, module_name="_vendor", async_mode=async_mode + serialize_namespace, self.code_model.namespace, module_name="_vendor", async_mode=async_mode ), "raise_if_not_implemented", ImportType.LOCAL, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 2a76ac086c..7e97fb455b 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -354,26 +354,28 @@ def _serialize_client_and_config_files( # when there is client.py, there must be __init__.py self.write_file( exec_path / Path(f"{async_path}__init__.py"), - general_serializer.serialize_init_file(clients), - ) - - # write client file - self.write_file( - exec_path / Path(f"{async_path}{self.code_model.client_filename}.py"), - general_serializer.serialize_service_client_file(clients), - ) - - # write config file - self.write_file( - exec_path / Path(f"{async_path}_configuration.py"), - general_serializer.serialize_config_file(clients), + general_serializer.serialize_init_file([c for c in clients if c.has_operations]), ) # if there was a patch file before, we keep it self._keep_patch_file(exec_path / Path(f"{async_path}_patch.py"), env) - # sometimes we need define additional Mixin class for client in _vendor.py - self._serialize_and_write_vendor_file(env, namespace) + if self.code_model.clients_has_operations(clients): + + # write client file + self.write_file( + exec_path / Path(f"{async_path}{self.code_model.client_filename}.py"), + general_serializer.serialize_service_client_file(clients), + ) + + # write config file + self.write_file( + exec_path / Path(f"{async_path}_configuration.py"), + general_serializer.serialize_config_file(clients), + ) + + # sometimes we need define additional Mixin class for client in _vendor.py + self._serialize_and_write_vendor_file(env, namespace) def _serialize_and_write_vendor_file(self, env: Environment, namespace: str) -> None: exec_path = self.exec_path(namespace) From 6aa420390708a0b3b02079c46641c854fdb4476b Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 13 Dec 2024 14:40:00 +0800 Subject: [PATCH 44/49] fix typing --- .../generator/pygen/codegen/models/combined_type.py | 4 +++- .../generator/pygen/codegen/models/enum_type.py | 2 +- .../generator/pygen/codegen/models/model_type.py | 2 +- .../generator/pygen/codegen/models/utils.py | 1 + .../generator/pygen/codegen/serializers/types_serializer.py | 3 ++- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/combined_type.py b/packages/http-client-python/generator/pygen/codegen/models/combined_type.py index 88452b5db1..8170758f23 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/combined_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/combined_type.py @@ -8,6 +8,7 @@ from .imports import FileImport, ImportType, TypingSection from .base import BaseType from .model_type import ModelType +from .utils import NamespaceType if TYPE_CHECKING: from .code_model import CodeModel @@ -113,7 +114,8 @@ def instance_check_template(self) -> str: def imports(self, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - if self.name and not kwargs.get("is_types_file"): + serialize_namespace_type = kwargs.get("serialize_namespace_type") + if self.name and serialize_namespace_type != NamespaceType.TYPES_FILE: file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace), "_types", diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index f3879a003e..f2ac322af9 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -243,7 +243,7 @@ def imports(self, **kwargs: Any) -> FileImport: alias=alias, typing_section=TypingSection.REGULAR, ) - elif serialize_namespace_type == NamespaceType.MODEL: + elif serialize_namespace_type in [NamespaceType.MODEL, NamespaceType.TYPES_FILE]: file_import.add_submodule_import( relative_path, "models", diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index f999abefbd..f3e0d26ddd 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -321,7 +321,7 @@ def imports(self, **kwargs: Any) -> FileImport: ImportType.LOCAL, typing_section=TypingSection.REGULAR, ) - elif serialize_namespace_type == NamespaceType.MODEL: + elif serialize_namespace_type in [NamespaceType.MODEL, NamespaceType.TYPES_FILE]: file_import.add_submodule_import( relative_path, "models", diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index c0ee5a950b..6cf97920ee 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -29,3 +29,4 @@ class NamespaceType(str, Enum): MODEL = "model" OPERATION = "operation" CLIENT = "client" + TYPES_FILE = "types_file" diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index fae40d73bf..30a17a6957 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -4,6 +4,7 @@ # license information. # -------------------------------------------------------------------------- from ..models.imports import FileImport, ImportType +from ..models.utils import NamespaceType from .import_serializer import FileImportSerializer from .base_serializer import BaseSerializer @@ -18,7 +19,7 @@ def imports(self) -> FileImport: ImportType.STDLIB, ) for nu in self.code_model.named_unions: - file_import.merge(nu.imports(is_types_file=True, serialize_namespace=self.serialize_namespace)) + file_import.merge(nu.imports(serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.TYPES_FILE)) return file_import def serialize(self) -> str: From 57005974094de0b3a17bda91746f0c18b362d8d0 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 13 Dec 2024 15:29:09 +0800 Subject: [PATCH 45/49] fix typing --- packages/http-client-python/emitter/src/utils.ts | 2 +- .../pygen/codegen/models/operation_group.py | 16 +++++++++++++++- .../serializers/operations_init_serializer.py | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/http-client-python/emitter/src/utils.ts b/packages/http-client-python/emitter/src/utils.ts index d451a0163c..89b69750fc 100644 --- a/packages/http-client-python/emitter/src/utils.ts +++ b/packages/http-client-python/emitter/src/utils.ts @@ -218,5 +218,5 @@ export function getClientNamespace FileImport: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) for operation in self.operations: file_import.merge(operation.imports(async_mode, **kwargs)) - if not self.code_model.options["combine_operation_files"]: + if (not self.code_model.options["combine_operation_files"]): for og in self.operation_groups: file_import.add_submodule_import( self.code_model.get_relative_import_path( @@ -116,6 +116,20 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: og.class_name, ImportType.LOCAL, ) + else: + for og in self.operation_groups: + namespace = self.code_model.get_serialize_namespace(og.client_namespace, async_mode, NamespaceType.OPERATION) + if namespace != serialize_namespace: + file_import.add_submodule_import( + self.code_model.get_relative_import_path( + serialize_namespace, + og.client_namespace, + imported_namespace_type=NamespaceType.OPERATION, + async_mode=async_mode, + ), + og.class_name, + ImportType.LOCAL, + ) # for multiapi if ( (self.code_model.public_model_types) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py index 0b2a0685fd..80f943db3d 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py @@ -18,7 +18,7 @@ def __init__( async_mode: bool, ) -> None: self.code_model = code_model - self.operation_groups = [og for og in operation_groups if not og.has_parent_operation_group] + self.operation_groups = operation_groups self.env = env self.async_mode = async_mode From 49cc526d8fb6013ca9ae85c632139b5327cc58a0 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 13 Dec 2024 16:21:13 +0800 Subject: [PATCH 46/49] review --- .../http-client-python/emitter/src/utils.ts | 4 ++- .../pygen/codegen/models/base_builder.py | 4 +-- .../pygen/codegen/models/enum_type.py | 9 ++++--- .../pygen/codegen/models/model_type.py | 8 +++--- .../pygen/codegen/models/operation_group.py | 26 ++++++++++--------- .../pygen/codegen/models/parameter.py | 4 +-- .../pygen/codegen/models/parameter_list.py | 14 +++++----- .../codegen/serializers/builder_serializer.py | 23 +++++++++++----- .../codegen/serializers/client_serializer.py | 4 ++- .../codegen/serializers/types_serializer.py | 6 ++++- 10 files changed, 64 insertions(+), 38 deletions(-) diff --git a/packages/http-client-python/emitter/src/utils.ts b/packages/http-client-python/emitter/src/utils.ts index 89b69750fc..f4f5859936 100644 --- a/packages/http-client-python/emitter/src/utils.ts +++ b/packages/http-client-python/emitter/src/utils.ts @@ -218,5 +218,7 @@ export function getClientNamespace str: ) return self._description or self.name - def method_signature(self, async_mode: bool) -> List[str]: + def method_signature(self, async_mode: bool, **kwargs: Any) -> List[str]: if self.abstract: return ["*args,", "**kwargs"] - return self.parameters.method_signature(async_mode) + return self.parameters.method_signature(async_mode, **kwargs) diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index f2ac322af9..91e9ab68b6 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -172,9 +172,12 @@ def type_annotation(self, **kwargs: Any) -> str: :rtype: str """ if self.code_model.options["models_mode"]: - serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) - module_name = f"{model_alias}." if kwargs.get("need_model_alias", True) else "" + + module_name = "" + if kwargs.get("need_model_alias", True): + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) + module_name = f"{model_alias}." file_name = f"{self.code_model.enums_filename}." if self.internal else "" model_name = module_name + file_name + self.name # we don't need quoted annotation in operation files, and need it in model folder files. diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index f3e0d26ddd..c806ba46f3 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -281,9 +281,11 @@ class GeneratedModelType(ModelType): def type_annotation(self, **kwargs: Any) -> str: is_operation_file = kwargs.pop("is_operation_file", False) skip_quote = kwargs.get("skip_quote", False) - serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) - module_name = f"{model_alias}." if kwargs.get("need_model_alias", True) else "" + module_name = "" + if kwargs.get("need_model_alias", True): + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) + module_name = f"{model_alias}." file_name = f"{self.code_model.models_filename}." if self.internal else "" retval = module_name + file_name + self.name return retval if is_operation_file or skip_quote else f'"{retval}"' diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index 2b502a7f62..a53438ae46 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -104,7 +104,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) for operation in self.operations: file_import.merge(operation.imports(async_mode, **kwargs)) - if (not self.code_model.options["combine_operation_files"]): + if not self.code_model.options["combine_operation_files"]: for og in self.operation_groups: file_import.add_submodule_import( self.code_model.get_relative_import_path( @@ -118,18 +118,20 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ) else: for og in self.operation_groups: - namespace = self.code_model.get_serialize_namespace(og.client_namespace, async_mode, NamespaceType.OPERATION) + namespace = self.code_model.get_serialize_namespace( + og.client_namespace, async_mode, NamespaceType.OPERATION + ) if namespace != serialize_namespace: - file_import.add_submodule_import( - self.code_model.get_relative_import_path( - serialize_namespace, - og.client_namespace, - imported_namespace_type=NamespaceType.OPERATION, - async_mode=async_mode, - ), - og.class_name, - ImportType.LOCAL, - ) + file_import.add_submodule_import( + self.code_model.get_relative_import_path( + serialize_namespace, + og.client_namespace, + imported_namespace_type=NamespaceType.OPERATION, + async_mode=async_mode, + ), + og.class_name, + ImportType.LOCAL, + ) # for multiapi if ( (self.code_model.public_model_types) diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter.py b/packages/http-client-python/generator/pygen/codegen/models/parameter.py index 19ec0c380d..6d4d5e5aa3 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter.py @@ -212,8 +212,8 @@ def docstring_type_keyword(self) -> str: @abc.abstractmethod def in_method_signature(self) -> bool: ... - def method_signature(self, async_mode: bool) -> str: - type_annotation = self.type_annotation(async_mode=async_mode) + def method_signature(self, async_mode: bool, **kwargs: Any) -> str: + type_annotation = self.type_annotation(async_mode=async_mode, **kwargs) if self.client_default_value is not None or self.optional: return f"{self.client_name}: {type_annotation} = {self.client_default_value_declaration}," if self.default_to_unset_sentinel: diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter_list.py b/packages/http-client-python/generator/pygen/codegen/models/parameter_list.py index fe0210c301..87666eada9 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter_list.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter_list.py @@ -200,22 +200,22 @@ def method(self) -> List[Union[ParameterType, BodyParameterType]]: """Sorted method params. First positional, then keyword only, then kwarg""" return self.positional + self.keyword_only + self.kwarg - def method_signature(self, async_mode: bool) -> List[str]: + def method_signature(self, async_mode: bool, **kwargs: Any) -> List[str]: """Method signature for this parameter list.""" return method_signature_helper( - positional=self.method_signature_positional(async_mode), - keyword_only=self.method_signature_keyword_only(async_mode), + positional=self.method_signature_positional(async_mode, **kwargs), + keyword_only=self.method_signature_keyword_only(async_mode, **kwargs), kwarg_params=self.method_signature_kwargs, ) - def method_signature_positional(self, async_mode: bool) -> List[str]: + def method_signature_positional(self, async_mode: bool, **kwargs: Any) -> List[str]: """Signature for positional parameters""" - return [parameter.method_signature(async_mode) for parameter in self.positional] + return [parameter.method_signature(async_mode, **kwargs) for parameter in self.positional] - def method_signature_keyword_only(self, async_mode: bool) -> List[str]: + def method_signature_keyword_only(self, async_mode: bool, **kwargs: Any) -> List[str]: """Signature for keyword only parameters""" result = [ - parameter.method_signature(async_mode) + parameter.method_signature(async_mode, **kwargs) for parameter in self.keyword_only if not parameter.hide_in_operation_signature ] diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 520c6dbf11..98c406d48e 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -240,7 +240,9 @@ def _method_signature(self, builder: BuilderType) -> str: function_def=self._function_def, method_name=builder.name, need_self_param=self._need_self_param, - method_param_signatures=builder.method_signature(self.async_mode), + method_param_signatures=builder.method_signature( + self.async_mode, serialize_namespace=self.serialize_namespace + ), pylint_disable=builder.pylint_disable(self.async_mode), ) @@ -298,6 +300,7 @@ def param_description(self, builder: BuilderType) -> List[str]: ) docstring_type = param.docstring_type( async_mode=self.async_mode, + serialize_namespace=self.serialize_namespace, ) description_list.append(f":{param.docstring_type_keyword} {param.client_name}: {docstring_type}") return description_list @@ -1023,11 +1026,14 @@ def handle_error_response(self, builder: OperationType) -> List[str]: if isinstance(e.status_codes[0], int): for status_code in e.status_codes: retval.append(f" {condition} response.status_code == {status_code}:") + type_annotation = e.type.type_annotation( # type: ignore + is_operation_file=True, skip_quote=True, serialize_namespace=self.serialize_namespace + ) if self.code_model.options["models_mode"] == "dpg": - retval.append(f" error = _failsafe_deserialize({e.type.type_annotation(is_operation_file=True, skip_quote=True)}, response.json())") # type: ignore # pylint: disable=line-too-long + retval.append(f" error = _failsafe_deserialize({type_annotation}, response.json())") else: retval.append( - f" error = self._deserialize.failsafe_deserialize({e.type.type_annotation(is_operation_file=True, skip_quote=True)}, " # type: ignore # pylint: disable=line-too-long + f" error = self._deserialize.failsafe_deserialize({type_annotation}, " "pipeline_response)" ) # add build-in error type @@ -1065,11 +1071,14 @@ def handle_error_response(self, builder: OperationType) -> List[str]: retval.append( f" {condition} {e.status_codes[0][0]} <= response.status_code <= {e.status_codes[0][1]}:" ) + type_annotation = e.type.type_annotation( # type: ignore + is_operation_file=True, skip_quote=True, serialize_namespace=self.serialize_namespace + ) if self.code_model.options["models_mode"] == "dpg": - retval.append(f" error = _failsafe_deserialize({e.type.type_annotation(is_operation_file=True, skip_quote=True)}, response.json())") # type: ignore # pylint: disable=line-too-long + retval.append(f" error = _failsafe_deserialize({type_annotation}, response.json())") else: retval.append( - f" error = self._deserialize.failsafe_deserialize({e.type.type_annotation(is_operation_file=True, skip_quote=True)}, " # type: ignore # pylint: disable=line-too-long + f" error = self._deserialize.failsafe_deserialize({type_annotation}, " "pipeline_response)" ) condition = "elif" @@ -1326,7 +1335,9 @@ def _extract_data_callback(self, builder: PagingOperationType) -> List[str]: access = f".{item_name}" if self.code_model.options["models_mode"] == "msrest" else f'["{item_name}"]' list_of_elem_deserialized = "" if self.code_model.options["models_mode"] == "dpg": - item_type = builder.item_type.type_annotation(is_operation_file=True) + item_type = builder.item_type.type_annotation( + is_operation_file=True, serialize_namespace=self.serialize_namespace + ) list_of_elem_deserialized = f"_deserialize({item_type}, deserialized{access})" else: list_of_elem_deserialized = f"deserialized{access}" diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py index 22540a1a85..6ac6de68bb 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/client_serializer.py @@ -25,7 +25,9 @@ def _init_signature(self, async_mode: bool) -> str: function_def="def", method_name="__init__", need_self_param=True, - method_param_signatures=self.client.parameters.method_signature(async_mode), + method_param_signatures=self.client.parameters.method_signature( + async_mode, serialize_namespace=self.serialize_namespace + ), pylint_disable=pylint_disable, ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 30a17a6957..749d8ca240 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -19,7 +19,11 @@ def imports(self) -> FileImport: ImportType.STDLIB, ) for nu in self.code_model.named_unions: - file_import.merge(nu.imports(serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.TYPES_FILE)) + file_import.merge( + nu.imports( + serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.TYPES_FILE + ) + ) return file_import def serialize(self) -> str: From 5b7fe1aa16dad5fd94d7aae4fda3bacda5a77e36 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 13 Dec 2024 20:37:13 +0800 Subject: [PATCH 47/49] review --- .../pygen/codegen/models/code_model.py | 3 +- .../pygen/codegen/models/enum_type.py | 5 ++- .../pygen/codegen/models/model_type.py | 7 ++-- .../pygen/codegen/models/operation_group.py | 1 + .../codegen/serializers/model_serializer.py | 37 ++++++++++++++----- .../serializers/operations_init_serializer.py | 2 +- 6 files changed, 40 insertions(+), 15 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index aefa1871dd..a3b790ee44 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -113,6 +113,7 @@ def get_relative_import_path( *, imported_namespace_type: NamespaceType = NamespaceType.CLIENT, async_mode: bool = False, + filename: str = "", module_name: Optional[str] = None, ) -> str: if imported_namespace is None: @@ -125,7 +126,7 @@ def get_relative_import_path( module_namespace = f".{self.operations_folder_name(imported_namespace)}" else: module_namespace = "" - imported_namespace = imported_namespace + async_namespace + module_namespace + imported_namespace = imported_namespace + async_namespace + module_namespace + filename key = f"{serialize_namespace}-{imported_namespace}" if key not in self._relative_import_path: diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 91e9ab68b6..ab0e5447a9 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -238,6 +238,7 @@ def imports(self, **kwargs: Any) -> FileImport: relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) serialize_namespace_type = kwargs.get("serialize_namespace_type") + called_by_property = kwargs.get("called_by_property", False) if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: file_import.add_submodule_import( relative_path, @@ -246,7 +247,9 @@ def imports(self, **kwargs: Any) -> FileImport: alias=alias, typing_section=TypingSection.REGULAR, ) - elif serialize_namespace_type in [NamespaceType.MODEL, NamespaceType.TYPES_FILE]: + elif serialize_namespace_type == NamespaceType.TYPES_FILE or ( + serialize_namespace_type == NamespaceType.MODEL and called_by_property + ): file_import.add_submodule_import( relative_path, "models", diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index c806ba46f3..9784bc7a5a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -307,6 +307,7 @@ def imports(self, **kwargs: Any) -> FileImport: relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) serialize_namespace_type = kwargs.get("serialize_namespace_type") + called_by_property = kwargs.get("called_by_property", False) # add import for models in operations or _types file if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: file_import.add_submodule_import( @@ -314,16 +315,16 @@ def imports(self, **kwargs: Any) -> FileImport: "models", ImportType.LOCAL, alias=alias, - typing_section=TypingSection.REGULAR, ) if self.is_form_data: file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace), "_model_base", ImportType.LOCAL, - typing_section=TypingSection.REGULAR, ) - elif serialize_namespace_type in [NamespaceType.MODEL, NamespaceType.TYPES_FILE]: + elif serialize_namespace_type == NamespaceType.TYPES_FILE or ( + serialize_namespace_type == NamespaceType.MODEL and called_by_property + ): file_import.add_submodule_import( relative_path, "models", diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py index a53438ae46..5a95e033cf 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation_group.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation_group.py @@ -128,6 +128,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: og.client_namespace, imported_namespace_type=NamespaceType.OPERATION, async_mode=async_mode, + filename=f".{og.filename}", ), og.class_name, ImportType.LOCAL, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 66447ff587..8869804a31 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -225,23 +225,42 @@ def super_call(self, model: ModelType) -> List[str]: def imports(self) -> FileImport: file_import = FileImport(self.code_model) - file_import.add_submodule_import( - self.code_model.get_relative_import_path(self.serialize_namespace), - "_model_base", - ImportType.LOCAL, - TypingSection.REGULAR, - ) - + if any(not m.parents for m in self.models): + file_import.add_submodule_import( + self.code_model.get_relative_import_path(self.serialize_namespace), + "_model_base", + ImportType.LOCAL, + TypingSection.REGULAR, + ) for model in self.models: if model.base == "json": continue - file_import.merge(model.imports(is_operation_file=False, serialize_namespace=self.serialize_namespace)) + file_import.merge( + model.imports( + is_operation_file=False, + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.MODEL, + ) + ) for prop in model.properties: file_import.merge( prop.imports( - serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.MODEL, + called_by_property=True, ) ) + for parent in model.parents: + if parent.client_namespace != model.client_namespace: + file_import.add_submodule_import( + self.code_model.get_relative_import_path( + self.serialize_namespace, + parent.client_namespace, + imported_namespace_type=NamespaceType.MODEL, + ), + parent.name, + ImportType.LOCAL, + ) if model.is_polymorphic: file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) if not model.internal and self.init_line(model): diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py index 80f943db3d..0b2a0685fd 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/operations_init_serializer.py @@ -18,7 +18,7 @@ def __init__( async_mode: bool, ) -> None: self.code_model = code_model - self.operation_groups = operation_groups + self.operation_groups = [og for og in operation_groups if not og.has_parent_operation_group] self.env = env self.async_mode = async_mode From c592198adf596143cd9cef1a3ed8b622dfe7f2d3 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 16 Dec 2024 11:27:46 +0800 Subject: [PATCH 48/49] review --- .../pygen/codegen/serializers/model_serializer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 8869804a31..512ff4e644 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -156,6 +156,14 @@ def imports(self) -> FileImport: serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL ) ) + for prop in model.properties: + file_import.merge( + prop.imports( + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.MODEL, + called_by_property=True, + ) + ) return file_import From 21d84486028f6101712c5531f8815b99f0a3b5c1 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 16 Dec 2024 13:39:46 +0800 Subject: [PATCH 49/49] review --- .../generator/pygen/codegen/serializers/model_serializer.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index 512ff4e644..538b4f8c5a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -153,12 +153,6 @@ def imports(self) -> FileImport: for param in self._init_line_parameters(model): file_import.merge( param.imports( - serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL - ) - ) - for prop in model.properties: - file_import.merge( - prop.imports( serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.MODEL, called_by_property=True,