pr3 workaround test

.flake8

@@ -3,8 +3,12 @@ max-line-length = 125
 
 # E203,W503: incompatibilities with black
 # E722: also signaled by pylint (disabling here globally and on case-by-case basis with pylint)
+# E501: line length
+# E203: whitespace before ':'
+# W503: line break before binary operator
+# E722: do not use bare 'except'
 ignore =
-  E501,  # line length
-  E203,  # whitespace before ':'
-  W503,  # line break before binary operator
-  E722,  # do not use bare 'except'
+  E501,
+  E203,
+  W503,
+  E722,

.pre-commit-config.yaml

@@ -9,13 +9,14 @@ repos:
         args:
           - --line-length=125
 
-  - repo: https://gitlab.com/pycqa/flake8
+  - repo: https://github.com/PyCQA/flake8
     rev: 4.0.1
     hooks:
       - id: flake8
         files: \.py$
         args:
-          - --config=.flake8
+         - --config=.flake8
+
 
   - repo: https://github.com/PyCQA/isort
     rev: 5.10.1

.pylintrc

@@ -7,7 +7,6 @@ enable=
   useless-suppression,
 
 disable=
-  bad-continuation,
   duplicate-code,
   fixme,
   import-outside-toplevel,
@@ -22,8 +21,15 @@ disable=
   too-many-nested-blocks,
   too-many-public-methods,
   too-many-statements,
-  too-public-methods,
   wrong-import-order,
+  import-error,
+  consider-using-f-string,
+  use-implicit-booleaness-not-comparison,
+  unspecified-encoding,
+  no-name-in-module,
+  use-list-literal,
+  use-dict-literal,
+
 
 [FORMAT]
 max-line-length=125

karapace/dependency.py

@@ -19,6 +19,9 @@ def __init__(self, name: str, subject: Subject, version: Version, target_schema:
         self.version = version
         self.schema = target_schema
 
+    def get_schema(self) -> "ValidatedTypedSchema":
+        return self.schema
+
     @staticmethod
     def of(reference: Reference, target_schema: "ValidatedTypedSchema") -> "Dependency":
         return Dependency(reference.name, reference.subject, reference.version, target_schema)

karapace/protobuf/compare_type_lists.py

@@ -0,0 +1,68 @@
+from itertools import chain
+from karapace.protobuf.compare_result import CompareResult, Modification
+from karapace.protobuf.compare_type_storage import CompareTypes
+from karapace.protobuf.enum_element import EnumElement
+from karapace.protobuf.exception import IllegalStateException
+from karapace.protobuf.message_element import MessageElement
+from karapace.protobuf.type_element import TypeElement
+from typing import List
+
+
+def compare_type_lists(
+    self_types_list: List[TypeElement],
+    other_types_list: List[TypeElement],
+    result: CompareResult,
+    compare_types: CompareTypes,
+) -> CompareResult:
+    self_types = {}
+    other_types = {}
+    self_indexes = {}
+    other_indexes = {}
+
+    type_: TypeElement
+    for i, type_ in enumerate(self_types_list):
+        self_types[type_.name] = type_
+        self_indexes[type_.name] = i
+        compare_types.add_self_type(compare_types.self_package_name, type_)
+
+    for i, type_ in enumerate(other_types_list):
+        other_types[type_.name] = type_
+        other_indexes[type_.name] = i
+        compare_types.add_other_type(compare_types.other_package_name, type_)
+
+    for name in chain(self_types.keys(), other_types.keys() - self_types.keys()):
+
+        result.push_path(str(name), True)
+
+        if self_types.get(name) is None and other_types.get(name) is not None:
+            if isinstance(other_types[name], MessageElement):
+                result.add_modification(Modification.MESSAGE_ADD)
+            elif isinstance(other_types[name], EnumElement):
+                result.add_modification(Modification.ENUM_ADD)
+            else:
+                raise IllegalStateException("Instance of element is not applicable")
+        elif self_types.get(name) is not None and other_types.get(name) is None:
+            if isinstance(self_types[name], MessageElement):
+                result.add_modification(Modification.MESSAGE_DROP)
+            elif isinstance(self_types[name], EnumElement):
+                result.add_modification(Modification.ENUM_DROP)
+            else:
+                raise IllegalStateException("Instance of element is not applicable")
+        else:
+            if other_indexes[name] != self_indexes[name]:
+                if isinstance(self_types[name], MessageElement):
+                    # incompatible type
+                    result.add_modification(Modification.MESSAGE_MOVE)
+                else:
+                    raise IllegalStateException("Instance of element is not applicable")
+            else:
+                if isinstance(self_types[name], MessageElement) and isinstance(other_types[name], MessageElement):
+                    self_types[name].compare(other_types[name], result, compare_types)
+                elif isinstance(self_types[name], EnumElement) and isinstance(other_types[name], EnumElement):
+                    self_types[name].compare(other_types[name], result, compare_types)
+                else:
+                    # incompatible type
+                    result.add_modification(Modification.TYPE_ALTER)
+        result.pop_path(True)
+
+    return result

karapace/protobuf/compare_type_storage.py

@@ -35,8 +35,9 @@ def compute_name(t: ProtoType, result_path: List[str], package_name: str, types:
 class CompareTypes:
     def __init__(self, self_package_name: str, other_package_name: str, result: CompareResult) -> None:
 
-        self.self_package_name = self_package_name
-        self.other_package_name = other_package_name
+        self.self_package_name = self_package_name or ""
+        self.other_package_name = other_package_name or ""
+
         self.self_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {}
         self.other_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {}
         self.locked_messages: List["MessageElement"] = []
@@ -90,17 +91,24 @@ def self_type_short_name(self, t: ProtoType) -> Optional[str]:
         if name is None:
             raise IllegalArgumentException(f"Cannot determine message type {t}")
         type_record: TypeRecord = self.self_types.get(name)
-        if name.startswith(type_record.package_name):
-            return name[(len(type_record.package_name) + 1) :]
+        package_name = type_record.package_name
+        if package_name is None:
+            package_name = ""
+        if name.startswith(package_name):
+            return name[(len(package_name) + 1) :]
+
         return name
 
     def other_type_short_name(self, t: ProtoType) -> Optional[str]:
         name = compute_name(t, self.result.path, self.other_package_name, self.other_types)
         if name is None:
             raise IllegalArgumentException(f"Cannot determine message type {t}")
         type_record: TypeRecord = self.other_types.get(name)
-        if name.startswith(type_record.package_name):
-            return name[(len(type_record.package_name) + 1) :]
+        package_name = type_record.package_name
+        if package_name is None:
+            package_name = ""
+        if name.startswith(package_name):
+            return name[(len(package_name) + 1) :]
         return name
 
     def lock_message(self, message: "MessageElement") -> bool:

karapace/protobuf/io.py

@@ -5,13 +5,14 @@
 from karapace.protobuf.protobuf_to_dict import dict_to_protobuf, protobuf_to_dict
 from karapace.protobuf.schema import ProtobufSchema
 from karapace.protobuf.type_element import TypeElement
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 import hashlib
 import importlib
 import importlib.util
 import os
 import subprocess
+import sys
 
 
 def calculate_class_name(name: str) -> str:
@@ -45,29 +46,77 @@ def find_message_name(schema: ProtobufSchema, indexes: List[int]) -> str:
     return ".".join(result)
 
 
+def crawl_dependencies_(schema: ProtobufSchema, deps_list: Dict[str, Dict[str, str]]):
+    if schema.dependencies:
+        for name, dependency in schema.dependencies.items():
+            crawl_dependencies_(dependency.schema, deps_list)
+            deps_list[name] = {
+                "schema": str(dependency.schema),
+                "unique_class_name": calculate_class_name(f"{dependency.version}_{dependency.name}"),
+            }
+
+
+def crawl_dependencies(schema: ProtobufSchema) -> Dict[str, Dict[str, str]]:
+    deps_list: Dict[str, Dict[str, str]] = {}
+    crawl_dependencies_(schema, deps_list)
+    return deps_list
+
+
+def replace_imports(string: str, deps_list: Optional[Dict[str, Dict[str, str]]]) -> str:
+    if deps_list is None:
+        return string
+    for key, value in deps_list.items():
+        unique_class_name = value["unique_class_name"] + ".proto"
+        string = string.replace('"' + key + '"', f'"{unique_class_name}"')
+    return string
+
+
 def get_protobuf_class_instance(schema: ProtobufSchema, class_name: str, cfg: Dict) -> Any:
     directory = cfg["protobuf_runtime_directory"]
-    proto_name = calculate_class_name(str(schema))
+    deps_list = crawl_dependencies(schema)
+    root_class_name = ""
+    for value in deps_list.values():
+        root_class_name = root_class_name + value["unique_class_name"]
+    root_class_name = root_class_name + str(schema)
+    proto_name = calculate_class_name(root_class_name)
+
     proto_path = f"{proto_name}.proto"
-    class_path = f"{directory}/{proto_name}_pb2.py"
-    if not os.path.isfile(proto_path):
-        with open(f"{directory}/{proto_name}.proto", mode="w", encoding="utf8") as proto_text:
-            proto_text.write(str(schema))
-
-    if not os.path.isfile(class_path):
-        subprocess.run(
-            [
-                "protoc",
-                "--python_out=./",
-                proto_path,
-            ],
-            check=True,
-            cwd=directory,
-        )
-
-    spec = importlib.util.spec_from_file_location(f"{proto_name}_pb2", class_path)
+    work_dir = f"{directory}/{proto_name}"
+
+    if not os.path.isdir(work_dir):
+        os.mkdir(work_dir)
+    class_path = f"{directory}/{proto_name}/{proto_name}_pb2.py"
+    if not os.path.exists(class_path):
+        with open(f"{directory}/{proto_name}/{proto_name}.proto", mode="w", encoding="utf8") as proto_text:
+            proto_text.write(replace_imports(str(schema), deps_list))
+
+        protoc_arguments = [
+            "protoc",
+            "--python_out=./",
+            proto_path,
+        ]
+        for value in deps_list.values():
+            proto_file_name = value["unique_class_name"] + ".proto"
+            dependency_path = f"{directory}/{proto_name}/{proto_file_name}"
+            protoc_arguments.append(proto_file_name)
+            with open(dependency_path, mode="w", encoding="utf8") as proto_text:
+                proto_text.write(replace_imports(value["schema"], deps_list))
+
+        if not os.path.isfile(class_path):
+            subprocess.run(
+                protoc_arguments,
+                check=True,
+                cwd=work_dir,
+            )
+
+    sys.path.append(f"./runtime/{proto_name}")
+    spec = importlib.util.spec_from_file_location(
+        f"{proto_name}_pb2",
+        class_path,
+    )
     tmp_module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(tmp_module)
+    sys.path.pop()
     class_to_call = getattr(tmp_module, class_name)
     return class_to_call()
 

karapace/protobuf/known_dependency.py

@@ -116,17 +116,19 @@ class KnownDependency:
         "google/type/postal_address.proto": ["google.type.PostalAddress"],
         "google/type/quaternion.proto": ["google.type.Quaternion"],
         "google/type/timeofday.proto": ["google.type.TimeOfDay"],
-        "confluent/meta.proto": [".confluent.Meta"],
-        "confluent/type/decimal.proto": [".confluent.type.Decimal"],
+        "confluent/meta.proto": ["confluent.Meta"],
+        "confluent/type/decimal.proto": ["confluent.type.Decimal"],
     }
 
     @classmethod
     def static_init(cls) -> None:
         for key, value in cls.map.items():
             for item in value:
                 cls.index[item] = key
+                cls.index["." + item] = key
                 dot = item.rfind(".")
                 cls.index_simple[item[dot + 1 :]] = key
+                cls.index_simple[item] = key
 
 
 @static_init

karapace/protobuf/message_element.py

@@ -22,7 +22,7 @@ def __init__(
         location: Location,
         name: str,
         documentation: str = "",
-        nested_types: List[str] = None,
+        nested_types: List[TypeElement] = None,
         options: List[OptionElement] = None,
         reserveds: List[ReservedElement] = None,
         fields: List[FieldElement] = None,
@@ -80,6 +80,7 @@ def to_schema(self) -> str:
         return "".join(result)
 
     def compare(self, other: "MessageElement", result: CompareResult, types: CompareTypes) -> None:
+        from karapace.protobuf.compare_type_lists import compare_type_lists
 
         if types.lock_message(self):
             field: FieldElement
@@ -138,5 +139,5 @@ def compare(self, other: "MessageElement", result: CompareResult, types: Compare
                     self_one_ofs[name].compare(other_one_ofs[name], result, types)
 
                 result.pop_path()
-
+            compare_type_lists(self.nested_types, other.nested_types, result, types)
             types.unlock_message(self)