Jeremy lig 4589 lightlypathregex pip (#1501)

partially closes lig-4589
- generate new specs with v4 config and lightlyPathRegex for future proofing

lightly/api/api_workflow_compute_worker.py

@@ -22,10 +22,10 @@
     DockerWorkerConfigV4Docker,
     DockerWorkerRegistryEntryData,
     DockerWorkerType,
-    SelectionConfigV3,
-    SelectionConfigV3Entry,
-    SelectionConfigV3EntryInput,
-    SelectionConfigV3EntryStrategy,
+    SelectionConfigV4,
+    SelectionConfigV4Entry,
+    SelectionConfigV4EntryInput,
+    SelectionConfigV4EntryStrategy,
     TagData,
 )
 from lightly.openapi_generated.swagger_client.rest import ApiException
@@ -175,7 +175,7 @@ def create_compute_worker_config(
         self,
         worker_config: Optional[Dict[str, Any]] = None,
         lightly_config: Optional[Dict[str, Any]] = None,
-        selection_config: Optional[Union[Dict[str, Any], SelectionConfigV3]] = None,
+        selection_config: Optional[Union[Dict[str, Any], SelectionConfigV4]] = None,
     ) -> str:
         """Creates a new configuration for a Lightly Worker run.
 
@@ -273,7 +273,7 @@ def schedule_compute_worker_run(
         self,
         worker_config: Optional[Dict[str, Any]] = None,
         lightly_config: Optional[Dict[str, Any]] = None,
-        selection_config: Optional[Union[Dict[str, Any], SelectionConfigV3]] = None,
+        selection_config: Optional[Union[Dict[str, Any], SelectionConfigV4]] = None,
         priority: str = DockerRunScheduledPriority.MID,
         runs_on: Optional[List[str]] = None,
     ) -> str:
@@ -638,17 +638,17 @@ def get_compute_worker_run_tags(self, run_id: str) -> List[TagData]:
         return tags_in_dataset
 
 
-def selection_config_from_dict(cfg: Dict[str, Any]) -> SelectionConfigV3:
-    """Recursively converts selection config from dict to a SelectionConfigV3 instance."""
+def selection_config_from_dict(cfg: Dict[str, Any]) -> SelectionConfigV4:
+    """Recursively converts selection config from dict to a SelectionConfigV4 instance."""
     strategies = []
     for entry in cfg.get("strategies", []):
         new_entry = copy.deepcopy(entry)
-        new_entry["input"] = SelectionConfigV3EntryInput(**entry["input"])
-        new_entry["strategy"] = SelectionConfigV3EntryStrategy(**entry["strategy"])
-        strategies.append(SelectionConfigV3Entry(**new_entry))
+        new_entry["input"] = SelectionConfigV4EntryInput(**entry["input"])
+        new_entry["strategy"] = SelectionConfigV4EntryStrategy(**entry["strategy"])
+        strategies.append(SelectionConfigV4Entry(**new_entry))
     new_cfg = copy.deepcopy(cfg)
     new_cfg["strategies"] = strategies
-    return SelectionConfigV3(**new_cfg)
+    return SelectionConfigV4(**new_cfg)
 
 
 _T = TypeVar("_T")

lightly/openapi_generated/swagger_client/__init__.py

@@ -253,6 +253,11 @@
 from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy import SelectionConfigV3EntryStrategy
 from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy_all_of import SelectionConfigV3EntryStrategyAllOf
 from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy_all_of_target_range import SelectionConfigV3EntryStrategyAllOfTargetRange
+from lightly.openapi_generated.swagger_client.models.selection_config_v4 import SelectionConfigV4
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_all_of import SelectionConfigV4AllOf
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry import SelectionConfigV4Entry
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_input import SelectionConfigV4EntryInput
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_strategy import SelectionConfigV4EntryStrategy
 from lightly.openapi_generated.swagger_client.models.selection_input_predictions_name import SelectionInputPredictionsName
 from lightly.openapi_generated.swagger_client.models.selection_input_type import SelectionInputType
 from lightly.openapi_generated.swagger_client.models.selection_strategy_threshold_operation import SelectionStrategyThresholdOperation

lightly/openapi_generated/swagger_client/models/__init__.py

@@ -220,6 +220,11 @@
 from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy import SelectionConfigV3EntryStrategy
 from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy_all_of import SelectionConfigV3EntryStrategyAllOf
 from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy_all_of_target_range import SelectionConfigV3EntryStrategyAllOfTargetRange
+from lightly.openapi_generated.swagger_client.models.selection_config_v4 import SelectionConfigV4
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_all_of import SelectionConfigV4AllOf
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry import SelectionConfigV4Entry
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_input import SelectionConfigV4EntryInput
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_strategy import SelectionConfigV4EntryStrategy
 from lightly.openapi_generated.swagger_client.models.selection_input_predictions_name import SelectionInputPredictionsName
 from lightly.openapi_generated.swagger_client.models.selection_input_type import SelectionInputType
 from lightly.openapi_generated.swagger_client.models.selection_strategy_threshold_operation import SelectionStrategyThresholdOperation

lightly/openapi_generated/swagger_client/models/docker_worker_config_v4.py

@@ -24,7 +24,7 @@
 from lightly.openapi_generated.swagger_client.models.docker_worker_config_v3_lightly import DockerWorkerConfigV3Lightly
 from lightly.openapi_generated.swagger_client.models.docker_worker_config_v4_docker import DockerWorkerConfigV4Docker
 from lightly.openapi_generated.swagger_client.models.docker_worker_type import DockerWorkerType
-from lightly.openapi_generated.swagger_client.models.selection_config_v3 import SelectionConfigV3
+from lightly.openapi_generated.swagger_client.models.selection_config_v4 import SelectionConfigV4
 
 class DockerWorkerConfigV4(BaseModel):
     """
@@ -33,7 +33,7 @@ class DockerWorkerConfigV4(BaseModel):
     worker_type: DockerWorkerType = Field(..., alias="workerType")
     docker: Optional[DockerWorkerConfigV4Docker] = None
     lightly: Optional[DockerWorkerConfigV3Lightly] = None
-    selection: Optional[SelectionConfigV3] = None
+    selection: Optional[SelectionConfigV4] = None
     __properties = ["workerType", "docker", "lightly", "selection"]
 
     class Config:
@@ -91,7 +91,7 @@ def from_dict(cls, obj: dict) -> DockerWorkerConfigV4:
             "worker_type": obj.get("workerType"),
             "docker": DockerWorkerConfigV4Docker.from_dict(obj.get("docker")) if obj.get("docker") is not None else None,
             "lightly": DockerWorkerConfigV3Lightly.from_dict(obj.get("lightly")) if obj.get("lightly") is not None else None,
-            "selection": SelectionConfigV3.from_dict(obj.get("selection")) if obj.get("selection") is not None else None
+            "selection": SelectionConfigV4.from_dict(obj.get("selection")) if obj.get("selection") is not None else None
         })
         return _obj
 

lightly/openapi_generated/swagger_client/models/selection_config_v4.py

@@ -0,0 +1,92 @@
+# coding: utf-8
+
+"""
+    Lightly API
+
+    Lightly.ai enables you to do self-supervised learning in an easy and intuitive way. The lightly.ai OpenAPI spec defines how one can interact with our REST API to unleash the full potential of lightly.ai  # noqa: E501
+
+    The version of the OpenAPI document: 1.0.0
+    Contact: [email protected]
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+
+from typing import List, Optional, Union
+from pydantic import Extra,  BaseModel, Field, confloat, conint, conlist, constr
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry import SelectionConfigV4Entry
+
+class SelectionConfigV4(BaseModel):
+    """
+    SelectionConfigV4
+    """
+    n_samples: Optional[conint(strict=True, ge=-1)] = Field(None, alias="nSamples")
+    proportion_samples: Optional[Union[confloat(le=1.0, ge=0.0, strict=True), conint(le=1, ge=0, strict=True)]] = Field(None, alias="proportionSamples")
+    strategies: conlist(SelectionConfigV4Entry, min_items=1) = Field(...)
+    lightly_path_regex: Optional[constr(strict=True, min_length=1)] = Field(None, alias="lightlyPathRegex", description="The Lightly Path Regex to extract information from filenames for metadata balancing and more. Docs are coming soon.")
+    __properties = ["nSamples", "proportionSamples", "strategies", "lightlyPathRegex"]
+
+    class Config:
+        """Pydantic configuration"""
+        allow_population_by_field_name = True
+        validate_assignment = True
+        use_enum_values = True
+        extra = Extra.forbid
+
+    def to_str(self, by_alias: bool = False) -> str:
+        """Returns the string representation of the model"""
+        return pprint.pformat(self.dict(by_alias=by_alias))
+
+    def to_json(self, by_alias: bool = False) -> str:
+        """Returns the JSON representation of the model"""
+        return json.dumps(self.to_dict(by_alias=by_alias))
+
+    @classmethod
+    def from_json(cls, json_str: str) -> SelectionConfigV4:
+        """Create an instance of SelectionConfigV4 from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self, by_alias: bool = False):
+        """Returns the dictionary representation of the model"""
+        _dict = self.dict(by_alias=by_alias,
+                          exclude={
+                          },
+                          exclude_none=True)
+        # override the default output from pydantic by calling `to_dict()` of each item in strategies (list)
+        _items = []
+        if self.strategies:
+            for _item in self.strategies:
+                if _item:
+                    _items.append(_item.to_dict(by_alias=by_alias))
+            _dict['strategies' if by_alias else 'strategies'] = _items
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: dict) -> SelectionConfigV4:
+        """Create an instance of SelectionConfigV4 from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return SelectionConfigV4.parse_obj(obj)
+
+        # raise errors for additional fields in the input
+        for _key in obj.keys():
+            if _key not in cls.__properties:
+                raise ValueError("Error due to additional fields (not defined in SelectionConfigV4) in the input: " + str(obj))
+
+        _obj = SelectionConfigV4.parse_obj({
+            "n_samples": obj.get("nSamples"),
+            "proportion_samples": obj.get("proportionSamples"),
+            "strategies": [SelectionConfigV4Entry.from_dict(_item) for _item in obj.get("strategies")] if obj.get("strategies") is not None else None,
+            "lightly_path_regex": obj.get("lightlyPathRegex")
+        })
+        return _obj
+

lightly/openapi_generated/swagger_client/models/selection_config_v4_all_of.py

@@ -0,0 +1,88 @@
+# coding: utf-8
+
+"""
+    Lightly API
+
+    Lightly.ai enables you to do self-supervised learning in an easy and intuitive way. The lightly.ai OpenAPI spec defines how one can interact with our REST API to unleash the full potential of lightly.ai  # noqa: E501
+
+    The version of the OpenAPI document: 1.0.0
+    Contact: [email protected]
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+
+from typing import List, Optional
+from pydantic import Extra,  BaseModel, Field, conlist, constr
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry import SelectionConfigV4Entry
+
+class SelectionConfigV4AllOf(BaseModel):
+    """
+    SelectionConfigV4AllOf
+    """
+    strategies: conlist(SelectionConfigV4Entry, min_items=1) = Field(...)
+    lightly_path_regex: Optional[constr(strict=True, min_length=1)] = Field(None, alias="lightlyPathRegex", description="The Lightly Path Regex to extract information from filenames for metadata balancing and more. Docs are coming soon.")
+    __properties = ["strategies", "lightlyPathRegex"]
+
+    class Config:
+        """Pydantic configuration"""
+        allow_population_by_field_name = True
+        validate_assignment = True
+        use_enum_values = True
+        extra = Extra.forbid
+
+    def to_str(self, by_alias: bool = False) -> str:
+        """Returns the string representation of the model"""
+        return pprint.pformat(self.dict(by_alias=by_alias))
+
+    def to_json(self, by_alias: bool = False) -> str:
+        """Returns the JSON representation of the model"""
+        return json.dumps(self.to_dict(by_alias=by_alias))
+
+    @classmethod
+    def from_json(cls, json_str: str) -> SelectionConfigV4AllOf:
+        """Create an instance of SelectionConfigV4AllOf from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self, by_alias: bool = False):
+        """Returns the dictionary representation of the model"""
+        _dict = self.dict(by_alias=by_alias,
+                          exclude={
+                          },
+                          exclude_none=True)
+        # override the default output from pydantic by calling `to_dict()` of each item in strategies (list)
+        _items = []
+        if self.strategies:
+            for _item in self.strategies:
+                if _item:
+                    _items.append(_item.to_dict(by_alias=by_alias))
+            _dict['strategies' if by_alias else 'strategies'] = _items
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: dict) -> SelectionConfigV4AllOf:
+        """Create an instance of SelectionConfigV4AllOf from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return SelectionConfigV4AllOf.parse_obj(obj)
+
+        # raise errors for additional fields in the input
+        for _key in obj.keys():
+            if _key not in cls.__properties:
+                raise ValueError("Error due to additional fields (not defined in SelectionConfigV4AllOf) in the input: " + str(obj))
+
+        _obj = SelectionConfigV4AllOf.parse_obj({
+            "strategies": [SelectionConfigV4Entry.from_dict(_item) for _item in obj.get("strategies")] if obj.get("strategies") is not None else None,
+            "lightly_path_regex": obj.get("lightlyPathRegex")
+        })
+        return _obj
+

lightly/openapi_generated/swagger_client/models/selection_config_v4_entry.py

@@ -0,0 +1,88 @@
+# coding: utf-8
+
+"""
+    Lightly API
+
+    Lightly.ai enables you to do self-supervised learning in an easy and intuitive way. The lightly.ai OpenAPI spec defines how one can interact with our REST API to unleash the full potential of lightly.ai  # noqa: E501
+
+    The version of the OpenAPI document: 1.0.0
+    Contact: [email protected]
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+
+
+from pydantic import Extra,  BaseModel, Field
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_input import SelectionConfigV4EntryInput
+from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_strategy import SelectionConfigV4EntryStrategy
+
+class SelectionConfigV4Entry(BaseModel):
+    """
+    SelectionConfigV4Entry
+    """
+    input: SelectionConfigV4EntryInput = Field(...)
+    strategy: SelectionConfigV4EntryStrategy = Field(...)
+    __properties = ["input", "strategy"]
+
+    class Config:
+        """Pydantic configuration"""
+        allow_population_by_field_name = True
+        validate_assignment = True
+        use_enum_values = True
+        extra = Extra.forbid
+
+    def to_str(self, by_alias: bool = False) -> str:
+        """Returns the string representation of the model"""
+        return pprint.pformat(self.dict(by_alias=by_alias))
+
+    def to_json(self, by_alias: bool = False) -> str:
+        """Returns the JSON representation of the model"""
+        return json.dumps(self.to_dict(by_alias=by_alias))
+
+    @classmethod
+    def from_json(cls, json_str: str) -> SelectionConfigV4Entry:
+        """Create an instance of SelectionConfigV4Entry from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self, by_alias: bool = False):
+        """Returns the dictionary representation of the model"""
+        _dict = self.dict(by_alias=by_alias,
+                          exclude={
+                          },
+                          exclude_none=True)
+        # override the default output from pydantic by calling `to_dict()` of input
+        if self.input:
+            _dict['input' if by_alias else 'input'] = self.input.to_dict(by_alias=by_alias)
+        # override the default output from pydantic by calling `to_dict()` of strategy
+        if self.strategy:
+            _dict['strategy' if by_alias else 'strategy'] = self.strategy.to_dict(by_alias=by_alias)
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: dict) -> SelectionConfigV4Entry:
+        """Create an instance of SelectionConfigV4Entry from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return SelectionConfigV4Entry.parse_obj(obj)
+
+        # raise errors for additional fields in the input
+        for _key in obj.keys():
+            if _key not in cls.__properties:
+                raise ValueError("Error due to additional fields (not defined in SelectionConfigV4Entry) in the input: " + str(obj))
+
+        _obj = SelectionConfigV4Entry.parse_obj({
+            "input": SelectionConfigV4EntryInput.from_dict(obj.get("input")) if obj.get("input") is not None else None,
+            "strategy": SelectionConfigV4EntryStrategy.from_dict(obj.get("strategy")) if obj.get("strategy") is not None else None
+        })
+        return _obj
+