From 9bcf6d509bcf483ca0235376ca20f4c837f791cd Mon Sep 17 00:00:00 2001 From: "Jeremy A. Prescott" Date: Wed, 14 Feb 2024 10:40:17 +0100 Subject: [PATCH] Jeremy lig 4589 lightlypathregex pip (#1501) partially closes lig-4589 - generate new specs with v4 config and lightlyPathRegex for future proofing --- lightly/api/api_workflow_compute_worker.py | 24 ++-- .../swagger_client/__init__.py | 5 + .../swagger_client/models/__init__.py | 5 + .../models/docker_worker_config_v4.py | 6 +- .../models/selection_config_v4.py | 92 ++++++++++++ .../models/selection_config_v4_all_of.py | 88 ++++++++++++ .../models/selection_config_v4_entry.py | 88 ++++++++++++ .../models/selection_config_v4_entry_input.py | 136 ++++++++++++++++++ .../selection_config_v4_entry_strategy.py | 102 +++++++++++++ .../test_api_workflow_compute_worker.py | 42 +++--- 10 files changed, 552 insertions(+), 36 deletions(-) create mode 100644 lightly/openapi_generated/swagger_client/models/selection_config_v4.py create mode 100644 lightly/openapi_generated/swagger_client/models/selection_config_v4_all_of.py create mode 100644 lightly/openapi_generated/swagger_client/models/selection_config_v4_entry.py create mode 100644 lightly/openapi_generated/swagger_client/models/selection_config_v4_entry_input.py create mode 100644 lightly/openapi_generated/swagger_client/models/selection_config_v4_entry_strategy.py diff --git a/lightly/api/api_workflow_compute_worker.py b/lightly/api/api_workflow_compute_worker.py index fe3a76bd8..4174d83c7 100644 --- a/lightly/api/api_workflow_compute_worker.py +++ b/lightly/api/api_workflow_compute_worker.py @@ -22,10 +22,10 @@ DockerWorkerConfigV4Docker, DockerWorkerRegistryEntryData, DockerWorkerType, - SelectionConfigV3, - SelectionConfigV3Entry, - SelectionConfigV3EntryInput, - SelectionConfigV3EntryStrategy, + SelectionConfigV4, + SelectionConfigV4Entry, + SelectionConfigV4EntryInput, + SelectionConfigV4EntryStrategy, TagData, ) from lightly.openapi_generated.swagger_client.rest import ApiException @@ -175,7 +175,7 @@ def create_compute_worker_config( self, worker_config: Optional[Dict[str, Any]] = None, lightly_config: Optional[Dict[str, Any]] = None, - selection_config: Optional[Union[Dict[str, Any], SelectionConfigV3]] = None, + selection_config: Optional[Union[Dict[str, Any], SelectionConfigV4]] = None, ) -> str: """Creates a new configuration for a Lightly Worker run. @@ -273,7 +273,7 @@ def schedule_compute_worker_run( self, worker_config: Optional[Dict[str, Any]] = None, lightly_config: Optional[Dict[str, Any]] = None, - selection_config: Optional[Union[Dict[str, Any], SelectionConfigV3]] = None, + selection_config: Optional[Union[Dict[str, Any], SelectionConfigV4]] = None, priority: str = DockerRunScheduledPriority.MID, runs_on: Optional[List[str]] = None, ) -> str: @@ -638,17 +638,17 @@ def get_compute_worker_run_tags(self, run_id: str) -> List[TagData]: return tags_in_dataset -def selection_config_from_dict(cfg: Dict[str, Any]) -> SelectionConfigV3: - """Recursively converts selection config from dict to a SelectionConfigV3 instance.""" +def selection_config_from_dict(cfg: Dict[str, Any]) -> SelectionConfigV4: + """Recursively converts selection config from dict to a SelectionConfigV4 instance.""" strategies = [] for entry in cfg.get("strategies", []): new_entry = copy.deepcopy(entry) - new_entry["input"] = SelectionConfigV3EntryInput(**entry["input"]) - new_entry["strategy"] = SelectionConfigV3EntryStrategy(**entry["strategy"]) - strategies.append(SelectionConfigV3Entry(**new_entry)) + new_entry["input"] = SelectionConfigV4EntryInput(**entry["input"]) + new_entry["strategy"] = SelectionConfigV4EntryStrategy(**entry["strategy"]) + strategies.append(SelectionConfigV4Entry(**new_entry)) new_cfg = copy.deepcopy(cfg) new_cfg["strategies"] = strategies - return SelectionConfigV3(**new_cfg) + return SelectionConfigV4(**new_cfg) _T = TypeVar("_T") diff --git a/lightly/openapi_generated/swagger_client/__init__.py b/lightly/openapi_generated/swagger_client/__init__.py index ba1c88005..708f32826 100644 --- a/lightly/openapi_generated/swagger_client/__init__.py +++ b/lightly/openapi_generated/swagger_client/__init__.py @@ -253,6 +253,11 @@ from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy import SelectionConfigV3EntryStrategy from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy_all_of import SelectionConfigV3EntryStrategyAllOf from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy_all_of_target_range import SelectionConfigV3EntryStrategyAllOfTargetRange +from lightly.openapi_generated.swagger_client.models.selection_config_v4 import SelectionConfigV4 +from lightly.openapi_generated.swagger_client.models.selection_config_v4_all_of import SelectionConfigV4AllOf +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry import SelectionConfigV4Entry +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_input import SelectionConfigV4EntryInput +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_strategy import SelectionConfigV4EntryStrategy from lightly.openapi_generated.swagger_client.models.selection_input_predictions_name import SelectionInputPredictionsName from lightly.openapi_generated.swagger_client.models.selection_input_type import SelectionInputType from lightly.openapi_generated.swagger_client.models.selection_strategy_threshold_operation import SelectionStrategyThresholdOperation diff --git a/lightly/openapi_generated/swagger_client/models/__init__.py b/lightly/openapi_generated/swagger_client/models/__init__.py index e4622cfb8..fd3becb3c 100644 --- a/lightly/openapi_generated/swagger_client/models/__init__.py +++ b/lightly/openapi_generated/swagger_client/models/__init__.py @@ -220,6 +220,11 @@ from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy import SelectionConfigV3EntryStrategy from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy_all_of import SelectionConfigV3EntryStrategyAllOf from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy_all_of_target_range import SelectionConfigV3EntryStrategyAllOfTargetRange +from lightly.openapi_generated.swagger_client.models.selection_config_v4 import SelectionConfigV4 +from lightly.openapi_generated.swagger_client.models.selection_config_v4_all_of import SelectionConfigV4AllOf +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry import SelectionConfigV4Entry +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_input import SelectionConfigV4EntryInput +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_strategy import SelectionConfigV4EntryStrategy from lightly.openapi_generated.swagger_client.models.selection_input_predictions_name import SelectionInputPredictionsName from lightly.openapi_generated.swagger_client.models.selection_input_type import SelectionInputType from lightly.openapi_generated.swagger_client.models.selection_strategy_threshold_operation import SelectionStrategyThresholdOperation diff --git a/lightly/openapi_generated/swagger_client/models/docker_worker_config_v4.py b/lightly/openapi_generated/swagger_client/models/docker_worker_config_v4.py index 9e1d632bd..678612140 100644 --- a/lightly/openapi_generated/swagger_client/models/docker_worker_config_v4.py +++ b/lightly/openapi_generated/swagger_client/models/docker_worker_config_v4.py @@ -24,7 +24,7 @@ from lightly.openapi_generated.swagger_client.models.docker_worker_config_v3_lightly import DockerWorkerConfigV3Lightly from lightly.openapi_generated.swagger_client.models.docker_worker_config_v4_docker import DockerWorkerConfigV4Docker from lightly.openapi_generated.swagger_client.models.docker_worker_type import DockerWorkerType -from lightly.openapi_generated.swagger_client.models.selection_config_v3 import SelectionConfigV3 +from lightly.openapi_generated.swagger_client.models.selection_config_v4 import SelectionConfigV4 class DockerWorkerConfigV4(BaseModel): """ @@ -33,7 +33,7 @@ class DockerWorkerConfigV4(BaseModel): worker_type: DockerWorkerType = Field(..., alias="workerType") docker: Optional[DockerWorkerConfigV4Docker] = None lightly: Optional[DockerWorkerConfigV3Lightly] = None - selection: Optional[SelectionConfigV3] = None + selection: Optional[SelectionConfigV4] = None __properties = ["workerType", "docker", "lightly", "selection"] class Config: @@ -91,7 +91,7 @@ def from_dict(cls, obj: dict) -> DockerWorkerConfigV4: "worker_type": obj.get("workerType"), "docker": DockerWorkerConfigV4Docker.from_dict(obj.get("docker")) if obj.get("docker") is not None else None, "lightly": DockerWorkerConfigV3Lightly.from_dict(obj.get("lightly")) if obj.get("lightly") is not None else None, - "selection": SelectionConfigV3.from_dict(obj.get("selection")) if obj.get("selection") is not None else None + "selection": SelectionConfigV4.from_dict(obj.get("selection")) if obj.get("selection") is not None else None }) return _obj diff --git a/lightly/openapi_generated/swagger_client/models/selection_config_v4.py b/lightly/openapi_generated/swagger_client/models/selection_config_v4.py new file mode 100644 index 000000000..75ecb7e82 --- /dev/null +++ b/lightly/openapi_generated/swagger_client/models/selection_config_v4.py @@ -0,0 +1,92 @@ +# coding: utf-8 + +""" + Lightly API + + Lightly.ai enables you to do self-supervised learning in an easy and intuitive way. The lightly.ai OpenAPI spec defines how one can interact with our REST API to unleash the full potential of lightly.ai # noqa: E501 + + The version of the OpenAPI document: 1.0.0 + Contact: support@lightly.ai + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + + +from typing import List, Optional, Union +from pydantic import Extra, BaseModel, Field, confloat, conint, conlist, constr +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry import SelectionConfigV4Entry + +class SelectionConfigV4(BaseModel): + """ + SelectionConfigV4 + """ + n_samples: Optional[conint(strict=True, ge=-1)] = Field(None, alias="nSamples") + proportion_samples: Optional[Union[confloat(le=1.0, ge=0.0, strict=True), conint(le=1, ge=0, strict=True)]] = Field(None, alias="proportionSamples") + strategies: conlist(SelectionConfigV4Entry, min_items=1) = Field(...) + lightly_path_regex: Optional[constr(strict=True, min_length=1)] = Field(None, alias="lightlyPathRegex", description="The Lightly Path Regex to extract information from filenames for metadata balancing and more. Docs are coming soon.") + __properties = ["nSamples", "proportionSamples", "strategies", "lightlyPathRegex"] + + class Config: + """Pydantic configuration""" + allow_population_by_field_name = True + validate_assignment = True + use_enum_values = True + extra = Extra.forbid + + def to_str(self, by_alias: bool = False) -> str: + """Returns the string representation of the model""" + return pprint.pformat(self.dict(by_alias=by_alias)) + + def to_json(self, by_alias: bool = False) -> str: + """Returns the JSON representation of the model""" + return json.dumps(self.to_dict(by_alias=by_alias)) + + @classmethod + def from_json(cls, json_str: str) -> SelectionConfigV4: + """Create an instance of SelectionConfigV4 from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self, by_alias: bool = False): + """Returns the dictionary representation of the model""" + _dict = self.dict(by_alias=by_alias, + exclude={ + }, + exclude_none=True) + # override the default output from pydantic by calling `to_dict()` of each item in strategies (list) + _items = [] + if self.strategies: + for _item in self.strategies: + if _item: + _items.append(_item.to_dict(by_alias=by_alias)) + _dict['strategies' if by_alias else 'strategies'] = _items + return _dict + + @classmethod + def from_dict(cls, obj: dict) -> SelectionConfigV4: + """Create an instance of SelectionConfigV4 from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return SelectionConfigV4.parse_obj(obj) + + # raise errors for additional fields in the input + for _key in obj.keys(): + if _key not in cls.__properties: + raise ValueError("Error due to additional fields (not defined in SelectionConfigV4) in the input: " + str(obj)) + + _obj = SelectionConfigV4.parse_obj({ + "n_samples": obj.get("nSamples"), + "proportion_samples": obj.get("proportionSamples"), + "strategies": [SelectionConfigV4Entry.from_dict(_item) for _item in obj.get("strategies")] if obj.get("strategies") is not None else None, + "lightly_path_regex": obj.get("lightlyPathRegex") + }) + return _obj + diff --git a/lightly/openapi_generated/swagger_client/models/selection_config_v4_all_of.py b/lightly/openapi_generated/swagger_client/models/selection_config_v4_all_of.py new file mode 100644 index 000000000..2081502e6 --- /dev/null +++ b/lightly/openapi_generated/swagger_client/models/selection_config_v4_all_of.py @@ -0,0 +1,88 @@ +# coding: utf-8 + +""" + Lightly API + + Lightly.ai enables you to do self-supervised learning in an easy and intuitive way. The lightly.ai OpenAPI spec defines how one can interact with our REST API to unleash the full potential of lightly.ai # noqa: E501 + + The version of the OpenAPI document: 1.0.0 + Contact: support@lightly.ai + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + + +from typing import List, Optional +from pydantic import Extra, BaseModel, Field, conlist, constr +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry import SelectionConfigV4Entry + +class SelectionConfigV4AllOf(BaseModel): + """ + SelectionConfigV4AllOf + """ + strategies: conlist(SelectionConfigV4Entry, min_items=1) = Field(...) + lightly_path_regex: Optional[constr(strict=True, min_length=1)] = Field(None, alias="lightlyPathRegex", description="The Lightly Path Regex to extract information from filenames for metadata balancing and more. Docs are coming soon.") + __properties = ["strategies", "lightlyPathRegex"] + + class Config: + """Pydantic configuration""" + allow_population_by_field_name = True + validate_assignment = True + use_enum_values = True + extra = Extra.forbid + + def to_str(self, by_alias: bool = False) -> str: + """Returns the string representation of the model""" + return pprint.pformat(self.dict(by_alias=by_alias)) + + def to_json(self, by_alias: bool = False) -> str: + """Returns the JSON representation of the model""" + return json.dumps(self.to_dict(by_alias=by_alias)) + + @classmethod + def from_json(cls, json_str: str) -> SelectionConfigV4AllOf: + """Create an instance of SelectionConfigV4AllOf from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self, by_alias: bool = False): + """Returns the dictionary representation of the model""" + _dict = self.dict(by_alias=by_alias, + exclude={ + }, + exclude_none=True) + # override the default output from pydantic by calling `to_dict()` of each item in strategies (list) + _items = [] + if self.strategies: + for _item in self.strategies: + if _item: + _items.append(_item.to_dict(by_alias=by_alias)) + _dict['strategies' if by_alias else 'strategies'] = _items + return _dict + + @classmethod + def from_dict(cls, obj: dict) -> SelectionConfigV4AllOf: + """Create an instance of SelectionConfigV4AllOf from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return SelectionConfigV4AllOf.parse_obj(obj) + + # raise errors for additional fields in the input + for _key in obj.keys(): + if _key not in cls.__properties: + raise ValueError("Error due to additional fields (not defined in SelectionConfigV4AllOf) in the input: " + str(obj)) + + _obj = SelectionConfigV4AllOf.parse_obj({ + "strategies": [SelectionConfigV4Entry.from_dict(_item) for _item in obj.get("strategies")] if obj.get("strategies") is not None else None, + "lightly_path_regex": obj.get("lightlyPathRegex") + }) + return _obj + diff --git a/lightly/openapi_generated/swagger_client/models/selection_config_v4_entry.py b/lightly/openapi_generated/swagger_client/models/selection_config_v4_entry.py new file mode 100644 index 000000000..b2d584866 --- /dev/null +++ b/lightly/openapi_generated/swagger_client/models/selection_config_v4_entry.py @@ -0,0 +1,88 @@ +# coding: utf-8 + +""" + Lightly API + + Lightly.ai enables you to do self-supervised learning in an easy and intuitive way. The lightly.ai OpenAPI spec defines how one can interact with our REST API to unleash the full potential of lightly.ai # noqa: E501 + + The version of the OpenAPI document: 1.0.0 + Contact: support@lightly.ai + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + + + +from pydantic import Extra, BaseModel, Field +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_input import SelectionConfigV4EntryInput +from lightly.openapi_generated.swagger_client.models.selection_config_v4_entry_strategy import SelectionConfigV4EntryStrategy + +class SelectionConfigV4Entry(BaseModel): + """ + SelectionConfigV4Entry + """ + input: SelectionConfigV4EntryInput = Field(...) + strategy: SelectionConfigV4EntryStrategy = Field(...) + __properties = ["input", "strategy"] + + class Config: + """Pydantic configuration""" + allow_population_by_field_name = True + validate_assignment = True + use_enum_values = True + extra = Extra.forbid + + def to_str(self, by_alias: bool = False) -> str: + """Returns the string representation of the model""" + return pprint.pformat(self.dict(by_alias=by_alias)) + + def to_json(self, by_alias: bool = False) -> str: + """Returns the JSON representation of the model""" + return json.dumps(self.to_dict(by_alias=by_alias)) + + @classmethod + def from_json(cls, json_str: str) -> SelectionConfigV4Entry: + """Create an instance of SelectionConfigV4Entry from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self, by_alias: bool = False): + """Returns the dictionary representation of the model""" + _dict = self.dict(by_alias=by_alias, + exclude={ + }, + exclude_none=True) + # override the default output from pydantic by calling `to_dict()` of input + if self.input: + _dict['input' if by_alias else 'input'] = self.input.to_dict(by_alias=by_alias) + # override the default output from pydantic by calling `to_dict()` of strategy + if self.strategy: + _dict['strategy' if by_alias else 'strategy'] = self.strategy.to_dict(by_alias=by_alias) + return _dict + + @classmethod + def from_dict(cls, obj: dict) -> SelectionConfigV4Entry: + """Create an instance of SelectionConfigV4Entry from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return SelectionConfigV4Entry.parse_obj(obj) + + # raise errors for additional fields in the input + for _key in obj.keys(): + if _key not in cls.__properties: + raise ValueError("Error due to additional fields (not defined in SelectionConfigV4Entry) in the input: " + str(obj)) + + _obj = SelectionConfigV4Entry.parse_obj({ + "input": SelectionConfigV4EntryInput.from_dict(obj.get("input")) if obj.get("input") is not None else None, + "strategy": SelectionConfigV4EntryStrategy.from_dict(obj.get("strategy")) if obj.get("strategy") is not None else None + }) + return _obj + diff --git a/lightly/openapi_generated/swagger_client/models/selection_config_v4_entry_input.py b/lightly/openapi_generated/swagger_client/models/selection_config_v4_entry_input.py new file mode 100644 index 000000000..73e239a05 --- /dev/null +++ b/lightly/openapi_generated/swagger_client/models/selection_config_v4_entry_input.py @@ -0,0 +1,136 @@ +# coding: utf-8 + +""" + Lightly API + + Lightly.ai enables you to do self-supervised learning in an easy and intuitive way. The lightly.ai OpenAPI spec defines how one can interact with our REST API to unleash the full potential of lightly.ai # noqa: E501 + + The version of the OpenAPI document: 1.0.0 + Contact: support@lightly.ai + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + + +from typing import List, Optional +from pydantic import Extra, BaseModel, Field, StrictInt, conlist, constr, validator +from lightly.openapi_generated.swagger_client.models.selection_input_predictions_name import SelectionInputPredictionsName +from lightly.openapi_generated.swagger_client.models.selection_input_type import SelectionInputType + +class SelectionConfigV4EntryInput(BaseModel): + """ + SelectionConfigV4EntryInput + """ + type: SelectionInputType = Field(...) + task: Optional[constr(strict=True)] = Field(None, description="Since we sometimes stitch together SelectionInputTask+ActiveLearningScoreType, they need to follow the same specs of ActiveLearningScoreType. However, this can be an empty string due to internal logic. ") + score: Optional[constr(strict=True, min_length=1)] = Field(None, description="Type of active learning score") + key: Optional[constr(strict=True, min_length=1)] = None + name: Optional[SelectionInputPredictionsName] = None + dataset_id: Optional[constr(strict=True)] = Field(None, alias="datasetId", description="MongoDB ObjectId") + tag_name: Optional[constr(strict=True, min_length=3)] = Field(None, alias="tagName", description="The name of the tag") + random_seed: Optional[StrictInt] = Field(None, alias="randomSeed") + categories: Optional[conlist(constr(strict=True, min_length=1), min_items=1, unique_items=True)] = None + __properties = ["type", "task", "score", "key", "name", "datasetId", "tagName", "randomSeed", "categories"] + + @validator('task') + def task_validate_regular_expression(cls, value): + """Validates the regular expression""" + if value is None: + return value + + if not re.match(r"^[a-zA-Z0-9_+=,.@:\/-]*$", value): + raise ValueError(r"must validate the regular expression /^[a-zA-Z0-9_+=,.@:\/-]*$/") + return value + + @validator('score') + def score_validate_regular_expression(cls, value): + """Validates the regular expression""" + if value is None: + return value + + if not re.match(r"^[a-zA-Z0-9_+=,.@:\/-]*$", value): + raise ValueError(r"must validate the regular expression /^[a-zA-Z0-9_+=,.@:\/-]*$/") + return value + + @validator('dataset_id') + def dataset_id_validate_regular_expression(cls, value): + """Validates the regular expression""" + if value is None: + return value + + if not re.match(r"^[a-f0-9]{24}$", value): + raise ValueError(r"must validate the regular expression /^[a-f0-9]{24}$/") + return value + + @validator('tag_name') + def tag_name_validate_regular_expression(cls, value): + """Validates the regular expression""" + if value is None: + return value + + if not re.match(r"^[a-zA-Z0-9][a-zA-Z0-9 .:;=@_-]+$", value): + raise ValueError(r"must validate the regular expression /^[a-zA-Z0-9][a-zA-Z0-9 .:;=@_-]+$/") + return value + + class Config: + """Pydantic configuration""" + allow_population_by_field_name = True + validate_assignment = True + use_enum_values = True + extra = Extra.forbid + + def to_str(self, by_alias: bool = False) -> str: + """Returns the string representation of the model""" + return pprint.pformat(self.dict(by_alias=by_alias)) + + def to_json(self, by_alias: bool = False) -> str: + """Returns the JSON representation of the model""" + return json.dumps(self.to_dict(by_alias=by_alias)) + + @classmethod + def from_json(cls, json_str: str) -> SelectionConfigV4EntryInput: + """Create an instance of SelectionConfigV4EntryInput from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self, by_alias: bool = False): + """Returns the dictionary representation of the model""" + _dict = self.dict(by_alias=by_alias, + exclude={ + }, + exclude_none=True) + return _dict + + @classmethod + def from_dict(cls, obj: dict) -> SelectionConfigV4EntryInput: + """Create an instance of SelectionConfigV4EntryInput from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return SelectionConfigV4EntryInput.parse_obj(obj) + + # raise errors for additional fields in the input + for _key in obj.keys(): + if _key not in cls.__properties: + raise ValueError("Error due to additional fields (not defined in SelectionConfigV4EntryInput) in the input: " + str(obj)) + + _obj = SelectionConfigV4EntryInput.parse_obj({ + "type": obj.get("type"), + "task": obj.get("task"), + "score": obj.get("score"), + "key": obj.get("key"), + "name": obj.get("name"), + "dataset_id": obj.get("datasetId"), + "tag_name": obj.get("tagName"), + "random_seed": obj.get("randomSeed"), + "categories": obj.get("categories") + }) + return _obj + diff --git a/lightly/openapi_generated/swagger_client/models/selection_config_v4_entry_strategy.py b/lightly/openapi_generated/swagger_client/models/selection_config_v4_entry_strategy.py new file mode 100644 index 000000000..5f12dac71 --- /dev/null +++ b/lightly/openapi_generated/swagger_client/models/selection_config_v4_entry_strategy.py @@ -0,0 +1,102 @@ +# coding: utf-8 + +""" + Lightly API + + Lightly.ai enables you to do self-supervised learning in an easy and intuitive way. The lightly.ai OpenAPI spec defines how one can interact with our REST API to unleash the full potential of lightly.ai # noqa: E501 + + The version of the OpenAPI document: 1.0.0 + Contact: support@lightly.ai + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + + +from typing import Any, Dict, Optional, Union +from pydantic import Extra, BaseModel, Field, StrictFloat, StrictInt, confloat, conint +from lightly.openapi_generated.swagger_client.models.selection_config_v3_entry_strategy_all_of_target_range import SelectionConfigV3EntryStrategyAllOfTargetRange +from lightly.openapi_generated.swagger_client.models.selection_strategy_threshold_operation import SelectionStrategyThresholdOperation +from lightly.openapi_generated.swagger_client.models.selection_strategy_type_v3 import SelectionStrategyTypeV3 + +class SelectionConfigV4EntryStrategy(BaseModel): + """ + SelectionConfigV4EntryStrategy + """ + type: SelectionStrategyTypeV3 = Field(...) + stopping_condition_minimum_distance: Optional[Union[StrictFloat, StrictInt]] = None + threshold: Optional[Union[StrictFloat, StrictInt]] = None + operation: Optional[SelectionStrategyThresholdOperation] = None + target: Optional[Dict[str, Any]] = None + num_nearest_neighbors: Optional[Union[confloat(ge=2, strict=True), conint(ge=2, strict=True)]] = Field(None, alias="numNearestNeighbors", description="It is the number of nearest datapoints used to compute the typicality of each sample. ") + stopping_condition_minimum_typicality: Optional[Union[confloat(gt=0, strict=True), conint(gt=0, strict=True)]] = Field(None, alias="stoppingConditionMinimumTypicality", description="It is the minimal allowed typicality of the selected samples. When the typicality of the selected samples reaches this, the selection stops. It should be a number between 0 and 1. ") + strength: Optional[Union[confloat(le=1000000000, ge=-1000000000, strict=True), conint(le=1000000000, ge=-1000000000, strict=True)]] = Field(None, description="The relative strength of this strategy compared to other strategies. The default value is 1.0, which is set in the worker for backwards compatibility. The minimum and maximum values of +-10^9 are used to prevent numerical issues. ") + stopping_condition_max_sum: Optional[Union[confloat(ge=0.0, strict=True), conint(ge=0, strict=True)]] = Field(None, alias="stoppingConditionMaxSum", description="When the sum of inputs reaches this, the selection stops. Only compatible with the WEIGHTS strategy. Similar to the stopping_condition_minimum_distance for the DIVERSITY strategy. ") + target_range: Optional[SelectionConfigV3EntryStrategyAllOfTargetRange] = Field(None, alias="targetRange") + __properties = ["type", "stopping_condition_minimum_distance", "threshold", "operation", "target", "numNearestNeighbors", "stoppingConditionMinimumTypicality", "strength", "stoppingConditionMaxSum", "targetRange"] + + class Config: + """Pydantic configuration""" + allow_population_by_field_name = True + validate_assignment = True + use_enum_values = True + extra = Extra.forbid + + def to_str(self, by_alias: bool = False) -> str: + """Returns the string representation of the model""" + return pprint.pformat(self.dict(by_alias=by_alias)) + + def to_json(self, by_alias: bool = False) -> str: + """Returns the JSON representation of the model""" + return json.dumps(self.to_dict(by_alias=by_alias)) + + @classmethod + def from_json(cls, json_str: str) -> SelectionConfigV4EntryStrategy: + """Create an instance of SelectionConfigV4EntryStrategy from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self, by_alias: bool = False): + """Returns the dictionary representation of the model""" + _dict = self.dict(by_alias=by_alias, + exclude={ + }, + exclude_none=True) + # override the default output from pydantic by calling `to_dict()` of target_range + if self.target_range: + _dict['targetRange' if by_alias else 'target_range'] = self.target_range.to_dict(by_alias=by_alias) + return _dict + + @classmethod + def from_dict(cls, obj: dict) -> SelectionConfigV4EntryStrategy: + """Create an instance of SelectionConfigV4EntryStrategy from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return SelectionConfigV4EntryStrategy.parse_obj(obj) + + # raise errors for additional fields in the input + for _key in obj.keys(): + if _key not in cls.__properties: + raise ValueError("Error due to additional fields (not defined in SelectionConfigV4EntryStrategy) in the input: " + str(obj)) + + _obj = SelectionConfigV4EntryStrategy.parse_obj({ + "type": obj.get("type"), + "stopping_condition_minimum_distance": obj.get("stopping_condition_minimum_distance"), + "threshold": obj.get("threshold"), + "operation": obj.get("operation"), + "target": obj.get("target"), + "num_nearest_neighbors": obj.get("numNearestNeighbors"), + "stopping_condition_minimum_typicality": obj.get("stoppingConditionMinimumTypicality"), + "strength": obj.get("strength"), + "stopping_condition_max_sum": obj.get("stoppingConditionMaxSum"), + "target_range": SelectionConfigV3EntryStrategyAllOfTargetRange.from_dict(obj.get("targetRange")) if obj.get("targetRange") is not None else None + }) + return _obj + diff --git a/tests/api_workflow/test_api_workflow_compute_worker.py b/tests/api_workflow/test_api_workflow_compute_worker.py index 3872cdbea..f5acd7ca1 100644 --- a/tests/api_workflow/test_api_workflow_compute_worker.py +++ b/tests/api_workflow/test_api_workflow_compute_worker.py @@ -33,10 +33,10 @@ DockerWorkerConfigV4, DockerWorkerState, DockerWorkerType, - SelectionConfigV3, - SelectionConfigV3Entry, - SelectionConfigV3EntryInput, - SelectionConfigV3EntryStrategy, + SelectionConfigV4, + SelectionConfigV4Entry, + SelectionConfigV4EntryInput, + SelectionConfigV4EntryStrategy, SelectionInputPredictionsName, SelectionInputType, SelectionStrategyThresholdOperation, @@ -101,16 +101,16 @@ def test_create_compute_worker_config__selection_config_is_class(self) -> None: "batch_size": 64, }, }, - selection_config=SelectionConfigV3( + selection_config=SelectionConfigV4( n_samples=20, strategies=[ - SelectionConfigV3Entry( - input=SelectionConfigV3EntryInput( + SelectionConfigV4Entry( + input=SelectionConfigV4EntryInput( type=SelectionInputType.EMBEDDINGS, dataset_id=utils.generate_id(), tag_name="some-tag-name", ), - strategy=SelectionConfigV3EntryStrategy( + strategy=SelectionConfigV4EntryStrategy( type=SelectionStrategyTypeV3.SIMILARITY, ), ) @@ -203,45 +203,45 @@ def _check_if_openapi_generated_obj_is_valid(self, obj) -> Any: return obj_api def test_selection_config(self): - selection_config = SelectionConfigV3( + selection_config = SelectionConfigV4( n_samples=1, strategies=[ - SelectionConfigV3Entry( - input=SelectionConfigV3EntryInput( + SelectionConfigV4Entry( + input=SelectionConfigV4EntryInput( type=SelectionInputType.EMBEDDINGS ), - strategy=SelectionConfigV3EntryStrategy( + strategy=SelectionConfigV4EntryStrategy( type=SelectionStrategyTypeV3.DIVERSITY, stopping_condition_minimum_distance=-1, ), ), - SelectionConfigV3Entry( - input=SelectionConfigV3EntryInput( + SelectionConfigV4Entry( + input=SelectionConfigV4EntryInput( type=SelectionInputType.SCORES, task="my-classification-task", score="uncertainty_margin", ), - strategy=SelectionConfigV3EntryStrategy( + strategy=SelectionConfigV4EntryStrategy( type=SelectionStrategyTypeV3.WEIGHTS ), ), - SelectionConfigV3Entry( - input=SelectionConfigV3EntryInput( + SelectionConfigV4Entry( + input=SelectionConfigV4EntryInput( type=SelectionInputType.METADATA, key="lightly.sharpness" ), - strategy=SelectionConfigV3EntryStrategy( + strategy=SelectionConfigV4EntryStrategy( type=SelectionStrategyTypeV3.THRESHOLD, threshold=20, operation=SelectionStrategyThresholdOperation.BIGGER_EQUAL, ), ), - SelectionConfigV3Entry( - input=SelectionConfigV3EntryInput( + SelectionConfigV4Entry( + input=SelectionConfigV4EntryInput( type=SelectionInputType.PREDICTIONS, task="my_object_detection_task", name=SelectionInputPredictionsName.CLASS_DISTRIBUTION, ), - strategy=SelectionConfigV3EntryStrategy( + strategy=SelectionConfigV4EntryStrategy( type=SelectionStrategyTypeV3.BALANCE, target={"Ambulance": 0.2, "Bus": 0.4}, ),