Skip to content

Commit

Permalink
Raise error for non existing config options (#1090)
Browse files Browse the repository at this point in the history 
* Do basic validation of configuration options when scheduling jobs.
* Print closest match if configuration option doesn't exist.
  • Loading branch information
@philippmwirth
philippmwirth authored Mar 7, 2023
1 parent eace6a5 commit 24a815a
Show file tree
Hide file tree
Showing 2 changed files with 205 additions and 10 deletions.
57 changes: 51 additions & 6 deletions lightly/api/api_workflow_compute_worker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import dataclasses
from functools import partial
import time
import difflib
from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union, Iterator

from lightly.api.utils import retry
Expand Down Expand Up @@ -32,6 +33,9 @@
STATE_SCHEDULED_ID_NOT_FOUND = "CANCELED_OR_NOT_EXISTING"


class InvalidConfigurationError(RuntimeError):
pass


@dataclasses.dataclass
class ComputeWorkerRunInfo:
Expand Down Expand Up @@ -142,27 +146,30 @@ def create_compute_worker_config(
selection = selection_config_from_dict(cfg=selection_config)
else:
selection = selection_config
_validate_config(cfg=selection_config, obj=selection)

if worker_config is not None:
worker_config = _config_to_camel_case(cfg=worker_config)
worker_config_cc = _config_to_camel_case(cfg=worker_config)
deserialize_worker_config = _get_deserialize(
api_client=self.api_client,
klass=DockerWorkerConfigV2Docker,
)
worker_config = deserialize_worker_config(worker_config)
docker = deserialize_worker_config(worker_config_cc)
_validate_config(cfg=worker_config, obj=docker)

if lightly_config is not None:
lightly_config = _config_to_camel_case(cfg=lightly_config)
lightly_config_cc = _config_to_camel_case(cfg=lightly_config)
deserialize_lightly_config = _get_deserialize(
api_client=self.api_client,
klass=DockerWorkerConfigV2Lightly,
)
lightly_config = deserialize_lightly_config(lightly_config)
lightly = deserialize_lightly_config(lightly_config_cc)
_validate_config(cfg=lightly_config, obj=lightly)

config = DockerWorkerConfigV2(
worker_type=DockerWorkerType.FULL,
docker=worker_config,
lightly=lightly_config,
docker=docker,
lightly=lightly,
selection=selection,
)
request = DockerWorkerConfigV2CreateRequest(config=config, creator=self._creator)
Expand Down Expand Up @@ -489,3 +496,41 @@ def _snake_to_camel_case(snake: str) -> str:
return components[0] + "".join(
component.title() for component in components[1:]
)


def _validate_config(
cfg: Optional[Dict[str, Any]],
obj: Any,
) -> None:
"""Validates that all keys in cfg are legitimate configuration options.
Recursively checks if the keys in the cfg dictionary match the attributes of
the DockerWorkerConfigV2Docker/DockerWorkerConfigV2Lightly instances. If not,
suggests a best match based on the keys in 'swagger_types'.
Raises:
TypeError: If obj is not of swagger type.
"""

if cfg is None:
return

if not hasattr(type(obj), "swagger_types"):
raise TypeError(
f"Type {type(obj)} of argument 'obj' has not attribute 'swagger_types'"
)

for key, item in cfg.items():
if not hasattr(obj, key):
possible_options = list(type(obj).swagger_types.keys())
closest_match = difflib.get_close_matches(
word=key,
possibilities=possible_options,
n=1,
cutoff=0.
)[0]
error_msg = f"Option '{key}' does not exist! Did you mean '{closest_match}'?"
raise InvalidConfigurationError(error_msg)
if isinstance(item, dict):
_validate_config(item, getattr(obj, key))
158 changes: 154 additions & 4 deletions tests/api_workflow/test_api_workflow_compute_worker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
ComputeWorkerRunInfo,
_config_to_camel_case,
_snake_to_camel_case,
_validate_config,
InvalidConfigurationError,
)
from lightly.openapi_generated.swagger_client import (
SelectionConfig,
Expand All @@ -31,6 +33,10 @@
DockerRunScheduledPriority,
DockerRunScheduledState,
DockerRunState,
DockerWorkerConfigV2Docker,
DockerWorkerConfigV2DockerCorruptnessCheck,
DockerWorkerConfigV2Lightly,
DockerWorkerConfigV2LightlyLoader,
TagData,
)
from lightly.openapi_generated.swagger_client.rest import ApiException
Expand All @@ -55,13 +61,11 @@ def test_delete_compute_worker(self):
def test_create_compute_worker_config(self):
config_id = self.api_workflow_client.create_compute_worker_config(
worker_config={
"enable_corruptness_check": True,
"stopping_condition": {
"n_samples": 10,
},
},
lightly_config={
"resize": 224,
"loader": {
"batch_size": 64,
},
Expand All @@ -85,13 +89,11 @@ def test_create_compute_worker_config(self):
def test_schedule_compute_worker_run(self):
scheduled_run_id = self.api_workflow_client.schedule_compute_worker_run(
worker_config={
"enable_corruptness_check": True,
"stopping_condition": {
"n_samples": 10,
},
},
lightly_config={
"resize": 224,
"loader": {
"batch_size": 64,
},
Expand Down Expand Up @@ -669,3 +671,151 @@ def test__snake_to_camel_case() -> None:
assert _snake_to_camel_case("lorem_ipsum") == "loremIpsum"
assert _snake_to_camel_case("lorem_ipsum_dolor") == "loremIpsumDolor"
assert _snake_to_camel_case("loremIpsum") == "loremIpsum" # do nothing


def test__validate_config__docker(mocker: MockerFixture) -> None:

obj = DockerWorkerConfigV2Docker(
enable_training=False,
corruptness_check=DockerWorkerConfigV2DockerCorruptnessCheck(
corruption_threshold=0.1,
)
)
_validate_config(
cfg={
"enable_training": False,
"corruptness_check": {
"corruption_threshold": 0.1,
},
},
obj=obj
)


def test__validate_config__docker_typo(mocker: MockerFixture) -> None:

obj = DockerWorkerConfigV2Docker(
enable_training=False,
corruptness_check=DockerWorkerConfigV2DockerCorruptnessCheck(
corruption_threshold=0.1,
)
)

with pytest.raises(
InvalidConfigurationError,
match="Option 'enable_trainingx' does not exist! Did you mean 'enable_training'?"
):
_validate_config(
cfg={
"enable_trainingx": False,
"corruptness_check": {
"corruption_threshold": 0.1,
},
},
obj=obj
)

def test__validate_config__docker_typo_nested(mocker: MockerFixture) -> None:

obj = DockerWorkerConfigV2Docker(
enable_training=False,
corruptness_check=DockerWorkerConfigV2DockerCorruptnessCheck(
corruption_threshold=0.1,
)
)

with pytest.raises(
InvalidConfigurationError,
match="Option 'corruption_thresholdx' does not exist! Did you mean 'corruption_threshold'?"
):
_validate_config(
cfg={
"enable_training": False,
"corruptness_check": {
"corruption_thresholdx": 0.1,
},
},
obj=obj
)


def test__validate_config__lightly(mocker: MockerFixture) -> None:

obj = DockerWorkerConfigV2Lightly(
loader=DockerWorkerConfigV2LightlyLoader(
num_workers=-1,
batch_size=16,
shuffle=True,
)
)
_validate_config(
cfg={
"loader": {
"num_workers": -1,
"batch_size": 16,
"shuffle": True,
},
},
obj=obj
)


def test__validate_config__lightly_typo(mocker: MockerFixture) -> None:

obj = DockerWorkerConfigV2Lightly(
loader=DockerWorkerConfigV2LightlyLoader(
num_workers=-1,
batch_size=16,
shuffle=True,
)
)
with pytest.raises(
InvalidConfigurationError,
match="Option 'loaderx' does not exist! Did you mean 'loader'?"
):
_validate_config(
cfg={
"loaderx": {
"num_workers": -1,
"batch_size": 16,
"shuffle": True,
},
},
obj=obj
)


def test__validate_config__lightly_typo_nested(mocker: MockerFixture) -> None:

obj = DockerWorkerConfigV2Lightly(
loader=DockerWorkerConfigV2LightlyLoader(
num_workers=-1,
batch_size=16,
shuffle=True,
)
)
with pytest.raises(
InvalidConfigurationError,
match="Option 'num_workersx' does not exist! Did you mean 'num_workers'?"
):
_validate_config(
cfg={
"loader": {
"num_workersx": -1,
"batch_size": 16,
"shuffle": True,
},
},
obj=obj
)


def test__validate_config__raises_type_error(mocker: MockerFixture) -> None:
with pytest.raises(
TypeError,
match="of argument 'obj' has not attribute 'swagger_types'"
):
_validate_config(
cfg={},
obj=mocker.MagicMock()
)

0 comments on commit 24a815a

Please sign in to comment.