From cd8e13f26dc5eb86d068fbadc077e0586dac781b Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sat, 24 Jun 2023 03:33:37 +0200 Subject: [PATCH 01/66] feat: python project scaffold --- LICENSE.txt | 21 +++ pyproject.toml | 172 ++++++++++++++++++ .../asset_validation_schemas/__about__.py | 4 + .../asset_validation_schemas/__init__.py | 0 tests/__init__.py | 3 + 5 files changed, 200 insertions(+) create mode 100644 LICENSE.txt create mode 100644 pyproject.toml create mode 100644 src/readyplayerme/asset_validation_schemas/__about__.py create mode 100644 src/readyplayerme/asset_validation_schemas/__init__.py create mode 100644 tests/__init__.py diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..72b09ef --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023-present Ready Player Me + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..51c1385 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,172 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "readyplayerme-asset-validation-schemas" +dynamic = ["version"] +description = 'Data models that are used to check data gathered from assets for compatibility with the Ready Player Me avatar platform.' +readme = "README.md" +requires-python = ">=3.10" +license = "MIT" +keywords = ["gltf", "3D", "json", "schema", "validation"] +authors = [ + { name = "Ready Player Me", email = "info@readyplayer.me" }, +] +maintainers = [ + { name = "Olaf Haag", email = "olaf@readyplayer.me" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [ + "pydantic" +] + +[project.optional-dependencies] +tests = [ + "pytest", +] +dev = [ + "readyplayerme-asset-validation-schemas[tests]", + "pre-commit", +] + +[project.urls] +Documentation = "https://github.com/wolfprint3d/content-validation-schemas#readme" +Issues = "https://github.com/wolfprint3d/content-validation-schemas/issues" +Source = "https://github.com/wolfprint3d/content-validation-schemas" + +[tool.hatch.version] +path = "src/readyplayerme/asset_validation_schemas/__about__.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/readyplayerme"] + +[tool.hatch.envs.default] +dependencies = [ + "coverage[toml]>=6.5", + "pytest", +] + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "coverage run -m pytest {args:tests}" +cov-report = [ + "- coverage combine", + "coverage report", +] +cov = [ + "test-cov", + "cov-report", +] + +[[tool.hatch.envs.all.matrix]] +python = ["3.10", "3.11"] + +[tool.hatch.envs.lint] +detached = true +dependencies = [ + "black>=23.3.0", + "mypy>=1.3.0", + "ruff>=0.0.275", +] +[tool.hatch.envs.lint.scripts] +typing = "mypy --install-types --non-interactive {args:src/readyplayerme/asset_validation_schemas tests}" +style = [ + "ruff {args:.}", + "black --check --diff {args:.}", +] +fmt = [ + "black {args:.}", + "ruff --fix {args:.}", + "style", +] +all = [ + "style", + "typing", +] + +[tool.black] +target-version = ["py311"] +line-length = 120 +skip-string-normalization = true + +[tool.ruff] +target-version = "py311" +line-length = 120 +select = [ + "A", + "ARG", + "B", + "C", + "DTZ", + "E", + "EM", + "F", + "FBT", + "I", + "ICN", + "ISC", + "N", + "PLC", + "PLE", + "PLR", + "PLW", + "Q", + "RUF", + "S", + "T", + "TID", + "UP", + "W", + "YTT", +] +ignore = [ + # Allow non-abstract empty methods in abstract base classes + "B027", + # Allow boolean positional values in function calls, like `dict.get(... True)` + "FBT003", + # Ignore checks for possible passwords + "S105", "S106", "S107", + # Ignore complexity + "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", +] +unfixable = [ + # Don't touch unused imports + "F401", +] + +[tool.ruff.isort] +known-first-party = ["readyplayerme"] + +[tool.ruff.flake8-tidy-imports] +ban-relative-imports = "all" + +[tool.ruff.per-file-ignores] +# Tests can use magic values, assertions, and relative imports +"tests/**/*" = ["PLR2004", "S101", "TID252"] + +[tool.coverage.run] +source_pkgs = ["readyplayerme/asset_validation_schemas", "tests"] +branch = true +parallel = true +omit = [ + "src/readyplayerme/asset_validation_schemas/__about__.py", +] + +[tool.coverage.paths] +readyplayerme_content_validation_schemas = ["src/readyplayerme/asset_validation_schemas", "*/content-validation-schemas/src/readyplayerme/asset_validation_schemas"] +tests = ["tests", "*/content-validation-schemas/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] diff --git a/src/readyplayerme/asset_validation_schemas/__about__.py b/src/readyplayerme/asset_validation_schemas/__about__.py new file mode 100644 index 0000000..d6212c5 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/__about__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2023-present Ready Player Me +# +# SPDX-License-Identifier: MIT +__version__ = "0.0.1" diff --git a/src/readyplayerme/asset_validation_schemas/__init__.py b/src/readyplayerme/asset_validation_schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..54705e9 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2023-present Ready Player Me +# +# SPDX-License-Identifier: MIT From 5d519e2b7fb555f92955660fd0d7b8496cf589cf Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sat, 24 Jun 2023 03:35:18 +0200 Subject: [PATCH 02/66] feat: add python, linux, macOS files to gitignore --- .gitignore | 252 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 222 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 9261861..f89e80c 100644 --- a/.gitignore +++ b/.gitignore @@ -138,6 +138,175 @@ dist # SvelteKit build / generate output .svelte-kit +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + ### PyCharm ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 @@ -218,38 +387,13 @@ fabric.properties .idea/caches/build_file_checksums.ser ### PyCharm Patch ### -# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 - -# *.iml -# modules.xml -# .idea/misc.xml -# *.ipr - -# Sonarlint plugin -# https://plugins.jetbrains.com/plugin/7973-sonarlint -.idea/**/sonarlint/ +# Ignore everything but code style settings and run configurations +# that are supposed to be shared within teams. -# SonarQube Plugin -# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin -.idea/**/sonarIssues.xml +.idea/* -# Markdown Navigator plugin -# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced -.idea/**/markdown-navigator.xml -.idea/**/markdown-navigator-enh.xml -.idea/**/markdown-navigator/ - -# Cache file creation bug -# See https://youtrack.jetbrains.com/issue/JBR-2257 -.idea/$CACHE_FILE$ - -# CodeStream plugin -# https://plugins.jetbrains.com/plugin/12206-codestream -.idea/codestream.xml - -# Azure Toolkit for IntelliJ plugin -# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij -.idea/**/azureSettings.xml +!.idea/codeStyles +!.idea/runConfigurations ### VisualStudioCode ### .vscode/* @@ -295,3 +439,51 @@ $RECYCLE.BIN/ # Windows shortcuts *.lnk + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud From 117fd5231e11d57e15f9bf0e4aa9111154924764 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Mon, 26 Jun 2023 01:25:10 +0200 Subject: [PATCH 03/66] feat(codestyle): normalize strings --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 51c1385..089b6be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ all = [ [tool.black] target-version = ["py311"] line-length = 120 -skip-string-normalization = true +skip-string-normalization = false [tool.ruff] target-version = "py311" From 77c4d39ded584c0be55a22b4596a00c03c542816 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Mon, 26 Jun 2023 01:25:49 +0200 Subject: [PATCH 04/66] feat(codestyle): add mypy settings --- pyproject.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 089b6be..eb11c06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,6 +152,25 @@ ban-relative-imports = "all" # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] +[tool.mypy] +plugins = ["pydantic.mypy"] + +follow_imports = "silent" +warn_redundant_casts = true +warn_unused_ignores = true +disallow_any_generics = true +check_untyped_defs = true +no_implicit_reexport = true + +# for strict mypy: +disallow_untyped_defs = false + +[tool.pydantic-mypy] +init_forbid_extra = true +init_typed = true +warn_required_dynamic_aliases = true +warn_untyped_fields = true + [tool.coverage.run] source_pkgs = ["readyplayerme/asset_validation_schemas", "tests"] branch = true From a96c1452c53a9cfbeec731d7d7b58f3739db248b Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Mon, 26 Jun 2023 01:30:01 +0200 Subject: [PATCH 05/66] feat(schema): add python BaseModel add $schema and $id fields to json, remove property titles --- .../asset_validation_schemas/basemodel.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/basemodel.py diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py new file mode 100644 index 0000000..28ddd22 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -0,0 +1,30 @@ +from typing import Any + +from pydantic import BaseConfig, Extra +from pydantic import BaseModel as PydanticBaseModel + + +class SchemaConfig(BaseConfig): + extra = Extra.forbid + validate_assignment = True + validate_all = True + anystr_strip_whitespace = True + + @staticmethod + def schema_extra(schema: dict[str, Any], model: type["PydanticBaseModel"]) -> None: + # Add metaschema and id. + schema |= { + "$schema": "https://json-schema.org/draft/2020-12/schema", + # Get the "outer" class name with a lower case first letter. + "$id": f"{(name := model.__name__)[0].lower() + name[1:]}.schema.json", + } + # Remove "title" from properties. + for prop in schema.get("properties", {}).values(): + prop.pop("title", None) + + +class BaseModel(PydanticBaseModel): + """Global base class for all models.""" + + class Config(SchemaConfig): + ... From d063bc88b2dd1690347e1d04e6e1b7ffaa08a255 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Mon, 26 Jun 2023 01:36:28 +0200 Subject: [PATCH 06/66] feat(schema): mesh triangle count model --- .../mesh_triangle_count.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py diff --git a/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py b/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py new file mode 100644 index 0000000..0276ad5 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py @@ -0,0 +1,82 @@ +from typing import Any, ClassVar + +from pydantic import BaseModel as PydanticBaseModel +from pydantic import ( + ValidationError, + conint, + create_model, +) + +from readyplayerme.asset_validation_schemas.basemodel import SchemaConfig + +ERROR_CODE = "TRIANGLE_COUNT" + +# Triangle budgets could be set via config. +limits = { + "beard": 1000, + "body": 14000, + "body_custom": 14000, + "eyebrow": 60, + "eye": 60, + "facewear": 900, + "glasses": 1000, + "hair": 3000, + "head": 4574, + "headCustom": 6000, + "headwear": 2500, + "outfitBottom": 5000, + "outfitTop": 6000, + "outfitFootwear": 2000, + "halfbodyShirt": 1000, + "teeth": 1000, +} + + +def get_tricount_field_definitions(limits: dict[str, int]) -> Any: + """Turn simple integer limits into a dict of constrained positive integer field types.""" + return {field_name: (conint(gt=0, le=limit), None) for field_name, limit in limits.items()} + + +class TriCountConfig(SchemaConfig): + """Triangle count schema title and error messages.""" + + title = "Triangle Count" + error_msg_templates: ClassVar[dict[str, str]] = { + e: f"{ERROR_CODE} " + msg + for e, msg in { + "value_error.number.not_gt": "Mesh must have at least 1 triangle.", + "value_error.number.not_le": "Mesh exceeds triangle count budget. Allowed: {limit_value}.", + "value_error.extra": "Invalid extra mesh.", + }.items() + } + + @classmethod # As a staticmethod, TypeError is raised with super().schema_extra args. Works with classmethod. + def schema_extra(cls, schema: dict[str, Any], model: type["PydanticBaseModel"]) -> None: + super().schema_extra(schema, model) + # Add custom ajv-error messages to the schema. + for prop in schema.get("properties", {}).values(): + prop["errorMessage"] = { + "exclusiveMinimum": "Mesh ${1/name} must have at least 1 triangle.", + "maximum": "Mesh ${1/name} exceeds triangle count budget. Allowed: %d. Found: ${0}." % prop["maximum"], + } + + +# Dynamically create the model with the tri-count limits we defined earlier. +MeshTriangleCountModel: type[PydanticBaseModel] = create_model( + "MeshTriangleCount", + __config__=TriCountConfig, + # Populate the fields. + **get_tricount_field_definitions(limits), +) + + +if __name__ == "__main__": + # Convert model to JSON schema. + print(MeshTriangleCountModel.schema_json(indent=2)) + + # Example of validation in Python. + try: + # Multiple checks at once. Test non-existent field as well. + MeshTriangleCountModel(outfitBottom=6000, outfitTop=0, outfitFootwear=1000, foo=10) + except ValidationError as error: + print("\nValidation Errors:\n", error) From 3df56aee4741502826ae7fd3a1c0724398f766d6 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Mon, 10 Jul 2023 10:46:39 +0300 Subject: [PATCH 07/66] fix(schema): remove default values from properties --- src/readyplayerme/asset_validation_schemas/basemodel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index 28ddd22..6b39064 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -18,9 +18,10 @@ def schema_extra(schema: dict[str, Any], model: type["PydanticBaseModel"]) -> No # Get the "outer" class name with a lower case first letter. "$id": f"{(name := model.__name__)[0].lower() + name[1:]}.schema.json", } - # Remove "title" from properties. + # Remove "title" & "default" from properties. for prop in schema.get("properties", {}).values(): prop.pop("title", None) + prop.pop("default", None) class BaseModel(PydanticBaseModel): From ac2c8dddab5c10aa2b31de7b3e52276d86b721c8 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Mon, 10 Jul 2023 10:47:18 +0300 Subject: [PATCH 08/66] fix(schema): raise outfitBottom tri-count to 6000 --- .../asset_validation_schemas/mesh_triangle_count.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py b/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py index 0276ad5..af643c0 100644 --- a/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py +++ b/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py @@ -24,7 +24,7 @@ "head": 4574, "headCustom": 6000, "headwear": 2500, - "outfitBottom": 5000, + "outfitBottom": 6000, "outfitTop": 6000, "outfitFootwear": 2000, "halfbodyShirt": 1000, @@ -42,6 +42,7 @@ class TriCountConfig(SchemaConfig): title = "Triangle Count" error_msg_templates: ClassVar[dict[str, str]] = { + # Prepend the error code to the messages. e: f"{ERROR_CODE} " + msg for e, msg in { "value_error.number.not_gt": "Mesh must have at least 1 triangle.", @@ -77,6 +78,6 @@ def schema_extra(cls, schema: dict[str, Any], model: type["PydanticBaseModel"]) # Example of validation in Python. try: # Multiple checks at once. Test non-existent field as well. - MeshTriangleCountModel(outfitBottom=6000, outfitTop=0, outfitFootwear=1000, foo=10) + MeshTriangleCountModel(outfitBottom=6000, outfitTop=0, outfitFootwear=3000, foo=10) except ValidationError as error: print("\nValidation Errors:\n", error) From 7d21af101211a7bf44c35e151d9eb634b4fbe81a Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Mon, 10 Jul 2023 10:51:32 +0300 Subject: [PATCH 09/66] feat(schema): add common mesh schema python model --- .../asset_validation_schemas/common_mesh.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/common_mesh.py diff --git a/src/readyplayerme/asset_validation_schemas/common_mesh.py b/src/readyplayerme/asset_validation_schemas/common_mesh.py new file mode 100644 index 0000000..2491569 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/common_mesh.py @@ -0,0 +1,85 @@ +from enum import Enum +from typing import Any + +from pydantic import Field, ValidationError, validator + +from readyplayerme.asset_validation_schemas.basemodel import BaseModel + + +class IntegerType(str, Enum): + u8 = "u8" + u16 = "u16" + + +class RenderingMode(str, Enum): + TRIANGLES = "TRIANGLES" + LINES = "LINES" + POINTS = "POINTS" + + +class CommonMesh(BaseModel): + """Validation schema for common properties of meshes.""" + + class Config(BaseModel.Config): + title = "Common Mesh Properties" + error_msg_templates = {"value_error.number.not_le": "The value must be less than or equal to {limit_value}"} + + @classmethod + def schema_extra(cls, schema: dict[str, Any], model: type["BaseModel"]) -> None: + super().schema_extra(schema, model) + schema["required"] = ["mode", "primitives", "indices", "instances", "size"] + + mode: set[str] = Field( + {RenderingMode.TRIANGLES}, + description=f"The rendering mode of the mesh. Only {RenderingMode.TRIANGLES.value} are supported.", + errorMessage=f"Rendering mode must be {RenderingMode.TRIANGLES.value}.", + const=True, + ) + primitives: int = Field( + ..., + ge=1, + le=2, + description="Number of geometry primitives to be rendered with the given material.", + errorMessage="Number of primitives in the mesh must be 1, or 2 when an additional transparent material is used.", + ) + indices: set[str] = Field( + {IntegerType.u16}, + description="The index of the accessor that contains the vertex indices.", + errorMessage=f"Indices must be '{IntegerType.u16.value}' single-item array.", + const=True, + ) + instances: int = Field( + 1, + const=True, + description="Number of instances to render.", + errorMessage="Only 1 instance per mesh is supported.", + ) + size: int = Field( + ..., + gt=0, + le=524288, + description="Byte size. Buffers stored as GLB binary chunk have an implicit limit of (2^32)-1 bytes.", + errorMessage={ + "maximum": "Maximum allowed mesh size is 512 kB.", + "exclusiveMinimum": "Mesh has 0 bytes, seems to be empty!", + }, + ) + + @validator("size", pre=True) + def check_size(cls, value): + if value == 0: + msg = "Mesh size must be greater than 0 Bytes." + raise ValueError(msg) + return value + + +if __name__ == "__main__": + # Convert model to JSON schema. + print(CommonMesh.schema_json(indent=2)) + + # Example of validation in Python. + try: + # Multiple checks at once. Test non-existent field as well. + model = CommonMesh(mode=["LINES"], primitives=3, indices=["u8"], instances=2, size=1e7) + except ValidationError as error: + print("\nValidation Errors:\n", error) From eaf29f071ca2b5cd3f1388c984bfc90b6c5f6595 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Tue, 1 Aug 2023 17:52:13 +0200 Subject: [PATCH 10/66] chore: add project setup and templates --- .docstr.yaml | 18 +++ .github/CODEOWNERS | 4 + .github/ISSUE_TEMPLATE/bug_report.md | 39 +++++++ .github/ISSUE_TEMPLATE/feature_request.md | 49 +++++++++ .github/pull_request_template.md | 41 +++++++ .github/workflows/test.yml | 43 ++++++++ .pre-commit-config.yaml | 35 ++++++ CHANGELOG.md | 10 ++ CODE_OF_CONDUCT.md | 128 ++++++++++++++++++++++ CONTRIBUTING.md | 116 ++++++++++++++++++++ pyproject.toml | 81 ++++++++++++-- schemas/gltf-asset.schema.json | 11 ++ 12 files changed, 566 insertions(+), 9 deletions(-) create mode 100644 .docstr.yaml create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/test.yml create mode 100644 .pre-commit-config.yaml create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md diff --git a/.docstr.yaml b/.docstr.yaml new file mode 100644 index 0000000..2b750c0 --- /dev/null +++ b/.docstr.yaml @@ -0,0 +1,18 @@ +paths: # list or string + - src/readyplayerme +#badge: docs # Path to save SVG file +exclude: .*/tests # regex +verbose: 3 # int (0-4) +skip_magic: True # Boolean +skip_file_doc: True # Boolean +skip_init: True # Boolean +skip_class_def: False # Boolean +skip_private: True # Boolean +follow_links: True # Boolean +accept_empty: True # Boolean +ignore_names_file: .*/tests # regex +fail_under: 90 # int +percentage_only: False # Boolean +ignore_patterns: # Dict with key/value pairs of file-pattern/node-pattern + .*: + - process diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..d7fdfc7 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,4 @@ +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# @Olaf-Wolf3D will be requested for review when someone opens a pull request. +* @Olaf-Wolf3D @TechyDaniel diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..1206b5f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,39 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: 'bug' +assignees: '' + +--- + +**Which plugin** +If the bug relates to a pyblish plugin, write which one. + +**Describe the bug** +A clear and concise description of what the bug is. + +**Reproduction rate** +percentage of how often it happens, always or sometimes? + +**Reproduction steps** +describe the steps to reproduce the behavior, step 1. 2. 3. .. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + +- OS: [e.g. Windows] +- Version [e.g. 11] + +**Blender (please complete the following information):** + +- Version: [e.g. 3.3 LTS] +- Installation type: [e.g. installer, portable, blender launcher, steam, windows store] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..57c1cec --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,49 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to an existing Ready Player Me pyblish plugin?** +Mention plugin here. + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. Ideally, use Gherkin syntax. +Example: + +```Gherkin +Feature: Action to make all materials single-sided + Let users quickly make _all_ materials single-sided, regardless of validation state. + + Rule: Make all materials single-sided + Background: + Given I have materials + + Example: two-sided materials + Given I have run the validation + And at least one of the materials is two-sided + When I choose the single-sided materials action in the validator + Then all materials are single-sided + + Example: one-sided materials + Given I have run the validation + And there are no two-sided materials + Then I still have the option to make all materials one-sided + ... +``` + +**Describe the impact** +Gauge how much __impact__ this feature would have for its users [e.g. how much time it will save]. +What is its value? + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..aec0689 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,41 @@ +# Description + +Please include a summary of the changes and the related issue. +Please also include relevant motivation and context. +List any dependencies that are required for this change. + +Fixes # (issue) + +## Type of change + +Please delete options that are not relevant. + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] This change requires a documentation update + +# How Has This Been Tested? + +Please describe the tests that you ran to verify your changes. +Provide instructions so we can reproduce. +Please also list any relevant details for your test configuration. + +- [ ] Test A +- [ ] Test B + +**Test Configuration**: +* Blender version: +* Installation type [Ex. portable, installation, Steam, Windows store]: + +# Checklist: + +- [ ] I confirm that the changes meet the user experience and goals outlined in the design plan (if there is one). +- [ ] My code follows the style guidelines of this project +- [ ] I have performed a self-review of my code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] I have updated any version info, if necessary. +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..9d109b6 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,43 @@ +name: test + +on: + push: + branches: [main] + pull_request: + branches: [main] + types: [ready_for_review, reopened] + +concurrency: + group: test-${{ github.head_ref }} + cancel-in-progress: true + +env: + PYTHONUNBUFFERED: "1" + FORCE_COLOR: "1" + +jobs: + run: + name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ['3.10'] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Hatch + run: pip install --upgrade hatch + + - name: Run tests + run: hatch run cov + + - name: Run linting + run: hatch run lint:all diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..eb5bea9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +## Pre-commit setup +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-json + - id: check-toml + - id: check-yaml + # Ensures that a file is either empty, or ends with one newline. + - id: end-of-file-fixer + + # Remove trailing whitespace + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + + # Code style and formatting + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.0.280 + hooks: + - id: ruff + + - repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + args: [ + --line-length, "120", + ] + + - repo: https://github.com/HunterMcGushion/docstr_coverage + rev: v2.3.0 + hooks: + - id: docstr-coverage + args: ["--verbose", "2"] # override the .docstr.yaml to see less output diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..db498b3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# CHANGELOG + +All notable changes to this project will be documented in this file. +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## UNRELEASED + +### Added + +- Nothing diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..a7a3983 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +support@readyplayer.me. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..6e1641a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,116 @@ +# Welcome to the Ready Player Me Asset Validation Schemas contributing guide + +In this guide you will get an overview of the contribution workflow from opening an issue, creating a pull request, reviewing, and merging the pull request. + +## New contributor guide + +To get an overview of the project, read the [README](README.md). +Here are some resources to help you get started with open source contributions: + +- [Set up Git](https://docs.github.com/en/get-started/quickstart/set-up-git) +- [Collaborating with pull requests](https://docs.github.com/en/github/collaborating-with-pull-requests) +- Learn about [pre-commit hooks](https://pre-commit.com/) +- We use [black](https://black.readthedocs.io/en/stable/) formatting, but with a line-length of 120 characters. + +## Getting started + +You should also be familiar with [JSON schemas](https://json-schema.org/). +Learn about [Pydantic](https://docs.pydantic.dev/) as well. + +## Issues + +### Create a new issue + +If you spot a problem with the schemas or package, [search if an issue already exists](https://docs.github.com/en/github/searching-for-information-on-github/searching-on-github/searching-issues-and-pull-requests#search-by-the-title-body-or-comments). +If a related issue doesn't exist, you can open a new issue using a relevant [issue form](https://github.com/wolfprint3d/content-validation-schemas/issues/new/choose). + +### Solve an issue + +Scan through our [existing issues](https://github.com/wolfprint3d/content-validation-schemas/issues) to find one that interests you. +You can narrow down the search using `labels` as filters. + +### Labels + +Labels can help you find an issue you'd like to help with. +The [`good first issue` label](https://github.com/wolfprint3d/content-validation-schemas/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) is for problems or updates we think are ideal for new joiners. + +## Contribute + +### Make changes locally + +1. If you are a contributor from outside the Ready Player Me team, [fork the repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo). + +2. [Clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) the (forked) repository to your local machine. + +3. It's best to use a separate Python _environment_ for development to avoid conflicts with other Python projects and keep your system Python clean. + We encourage using an environment manager such as [conda](https://docs.conda.io/en/latest/), [micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html), or [poetry](https://python-poetry.org/). + You'll need a minimum Python version 3.10. + +4. We use [hatch](https://hatch.pypa.io/) as the Python package build backend and Python project manager. + We recommend to install it as well as it will provide you with additional development environments. + Why do I need et another environment manager, you might ask? + Because hatch on its own can only create environments with Python versions that are already installed on your system. 😔 + However, using hatch is not a necessity, but more of a convenience. + See [hatch's installation instructions](https://hatch.pypa.io/latest/install/) for more information on how to install it into your Python environment. + + Once you setup hatch, navigate to the cloned repository, and execute `hatch env create`. + This will create yet a new environment and install the development dependencies into it. + You can get the new environment path and add it to your IDE as a Python interpreter for this repository, `hatch run python -c "import sys;print(sys.executable)"`. + + If you decided against using hatch, we still recommend installing the pre-commit hooks. + +5. Create a working branch and prefix its name with _fix/_ if it's a bug fix, or _feature/_ if it's a new feature. + Start with your changes! + Have a look at the README for more information on how to use the package. + +6. Write or update tests for your changes. + +7. Run tests with `hatch run test` and code linting & formatting with `hatch run lint:all` locally. + +### Commit your update + +Once you are happy with your changes, it's time to commit them. +Use [Conventional Commit messages](https://www.conventionalcommits.org/en/v1.0.0/). +[Sign](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) your commits! + +If you followed the steps above, you should have a pre-commit hook installed that will automatically run the tests and linting before a commit succeeds. + +Keep your individual commits small, so any breaking change can more easily be traced back to a commit. +A commit ideally only changes one single responsibility at a time. +If you keep the whole of your changes small and the branch short-lived, there's less risk to run into any other conflicts that can arise with the base. + +Don't forget to [self-review](#self-review) to speed up the review process :zap:. + +### Pull Request + +When you're finished with the changes, create a __draft__ pull request, also known as a PR. + +- Fill the "Ready for review" template so that we can review your PR. This template helps reviewers understand your changes as well as the purpose of your pull request. +- Don't forget to [link PR to issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) if you are solving one. +- If you are a contributor from outside the Ready Player Me team, enable the checkbox to [allow maintainer edits](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork) so the branch can be updated for a merge. +Once you submit your PR, a Ready Player Me team member will review your proposal. +We may ask questions or request additional information. +- We may ask for changes to be made before a PR can be merged, either using [suggested changes](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/incorporating-feedback-in-your-pull-request) or pull request comments. +You can apply suggested changes directly through the UI. +You can make any other changes in your branch, and then commit them. +- As you update your PR and apply changes, mark each conversation as [resolved](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/commenting-on-a-pull-request#resolving-conversations). +- If you run into any merge issues, checkout this [git tutorial](https://github.com/skills/resolve-merge-conflicts) to help you resolve merge conflicts and other issues. + +### Self review + +You should always review your own PR first. + +Make sure that you: + +- [ ] Confirm that the changes meet the user experience and goals outlined in the design plan (if there is one). +- [ ] Update the version of the schemas. +- [ ] Update the documentation if necessary. +- [ ] If there are any failing checks in your PR, troubleshoot them until they're all passing. + +### Merging your PR + +Once your PR has the required approvals, a Ready Player Me team member will merge it. + +We use a [squash & merge](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/incorporating-changes-from-a-pull-request/about-pull-request-merges#squash-and-merge-your-commits) by default to merge a PR. + +The branch will be deleted automatically after the merge to prevent any more work being done the branch after it was merged. diff --git a/pyproject.toml b/pyproject.toml index eb11c06..e83824e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,18 +3,20 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "readyplayerme-asset-validation-schemas" +name = "readyplayerme.asset_validation_schemas" dynamic = ["version"] -description = 'Data models that are used to check data gathered from assets for compatibility with the Ready Player Me avatar platform.' +description = "This package provides pydantic data models to validate asset data for compatibility with Ready Player Me avatars. It also allows exporting JSON schemas for use in other applications." readme = "README.md" requires-python = ">=3.10" license = "MIT" -keywords = ["gltf", "3D", "json", "schema", "validation"] +keywords = ["ready player me", "gltf", "3D", "json", "schema", "validation"] authors = [ { name = "Ready Player Me", email = "info@readyplayer.me" }, ] maintainers = [ { name = "Olaf Haag", email = "olaf@readyplayer.me" }, + { name = "Daniel-Ionut Rancea", email = "daniel-ionut.rancea@readyplayer.me" }, + { name = "Ivan Sanandres Gutierrez", email = "ivan@readyplayer.me" }, ] classifiers = [ "Development Status :: 4 - Beta", @@ -25,7 +27,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "pydantic" + "pydantic>=2.1" ] [project.optional-dependencies] @@ -33,7 +35,7 @@ tests = [ "pytest", ] dev = [ - "readyplayerme-asset-validation-schemas[tests]", + "readyplayerme.asset_validation_schemas[tests]", "pre-commit", ] @@ -53,8 +55,15 @@ dependencies = [ "coverage[toml]>=6.5", "pytest", ] +features = [ + "dev", +] +post-install-commands = [ + "install-precommit", +] [tool.hatch.envs.default.scripts] +install-precommit = "pre-commit install" test = "pytest {args:tests}" test-cov = "coverage run -m pytest {args:tests}" cov-report = [ @@ -93,21 +102,24 @@ all = [ ] [tool.black] -target-version = ["py311"] +target-versions = ["py310", "py311"] line-length = 120 skip-string-normalization = false [tool.ruff] -target-version = "py311" +target-version = "py310" line-length = 120 select = [ "A", + "ANN", "ARG", "B", "C", + "D", "DTZ", "E", "EM", + "ERA", "F", "FBT", "I", @@ -122,6 +134,7 @@ select = [ "RUF", "S", "T", + "T20", "TID", "UP", "W", @@ -130,17 +143,48 @@ select = [ ignore = [ # Allow non-abstract empty methods in abstract base classes "B027", + # Allow unused arguments + "ARG002", # Allow boolean positional values in function calls, like `dict.get(... True)` "FBT003", - # Ignore checks for possible passwords - "S105", "S106", "S107", + # Allow try-except-pass & try-except-continue + "S110", "S112", # Ignore complexity "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", + # Exclude self, cls, args, kwargs from annotation, allow dynamically typed expressions (typing.Any) in type annotations + "ANN101", "ANN102", "ANN002", "ANN003", "ANN401", + # Don't require documentation for every function parameter. + "D417", "D102", ] +builtins = ["_"] unfixable = [ # Don't touch unused imports "F401", ] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", +] [tool.ruff.isort] known-first-party = ["readyplayerme"] @@ -148,6 +192,25 @@ known-first-party = ["readyplayerme"] [tool.ruff.flake8-tidy-imports] ban-relative-imports = "all" +[tool.ruff.flake8-quotes] +docstring-quotes = "double" + +[tool.ruff.flake8-annotations] +mypy-init-return = true +allow-star-arg-any = true +ignore-fully-untyped = true +suppress-none-returning = true + +[tool.ruff.flake8-unused-arguments] +ignore-variadic-names = true + +[tool.ruff.pycodestyle] +ignore-overlong-task-comments = true + +[tool.ruff.pydocstyle] +convention = "pep257" +ignore-decorators = ["typing.overload"] + [tool.ruff.per-file-ignores] # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] diff --git a/schemas/gltf-asset.schema.json b/schemas/gltf-asset.schema.json index 02df2e4..d77ccb0 100644 --- a/schemas/gltf-asset.schema.json +++ b/schemas/gltf-asset.schema.json @@ -51,6 +51,17 @@ } } }, + "gltfExtensions": { + "description": "The glTF extensions used in the asset.", + "type": "array", + "items": { + "type": "string" + }, + "maxItems": 0, + "errorMessage": { + "maxItems": "The gltf asset must not have extensions. Found: ${0}." + } + }, "gltfErrors": { "description": "The errors encountered during glTF validation.", "type": "array", From 50936171d221aa19bc889d115eae10433f21519f Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 3 Aug 2023 17:50:50 +0200 Subject: [PATCH 11/66] fix: update dev environment update pre-commit hooks, setup guide, merge dev envs and set local path --- .docstr.yaml | 2 +- .pre-commit-config.yaml | 4 ++- CONTRIBUTING.md | 80 +++++++++++++++++++++++++++++++++-------- pyproject.toml | 22 ++++++------ 4 files changed, 79 insertions(+), 29 deletions(-) diff --git a/.docstr.yaml b/.docstr.yaml index 2b750c0..0b75777 100644 --- a/.docstr.yaml +++ b/.docstr.yaml @@ -11,7 +11,7 @@ skip_private: True # Boolean follow_links: True # Boolean accept_empty: True # Boolean ignore_names_file: .*/tests # regex -fail_under: 90 # int +fail_under: 25 # int percentage_only: False # Boolean ignore_patterns: # Dict with key/value pairs of file-pattern/node-pattern .*: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb5bea9..2bc05ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,7 @@ repos: rev: v4.3.0 hooks: - id: check-json + - id: pretty-format-json - id: check-toml - id: check-yaml # Ensures that a file is either empty, or ends with one newline. @@ -16,9 +17,10 @@ repos: # Code style and formatting - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.0.280 + rev: v0.0.282 hooks: - id: ruff + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black rev: 23.7.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e1641a..e08cf4d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,7 @@ Here are some resources to help you get started with open source contributions: - [Collaborating with pull requests](https://docs.github.com/en/github/collaborating-with-pull-requests) - Learn about [pre-commit hooks](https://pre-commit.com/) - We use [black](https://black.readthedocs.io/en/stable/) formatting, but with a line-length of 120 characters. +- If you haven't yet setup an IDE, we recommend [Visual Studio Code](https://code.visualstudio.com/). See [Python in Visual Studio Code](https://code.visualstudio.com/docs/languages/python). ## Getting started @@ -36,36 +37,85 @@ The [`good first issue` label](https://github.com/wolfprint3d/content-validation ## Contribute -### Make changes locally +### Get the Code 1. If you are a contributor from outside the Ready Player Me team, [fork the repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo). 2. [Clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) the (forked) repository to your local machine. -3. It's best to use a separate Python _environment_ for development to avoid conflicts with other Python projects and keep your system Python clean. - We encourage using an environment manager such as [conda](https://docs.conda.io/en/latest/), [micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html), or [poetry](https://python-poetry.org/). +### Setting up Development Environment + +It's best to use a separate Python _environment_ for development to avoid conflicts with other Python projects and keep your system Python clean. In this section we'll provide instructions on how to set up such an environment. + +We use [hatch](https://hatch.pypa.io/) as the Python package build backend and Python project manager. +We recommend to install it as it will provide you with a project-specific development environment. However, using hatch is not a necessity, but more of a convenience. +Unfortunately, there are no pre-built binaries for hatch, and hatch on its own can only create environments with Python versions that are already installed on your system. So you'll need to first create a Python environment to install hatch into, in order to then spawn another environment for the project by using hatch. It's a bit like the chicken & egg problem paired with the movie Inception.😵‍💫 We'll walk you through it. + +1. We encourage using an environment manager such as [conda](https://docs.conda.io/en/latest/), [mamba/micromamba](https://mamba.readthedocs.io/en/latest/index.html), or [poetry](https://python-poetry.org/). You'll need a minimum Python version 3.10. + Here's an example on Windows: + + ```powershell + # Get mamba using winget. + winget install -e --id CondaForge.Mambaforge + + # Make mamba available in your shell. mamba may be either installed in %ProgramData% or %LocalAppData%. + %ProgramData%\mambaforge\.condabin\mamba init + # OR, if your mamba installation is in %LocalAppData% instead: + %LocalAppData%\mambaforge\.condabin\mamba init + # You may need to restart your terminal now. + + # Test that mamba is available. + mamba --version # This should print something like "mamba 1.4.1". + ``` + +2. You can read [hatch's installation instructions](https://hatch.pypa.io/latest/install/) for information on how to install it into your Python environment, or follow the instructions below. + + If you use conda/mamba, you can create a Python environment to which hatch gets installed with: + + ```powershell + mamba create -n hatch python=3.10 hatch + ``` + + In the command above, the `-n hatch` option just gives the environment the name _hatch_, but it can be anything. + The name _hatch_ for the environment was incidentally chosen here to match the 1 package we want to utilize in this environment. The `python=3.10 hatch` part of the command defines what we want to install into the environment. See [mamba's documentation](https://mamba.readthedocs.io/en/latest/user_guide/mamba.html#quickstart) for more details. -4. We use [hatch](https://hatch.pypa.io/) as the Python package build backend and Python project manager. - We recommend to install it as well as it will provide you with additional development environments. - Why do I need et another environment manager, you might ask? - Because hatch on its own can only create environments with Python versions that are already installed on your system. 😔 - However, using hatch is not a necessity, but more of a convenience. - See [hatch's installation instructions](https://hatch.pypa.io/latest/install/) for more information on how to install it into your Python environment. +3. Activate the hatch environment. - Once you setup hatch, navigate to the cloned repository, and execute `hatch env create`. - This will create yet a new environment and install the development dependencies into it. - You can get the new environment path and add it to your IDE as a Python interpreter for this repository, `hatch run python -c "import sys;print(sys.executable)"`. + ```mamba activate hatch``` + + OR if you're using Powershell (see [issue](https://github.com/mamba-org/mamba/issues/1717)): + + ```conda activate hatch``` + +4. Prepare the environment for development. + Once you setup hatch, navigate to the cloned repository, and execute: + + ```powershell + hatch env create + ``` + + This will create yet a new environment within a `.venv` folder of the project and install the development dependencies into it. + An IDE like [Visual Studio Code](https://code.visualstudio.com/) will automatically detect this environment and suggest to you to use it as the Python interpreter for this repository. + It also installs [pre-commit](https://pre-commit.com/) hooks into the repository, which will run linting and formatting checks before you commit your changes. + + Alternatively, you can get the new environment path and add it to your IDE as a Python interpreter for this repository with: + + ```hatch run python -c "import sys;print(sys.executable)"``` If you decided against using hatch, we still recommend installing the pre-commit hooks. -5. Create a working branch and prefix its name with _fix/_ if it's a bug fix, or _feature/_ if it's a new feature. + ```pre-commit install``` + +### Branch Off & Make Your Changes + +1. Create a working branch and prefix its name with _fix/_ if it's a bug fix, or _feature/_ if it's a new feature. Start with your changes! Have a look at the README for more information on how to use the package. -6. Write or update tests for your changes. +2. Write or update tests for your changes. -7. Run tests with `hatch run test` and code linting & formatting with `hatch run lint:all` locally. +3. Run tests with `hatch run test` and or run linting & formatting and tests with `hatch run all` locally. ### Commit your update diff --git a/pyproject.toml b/pyproject.toml index e83824e..16f85c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,13 @@ path = "src/readyplayerme/asset_validation_schemas/__about__.py" packages = ["src/readyplayerme"] [tool.hatch.envs.default] +description = "Python virtual environment in project dir to quickly get up and running in an IDE like VSCode." +type = "virtual" +path = ".venv" dependencies = [ + "black>=23.3.0", + "mypy>=1.3.0", + "ruff>=0.0.275", "coverage[toml]>=6.5", "pytest", ] @@ -74,18 +80,6 @@ cov = [ "test-cov", "cov-report", ] - -[[tool.hatch.envs.all.matrix]] -python = ["3.10", "3.11"] - -[tool.hatch.envs.lint] -detached = true -dependencies = [ - "black>=23.3.0", - "mypy>=1.3.0", - "ruff>=0.0.275", -] -[tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive {args:src/readyplayerme/asset_validation_schemas tests}" style = [ "ruff {args:.}", @@ -99,8 +93,12 @@ fmt = [ all = [ "style", "typing", + "cov", ] +[[tool.hatch.envs.all.matrix]] +python = ["3.10", "3.11"] + [tool.black] target-versions = ["py310", "py311"] line-length = 120 From 15c14783a82891663c3a9497ca634edfd50de528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Sanandr=C3=A9s?= <43515426+Ivan-Sanandres@users.noreply.github.com> Date: Fri, 4 Aug 2023 18:07:23 +0200 Subject: [PATCH 12/66] feat(TECHART-292): added pydantic model for material names --- .../material_names.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/material_names.py diff --git a/src/readyplayerme/asset_validation_schemas/material_names.py b/src/readyplayerme/asset_validation_schemas/material_names.py new file mode 100644 index 0000000..3bbc640 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/material_names.py @@ -0,0 +1,105 @@ +from typing import Any, ClassVar, List, Literal + +from pydantic import BaseModel as PydanticBaseModel +from pydantic import ValidationError, constr, create_model + +from readyplayerme.asset_validation_schemas.basemodel import SchemaConfig + + +class MaterialNamesConfig(SchemaConfig): + """Material Names schema title and error messages.""" + + title = "Material Names" + error_msg_templates: ClassVar[dict[str, str]] = { + # Prepend the error code to the messages. + "literal": "Material name should be '${const}'. Found ${0} instead.", + "enum": "Material name should be one of ${allowed_values}. Found ${0} instead.", + } + + @classmethod + def schema_extra(cls, schema: dict[str, Any], model: type[PydanticBaseModel]) -> None: + super().schema_extra(schema, model) + # Add custom ajv-error messages to the schema. + for prop in schema.get("properties", {}).values(): + if "literal" in prop: + prop["errorMessage"] = cls.error_msg_templates["literal"].replace("${const}", prop["literal"]) + elif "enum" in prop: + prop["errorMessage"] = cls.error_msg_templates["enum"].replace("${allowed_values}", str(prop["enum"])) + +class MaterialNames(SchemaConfig): + beard: Literal["Wolf3D_Beard"] + facewear: Literal["Wolf3D_Facewear"] + glasses: Literal["Wolf3D_Glasses"] + hair: Literal["Wolf3D_Hair"] + halfBodyShirt: Literal["Wolf3D_Shirt"] + head: Literal["Wolf3D_Skin"] + headwear: Literal["Wolf3D_Headwear"] + modularBottom: Literal["Wolf3D_Outfit_Bottom"] + modularFootwear: Literal["Wolf3D_Outfit_Footwear"] + modularTop: Literal["Wolf3D_Outfit_Top"] + nonCustomizableAvatar: List[Literal[ + "Wolf3D_Body", + "Wolf3D_Eye", + "Wolf3D_Outfit_Bottom", + "Wolf3D_Outfit_Footwear", + "Wolf3D_Outfit_Top", + "Wolf3D_Skin", + "Wolf3D_Teeth", + "Wolf3D_Hair", + "Wolf3D_Beard", + "Wolf3D_Facewear" + "Wolf3D_Glasses" + "Wolf3D_Headwear" + ]] + outfit: List[Literal[ + "Wolf3D_Body", + "Wolf3D_Outfit_Bottom", + "Wolf3D_Outfit_Footwear", + "Wolf3D_Outfit_Top", + ]] + +MaterialNamesModel = create_model( + "MaterialNames", + beard=(Literal["Wolf3D_Beard"], ...), + facewear=(Literal["Wolf3D_Facewear"], ...), + glasses=(Literal["Wolf3D_Glasses"], ...), + hair=(Literal["Wolf3D_Hair"], ...), + halfBodyShirt=(Literal["Wolf3D_Shirt"], ...), + head=(Literal["Wolf3D_Skin"], ...), + headwear=(Literal["Wolf3D_Headwear"], ...), + modularBottom=(Literal["Wolf3D_Outfit_Bottom"], ...), + modularFootwear=(Literal["Wolf3D_Outfit_Footwear"], ...), + modularTop=(Literal["Wolf3D_Outfit_Top"], ...), + nonCustomizableAvatar=(List[Literal[ + "Wolf3D_Body", + "Wolf3D_Eye", + "Wolf3D_Outfit_Bottom", + "Wolf3D_Outfit_Footwear", + "Wolf3D_Outfit_Top", + "Wolf3D_Skin", + "Wolf3D_Teeth", + "Wolf3D_Hair", + "Wolf3D_Beard", + "Wolf3D_Facewear" + "Wolf3D_Glasses" + "Wolf3D_Headwear" + ]], ...), + outfit=(List[Literal[ + "Wolf3D_Body", + "Wolf3D_Outfit_Bottom", + "Wolf3D_Outfit_Footwear", + "Wolf3D_Outfit_Top", + ]], ...), + __config__=MaterialNamesConfig, +) + + +if __name__ == "__main__": + # Convert model to JSON schema. + print(MaterialNamesModel.schema_json(indent=2)) + # Example of validation in Python. + try: + # Test example validation. + MaterialNamesModel(beard="Wrong_Material_Name") + except ValidationError as error: + print("\nValidation Errors:\n", error) From e7a9e67f2dc528300e5f3de8d1c12d554e985021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Sanandr=C3=A9s?= <43515426+Ivan-Sanandres@users.noreply.github.com> Date: Fri, 4 Aug 2023 18:13:38 +0200 Subject: [PATCH 13/66] feat(TECHART-293): added pydantic model for mesh atributes --- .../mesh_atributes.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/mesh_atributes.py diff --git a/src/readyplayerme/asset_validation_schemas/mesh_atributes.py b/src/readyplayerme/asset_validation_schemas/mesh_atributes.py new file mode 100644 index 0000000..78d8835 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/mesh_atributes.py @@ -0,0 +1,52 @@ +from typing import Any, ClassVar, List, Literal + +from pydantic import BaseModel as PydanticBaseModel +from pydantic import ValidationError, constr + +from readyplayerme.asset_validation_schemas.basemodel import SchemaConfig + + +class MeshAttributesConfig(SchemaConfig): + """Mesh Attributes schema title and error messages.""" + + title = "Mesh Attributes" + error_msg_templates: ClassVar[dict[str, str]] = { + # Prepend the error code to the messages. + "enum": "Mesh ${2/name} error! Allowed attributes are: ${enum_values}. Found ${0}.", + "contains": "Mesh ${2/name} requires at least 5 vertex attributes: position, normal, 1 UV set, joint influences, and weights. Found ${0/length} attributes: ${0}.", + } + + @classmethod + def schema_extra(cls, schema: dict[str, Any], model: type[PydanticBaseModel]) -> None: + super().schema_extra(schema, model) + # Add custom ajv-error messages to the schema. + for prop in schema.get("properties", {}).values(): + if "enum" in prop: + prop["errorMessage"] = cls.error_msg_templates["enum"].replace( + "${enum_values}", ", ".join(prop["enum"]) + ) + elif "contains" in prop: + prop["errorMessage"] = cls.error_msg_templates["contains"] + +class MeshAttributes(SchemaConfig): + """Pydantic model for Mesh Attributes.""" + + unskinned: List[constr( + enum=["NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32", "TANGENT:f32"], + )] + skinned: List[constr( + enum=["JOINTS_0:u8", "NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32", "TANGENT:f32", "WEIGHTS_0:f32"], + )] + + +if __name__ == "__main__": + # Example of validation in Python + try: + # Test example validation. + data = { + "unskinned": ["NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32"], + "skinned": ["JOINTS_0:u8", "NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32", "WEIGHTS_0:f32"], + } + MeshAttributes(**data) + except ValidationError as error: + print("\nValidation Errors:\n", error) From c8e2553f3ae391943bd40a3b1b159164ea0d87c6 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Tue, 8 Aug 2023 03:14:37 +0200 Subject: [PATCH 14/66] fix(schema)!: update basemodel to pydantic v2 --- .../asset_validation_schemas/basemodel.py | 70 +++++++++++++------ 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index 6b39064..f397915 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -1,31 +1,59 @@ +"""Provide a global base class and config for all Ready Player Me pydantic models.""" +import abc from typing import Any -from pydantic import BaseConfig, Extra from pydantic import BaseModel as PydanticBaseModel +from pydantic import ConfigDict +from pydantic.alias_generators import to_camel +from pydantic.json_schema import GenerateJsonSchema -class SchemaConfig(BaseConfig): - extra = Extra.forbid - validate_assignment = True - validate_all = True - anystr_strip_whitespace = True +def json_schema_extra(schema: dict[str, Any], model: type["PydanticBaseModel"]) -> None: + """Provide extra JSON schema properties.""" + # Add id. + schema |= { + "$schema": GenerateJsonSchema.schema_dialect, + # Get the "outer" class name with a lower case first letter. + "$id": f"{(name := model.__name__)[0].lower() + name[1:]}.schema.json", + } + # Remove "title" & "default" from properties. + for prop in schema.get("properties", {}).values(): + prop.pop("title", None) + prop.pop("default", None) - @staticmethod - def schema_extra(schema: dict[str, Any], model: type["PydanticBaseModel"]) -> None: - # Add metaschema and id. - schema |= { - "$schema": "https://json-schema.org/draft/2020-12/schema", - # Get the "outer" class name with a lower case first letter. - "$id": f"{(name := model.__name__)[0].lower() + name[1:]}.schema.json", - } - # Remove "title" & "default" from properties. - for prop in schema.get("properties", {}).values(): - prop.pop("title", None) - prop.pop("default", None) +def get_model_config(**kwargs: Any) -> ConfigDict: + """Return a model config with some default values. -class BaseModel(PydanticBaseModel): + :param kwargs: Arguments to pass to the model config to add or override default values. + """ + default_dict = { + "validate_assignment": True, + "validate_default": True, + "strict": True, + "populate_by_name": True, + "extra": "forbid", + "hide_input_in_errors": True, + "alias_generator": to_camel, + "str_strip_whitespace": False, + "json_schema_extra": json_schema_extra, + "frozen": True, + } + updated_dict = default_dict | kwargs + return ConfigDict(**updated_dict) + + +class BaseModel(PydanticBaseModel, abc.ABC): """Global base class for all models.""" - class Config(SchemaConfig): - ... + model_config = get_model_config(title="Base Model", defer_build=True) + + +if __name__ == "__main__": + import json + import logging + + logging.basicConfig(level=logging.DEBUG) + + # Convert model to JSON schema. + logging.debug(json.dumps(BaseModel.model_json_schema(), indent=2)) From 6bf25f90babf7a9f9146e9adb054cd3cf4433625 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Tue, 8 Aug 2023 03:16:18 +0200 Subject: [PATCH 15/66] fix(schema)!: update mesh_triangle_count to v2 --- .../mesh_triangle_count.py | 135 +++++++++++++----- 1 file changed, 97 insertions(+), 38 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py b/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py index af643c0..ec4ec8c 100644 --- a/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py +++ b/src/readyplayerme/asset_validation_schemas/mesh_triangle_count.py @@ -1,15 +1,24 @@ -from typing import Any, ClassVar +"""Triangle count validation pydantic model.""" +from typing import Annotated, Any from pydantic import BaseModel as PydanticBaseModel from pydantic import ( + Field, ValidationError, - conint, + ValidationInfo, + ValidatorFunctionWrapHandler, create_model, ) +from pydantic.functional_validators import WrapValidator +from pydantic_core import ErrorDetails, InitErrorDetails, PydanticCustomError -from readyplayerme.asset_validation_schemas.basemodel import SchemaConfig +from readyplayerme.asset_validation_schemas.basemodel import get_model_config ERROR_CODE = "TRIANGLE_COUNT" +DOCS_URL = ( + "https://docs.readyplayer.me/asset-creation-guide/validation/validation-checks/" + "mesh-validations/check-mesh-triangle-count" +) # Triangle budgets could be set via config. limits = { @@ -22,62 +31,112 @@ "glasses": 1000, "hair": 3000, "head": 4574, - "headCustom": 6000, + "head_custom": 6000, "headwear": 2500, - "outfitBottom": 6000, - "outfitTop": 6000, - "outfitFootwear": 2000, - "halfbodyShirt": 1000, + "outfit_bottom": 6000, + "outfit_top": 6000, + "outfit_footwear": 2000, + "halfbody_shirt": 1000, "teeth": 1000, } +def convert_error(error: ErrorDetails, title: str = "Unknown Error", url: str = ""): + """Convert the error to a PydanticCustomError with a custom error type and message.""" + match error: + case {"type": "greater_than"}: + raise PydanticCustomError( + ERROR_CODE, + "Mesh must have at least 1 triangle." + # Include the URL if it's not empty. Indent by tab (4 spaces). + + "\n\tFor further information visit {url}.".expandtabs(4) * bool(url), + {"url": url}, + ) + case {"type": "less_than_equal"}: + raise PydanticCustomError( + ERROR_CODE, + "Mesh exceeds triangle count budget. Allowed: {limit}. Found: {wrong_value}." + + "\n\tFor further information visit {url}.".expandtabs(4) * bool(url), + {"limit": error["ctx"]["le"], "wrong_value": error["input"], "url": url}, + ) + case _: # Anything else raise ValidationError. + # Convert ErrorDetails to InitErrorDetails + ctx = error.get("ctx", {}) + init_error_details: InitErrorDetails = InitErrorDetails( + type=error["type"], + loc=error["loc"], + input=error["input"], + ctx=ctx, + ) + raise ValidationError.from_exception_data(title=title, line_errors=[init_error_details]) + + +def validate_tricount(value: Any, handler: ValidatorFunctionWrapHandler, _info: ValidationInfo) -> int: + """Wrap the validation function to raise custom error types. + + Return the validated value if no error occurred. + """ + try: + return handler(value) + except ValidationError as error: + for err in error.errors(): + convert_error(err, title=error.title, url=DOCS_URL) + return value # This line is unreachable, but makes type-checkers happy. + + +def get_triangle_count_type(le: int) -> Annotated: + """Return a constrained positive integer field type with custom error messages. + + :param le: The (inclusive) maximum of the integer (less-equal). + """ + return Annotated[ + int, + Field( + gt=0, + le=le, + json_schema_extra={ + "errorMessage": { + "exclusiveMinimum": "Mesh ${1/name} must have at least 1 triangle.", + # Use % formatting for error message instead of f-string, since we need to preserve "${0}". + "maximum": "Mesh ${1/name} exceeds triangle count budget. Allowed: %d. Found: ${0}." % le, + } + }, + ), + WrapValidator(validate_tricount), + ] + + def get_tricount_field_definitions(limits: dict[str, int]) -> Any: - """Turn simple integer limits into a dict of constrained positive integer field types.""" - return {field_name: (conint(gt=0, le=limit), None) for field_name, limit in limits.items()} - - -class TriCountConfig(SchemaConfig): - """Triangle count schema title and error messages.""" - - title = "Triangle Count" - error_msg_templates: ClassVar[dict[str, str]] = { - # Prepend the error code to the messages. - e: f"{ERROR_CODE} " + msg - for e, msg in { - "value_error.number.not_gt": "Mesh must have at least 1 triangle.", - "value_error.number.not_le": "Mesh exceeds triangle count budget. Allowed: {limit_value}.", - "value_error.extra": "Invalid extra mesh.", - }.items() + """Turn simple integer limits into a dict of constrained positive integer field types with custom error messages.""" + return { + field_name: ( # Tuple of (type definition, default value). + get_triangle_count_type(limit), + None, # Default value. + ) + for field_name, limit in limits.items() } - @classmethod # As a staticmethod, TypeError is raised with super().schema_extra args. Works with classmethod. - def schema_extra(cls, schema: dict[str, Any], model: type["PydanticBaseModel"]) -> None: - super().schema_extra(schema, model) - # Add custom ajv-error messages to the schema. - for prop in schema.get("properties", {}).values(): - prop["errorMessage"] = { - "exclusiveMinimum": "Mesh ${1/name} must have at least 1 triangle.", - "maximum": "Mesh ${1/name} exceeds triangle count budget. Allowed: %d. Found: ${0}." % prop["maximum"], - } - # Dynamically create the model with the tri-count limits we defined earlier. MeshTriangleCountModel: type[PydanticBaseModel] = create_model( "MeshTriangleCount", - __config__=TriCountConfig, + __config__=get_model_config(title="Triangle Count", validate_default=False), # Populate the fields. **get_tricount_field_definitions(limits), ) if __name__ == "__main__": + import json + import logging + + logging.basicConfig(level=logging.DEBUG) # Convert model to JSON schema. - print(MeshTriangleCountModel.schema_json(indent=2)) + logging.debug(json.dumps(MeshTriangleCountModel.model_json_schema(), indent=2)) # Example of validation in Python. try: # Multiple checks at once. Test non-existent field as well. MeshTriangleCountModel(outfitBottom=6000, outfitTop=0, outfitFootwear=3000, foo=10) - except ValidationError as error: - print("\nValidation Errors:\n", error) + except (PydanticCustomError, ValidationError, TypeError) as error: + logging.error("\nValidation Errors:\n %s", error) From b785648b5dfbbafb06cad90a147b72ee749a8740 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Tue, 8 Aug 2023 15:38:40 +0200 Subject: [PATCH 16/66] fix(schema)!: update common_mesh to pydantic v2 --- .../asset_validation_schemas/common_mesh.py | 75 +++++++++---------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/common_mesh.py b/src/readyplayerme/asset_validation_schemas/common_mesh.py index 2491569..ab40575 100644 --- a/src/readyplayerme/asset_validation_schemas/common_mesh.py +++ b/src/readyplayerme/asset_validation_schemas/common_mesh.py @@ -1,85 +1,82 @@ +"""Validation model for common properties of meshes.""" from enum import Enum -from typing import Any +from typing import Literal -from pydantic import Field, ValidationError, validator +from pydantic import Field, ValidationError +from pydantic.dataclasses import dataclass -from readyplayerme.asset_validation_schemas.basemodel import BaseModel +from readyplayerme.asset_validation_schemas.basemodel import get_model_config class IntegerType(str, Enum): + """Integer types for mesh attributes.""" + u8 = "u8" u16 = "u16" class RenderingMode(str, Enum): + """Rendering modes for meshes.""" + TRIANGLES = "TRIANGLES" LINES = "LINES" POINTS = "POINTS" -class CommonMesh(BaseModel): +@dataclass(config=get_model_config(title="Common Mesh Properties")) +class CommonMesh: """Validation schema for common properties of meshes.""" - class Config(BaseModel.Config): - title = "Common Mesh Properties" - error_msg_templates = {"value_error.number.not_le": "The value must be less than or equal to {limit_value}"} - - @classmethod - def schema_extra(cls, schema: dict[str, Any], model: type["BaseModel"]) -> None: - super().schema_extra(schema, model) - schema["required"] = ["mode", "primitives", "indices", "instances", "size"] - - mode: set[str] = Field( - {RenderingMode.TRIANGLES}, + mode: tuple[Literal[RenderingMode.TRIANGLES]] = Field( description=f"The rendering mode of the mesh. Only {RenderingMode.TRIANGLES.value} are supported.", - errorMessage=f"Rendering mode must be {RenderingMode.TRIANGLES.value}.", - const=True, + json_schema_extra={"errorMessage": f"Rendering mode must be {RenderingMode.TRIANGLES.value}."}, ) primitives: int = Field( ..., ge=1, le=2, description="Number of geometry primitives to be rendered with the given material.", - errorMessage="Number of primitives in the mesh must be 1, or 2 when an additional transparent material is used.", + json_schema_extra={ + "errorMessage": ( + "Number of primitives in the mesh must be 1, or 2 when an additional transparent material is used." + ) + }, ) - indices: set[str] = Field( - {IntegerType.u16}, + indices: tuple[Literal[IntegerType.u16]] = Field( description="The index of the accessor that contains the vertex indices.", - errorMessage=f"Indices must be '{IntegerType.u16.value}' single-item array.", - const=True, + json_schema_extra={"errorMessage": f"Indices must be '{IntegerType.u16.value}' single-item array."}, ) - instances: int = Field( - 1, - const=True, + instances: Literal[1] = Field( description="Number of instances to render.", - errorMessage="Only 1 instance per mesh is supported.", + json_schema_extra={"errorMessage": "Only 1 instance per mesh is supported."}, ) size: int = Field( ..., gt=0, le=524288, description="Byte size. Buffers stored as GLB binary chunk have an implicit limit of (2^32)-1 bytes.", - errorMessage={ - "maximum": "Maximum allowed mesh size is 512 kB.", - "exclusiveMinimum": "Mesh has 0 bytes, seems to be empty!", + json_schema_extra={ + "errorMessage": { + "maximum": "Maximum allowed mesh size is 512 kB.", + "exclusiveMinimum": "Mesh has 0 bytes, seems to be empty!", + } }, ) - @validator("size", pre=True) - def check_size(cls, value): - if value == 0: - msg = "Mesh size must be greater than 0 Bytes." - raise ValueError(msg) - return value - if __name__ == "__main__": + import json + import logging + + from pydantic import TypeAdapter + + logging.basicConfig(level=logging.DEBUG) # Convert model to JSON schema. - print(CommonMesh.schema_json(indent=2)) + logging.debug(json.dumps(TypeAdapter(CommonMesh).json_schema(), indent=2)) # Example of validation in Python. try: # Multiple checks at once. Test non-existent field as well. - model = CommonMesh(mode=["LINES"], primitives=3, indices=["u8"], instances=2, size=1e7) + model = CommonMesh(mode=("LINES",), primitives=3, indices=("u8",), instances=2, size=int(1e7), extra_prop="no!") except ValidationError as error: - print("\nValidation Errors:\n", error) + logging.debug("\nValidation Errors:\n %s", error) From fe50924311dab8730cfc922b1c0adc41fdcecfa5 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Tue, 8 Aug 2023 21:09:13 +0200 Subject: [PATCH 17/66] fix(schema)!: add unified errors to common_mesh json schema and Python errors share messages --- .../asset_validation_schemas/common_mesh.py | 86 ++++++++++++++++--- 1 file changed, 72 insertions(+), 14 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/common_mesh.py b/src/readyplayerme/asset_validation_schemas/common_mesh.py index ab40575..ec387a1 100644 --- a/src/readyplayerme/asset_validation_schemas/common_mesh.py +++ b/src/readyplayerme/asset_validation_schemas/common_mesh.py @@ -1,9 +1,16 @@ """Validation model for common properties of meshes.""" from enum import Enum -from typing import Literal - -from pydantic import Field, ValidationError +from typing import Any, Literal + +from pydantic import ( + Field, + FieldValidationInfo, + ValidationError, + ValidatorFunctionWrapHandler, + field_validator, +) from pydantic.dataclasses import dataclass +from pydantic_core import ErrorDetails, PydanticCustomError from readyplayerme.asset_validation_schemas.basemodel import get_model_config @@ -23,45 +30,96 @@ class RenderingMode(str, Enum): POINTS = "POINTS" +MAXIMUM_MESH_SIZE = 512 # Kb + +render_mode_error = f"Rendering mode must be {RenderingMode.TRIANGLES.value}." +primitives_error = "Number of primitives in the mesh must be 1, or 2 when an additional transparent material is used." +indices_error = f"Indices must be '{IntegerType.u16.value}' single-item array." +instances_error = "Only 1 instance per mesh is supported." +mesh_size_errors = { + "greater_than": f"Maximum allowed mesh size is {MAXIMUM_MESH_SIZE} kB.", + "less_than_equal": "Mesh has 0 bytes, seems to be empty!", +} + + +def get_error_type_msg(field_name: str, error: ErrorDetails) -> tuple[str, str] | tuple[None, None]: + """Convert the error to a custom error type and message. + + If the error type is not covered, return a None-tuple. + """ + match field_name, error["ctx"]: + case "mode", _: + return "RENDER_MODE", render_mode_error + case "primitives", _: + return "PRIMITIVES", primitives_error + case "indices", _: + return "INDICES", indices_error + case "instances", _: + return "INSTANCES", instances_error + case "size", {"le": _}: + return "MESH_SIZE", mesh_size_errors["greater_than"] + case "size", {"gt": _}: + return "MESH_SIZE", mesh_size_errors["less_than_equal"] + case _: + return None, None + + +def custom_error_validator(value: Any, handler: ValidatorFunctionWrapHandler, info: FieldValidationInfo) -> Any: + """Simplify the error message to avoid a gross error stemming from exhaustive checking of all union options.""" + try: + return handler(value) + except ValidationError as error: + for err in error.errors(): + error_type, error_msg = get_error_type_msg(info.field_name, err) + if not error_type or error_msg: # We didn't cover this error, so raise default. + raise + raise PydanticCustomError(error_type, error_msg) from error + + @dataclass(config=get_model_config(title="Common Mesh Properties")) class CommonMesh: """Validation schema for common properties of meshes.""" mode: tuple[Literal[RenderingMode.TRIANGLES]] = Field( + ..., description=f"The rendering mode of the mesh. Only {RenderingMode.TRIANGLES.value} are supported.", - json_schema_extra={"errorMessage": f"Rendering mode must be {RenderingMode.TRIANGLES.value}."}, + json_schema_extra={"errorMessage": render_mode_error}, ) + primitives: int = Field( ..., ge=1, le=2, description="Number of geometry primitives to be rendered with the given material.", - json_schema_extra={ - "errorMessage": ( - "Number of primitives in the mesh must be 1, or 2 when an additional transparent material is used." - ) - }, + json_schema_extra={"errorMessage": primitives_error}, ) + indices: tuple[Literal[IntegerType.u16]] = Field( + ..., description="The index of the accessor that contains the vertex indices.", - json_schema_extra={"errorMessage": f"Indices must be '{IntegerType.u16.value}' single-item array."}, + json_schema_extra={"errorMessage": indices_error}, ) + instances: Literal[1] = Field( + ..., description="Number of instances to render.", - json_schema_extra={"errorMessage": "Only 1 instance per mesh is supported."}, + json_schema_extra={"errorMessage": instances_error}, ) + size: int = Field( ..., gt=0, - le=524288, + le=MAXIMUM_MESH_SIZE * 1024, # Convert to bytes. description="Byte size. Buffers stored as GLB binary chunk have an implicit limit of (2^32)-1 bytes.", json_schema_extra={ "errorMessage": { - "maximum": "Maximum allowed mesh size is 512 kB.", - "exclusiveMinimum": "Mesh has 0 bytes, seems to be empty!", + "maximum": mesh_size_errors["greater_than"], + "exclusiveMinimum": mesh_size_errors["less_than_equal"], } }, ) + # Wrap all field validators in a custom error validator. + val_wrap = field_validator("*", mode="wrap")(custom_error_validator) if __name__ == "__main__": From 88e0176643906718f92d6e8a003a3fcfb88f5eba Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Wed, 9 Aug 2023 12:15:39 +0200 Subject: [PATCH 18/66] fix(schema): split json_schema_extra into subfuncs so they can be re-used independently by other models --- .../asset_validation_schemas/basemodel.py | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index f397915..2263905 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -8,18 +8,30 @@ from pydantic.json_schema import GenerateJsonSchema +def add_metaschema(schema: dict[str, Any]) -> None: + """Add the JSON schema metaschema to a schema.""" + schema["$schema"] = GenerateJsonSchema.schema_dialect + + +def add_schema_id(schema: dict[str, Any], model: type["PydanticBaseModel"]) -> None: + """Add the JSON schema id based on the model name to a schema.""" + schema["$id"] = f"{(name := model.__name__)[0].lower() + name[1:]}.schema.json" + + +def remove_keywords_from_properties(schema: dict[str, Any], keywords: list[str]) -> None: + """Remove given keywords from properties of a schema.""" + for prop in schema.get("properties", {}).values(): + for kw in keywords: + prop.pop(kw, None) + + def json_schema_extra(schema: dict[str, Any], model: type["PydanticBaseModel"]) -> None: """Provide extra JSON schema properties.""" - # Add id. - schema |= { - "$schema": GenerateJsonSchema.schema_dialect, - # Get the "outer" class name with a lower case first letter. - "$id": f"{(name := model.__name__)[0].lower() + name[1:]}.schema.json", - } + # Add metaschema and id. + add_metaschema(schema) + add_schema_id(schema, model) # Remove "title" & "default" from properties. - for prop in schema.get("properties", {}).values(): - prop.pop("title", None) - prop.pop("default", None) + remove_keywords_from_properties(schema, ["title", "default"]) def get_model_config(**kwargs: Any) -> ConfigDict: From 370f2e06ee34e6fafd4376d636c5f72ea8df2f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Sanandr=C3=A9s?= <43515426+Ivan-Sanandres@users.noreply.github.com> Date: Wed, 9 Aug 2023 18:42:34 +0200 Subject: [PATCH 19/66] fix(schema): updated material_names model --- .../material_names.py | 229 +++++++++++------- 1 file changed, 141 insertions(+), 88 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/material_names.py b/src/readyplayerme/asset_validation_schemas/material_names.py index 3bbc640..911d7b3 100644 --- a/src/readyplayerme/asset_validation_schemas/material_names.py +++ b/src/readyplayerme/asset_validation_schemas/material_names.py @@ -1,105 +1,158 @@ -from typing import Any, ClassVar, List, Literal +from enum import Enum +from typing import Annotated, Any, Literal from pydantic import BaseModel as PydanticBaseModel -from pydantic import ValidationError, constr, create_model +from pydantic import ( + Field, + FieldValidationInfo, + ValidationError, + ValidatorFunctionWrapHandler, + create_model, + field_validator, +) +from pydantic.functional_validators import WrapValidator +from pydantic_core import PydanticCustomError + +from readyplayerme.asset_validation_schemas.basemodel import get_model_config + +material_names = { + "beard": "Wolf3D_Beard", + "body": "Wolf3D_Body", + "facewear": "Wolf3D_Facewear", + "glasses": "Wolf3D_Glasses", + "hair": "Wolf3D_Hair", + "halfBodyShirt": "Wolf3D_Shirt", + "head": "Wolf3D_Skin", + "eye": "Wolf3D_Eye", + "teeth": "Wolf3D_Teeth", + "headwear": "Wolf3D_Headwear", + "bottom": "Wolf3D_Outfit_Bottom", + "footwear": "Wolf3D_Outfit_Footwear", + "top": "Wolf3D_Outfit_Top", +} + + +class OutfitMaterialNames(str, Enum): + bottom = material_names["bottom"] + footwear = material_names["footwear"] + top = material_names["top"] + body = material_names["body"] + + +class HeroAvatarMaterialNames(str, Enum): + bottom = material_names["bottom"] + footwear = material_names["footwear"] + top = material_names["top"] + body = material_names["body"] + head = material_names["head"] + eye = material_names["eye"] + teeth = material_names["teeth"] + hair = material_names["hair"] + beard = material_names["beard"] + facewear = material_names["facewear"] + glasses = material_names["glasses"] + headwear = material_names["headwear"] + + +ERROR_CODE = "MATERIAL_NAMES" +DOCS_URL = "https://docs.readyplayer.me/asset-creation-guide/validation/validation-checks/" + + +def get_error_type_msg(field_name: str, value: Any) -> tuple[str, str] | tuple[None, None]: + """Convert the error to a custom error type and message. + + If the error type is not covered, return a None-tuple. + """ + match field_name: + case key if key in material_names: + return ERROR_CODE, f"Material name should be '{material_names[key]}'. Found '{value}' instead." + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL) + case "outfit": + return ERROR_CODE, f"Material name should be one of {', '.join(OutfitMaterialNames)}. Found '{value}' instead." + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL) + case key if key in ("non_customizable_avatar", "nonCustomizableAvatar"): + return ERROR_CODE, f"Material name should be one of {', '.join(HeroAvatarMaterialNames)}. Found '{value}' instead." + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL) + case _: + return None, None -from readyplayerme.asset_validation_schemas.basemodel import SchemaConfig +def custom_error_validator(value: Any, handler: ValidatorFunctionWrapHandler, info: FieldValidationInfo) -> Any: + """Wrap the field validation function to raise custom error types. + + Return the validated value if no error occurred. + """ + try: + return handler(value) + except ValidationError as error: + for err in error.errors(): + error_type, error_msg = get_error_type_msg(info.field_name, err["input"]) + if not error_type or error_msg: # We didn't cover this error, so raise default. + raise + raise PydanticCustomError(error_type, error_msg) from error + + +def get_material_name_type(material_name: str) -> Annotated: + """Return a constrained positive integer field type with custom error messages. + + :param le: The (inclusive) maximum of the integer (less-equal). + """ + return Annotated[ + Literal[material_name], + Field(json_schema_extra={"errorMessage": "Material name should be '%s'. Found ${0} instead." % material_name}) + ] -class MaterialNamesConfig(SchemaConfig): - """Material Names schema title and error messages.""" - title = "Material Names" - error_msg_templates: ClassVar[dict[str, str]] = { - # Prepend the error code to the messages. - "literal": "Material name should be '${const}'. Found ${0} instead.", - "enum": "Material name should be one of ${allowed_values}. Found ${0} instead.", +def get_material_name_field_definitions(material_names: dict[str, str]) -> Any: + """Turn simple integer limits into a dict of constrained positive integer field types with custom error messages.""" + return { + field_name: ( # Tuple of (type definition, default value). + get_material_name_type(material_name), + None, # Default value. + ) + for field_name, material_name in material_names.items() } - @classmethod - def schema_extra(cls, schema: dict[str, Any], model: type[PydanticBaseModel]) -> None: - super().schema_extra(schema, model) - # Add custom ajv-error messages to the schema. - for prop in schema.get("properties", {}).values(): - if "literal" in prop: - prop["errorMessage"] = cls.error_msg_templates["literal"].replace("${const}", prop["literal"]) - elif "enum" in prop: - prop["errorMessage"] = cls.error_msg_templates["enum"].replace("${allowed_values}", str(prop["enum"])) - -class MaterialNames(SchemaConfig): - beard: Literal["Wolf3D_Beard"] - facewear: Literal["Wolf3D_Facewear"] - glasses: Literal["Wolf3D_Glasses"] - hair: Literal["Wolf3D_Hair"] - halfBodyShirt: Literal["Wolf3D_Shirt"] - head: Literal["Wolf3D_Skin"] - headwear: Literal["Wolf3D_Headwear"] - modularBottom: Literal["Wolf3D_Outfit_Bottom"] - modularFootwear: Literal["Wolf3D_Outfit_Footwear"] - modularTop: Literal["Wolf3D_Outfit_Top"] - nonCustomizableAvatar: List[Literal[ - "Wolf3D_Body", - "Wolf3D_Eye", - "Wolf3D_Outfit_Bottom", - "Wolf3D_Outfit_Footwear", - "Wolf3D_Outfit_Top", - "Wolf3D_Skin", - "Wolf3D_Teeth", - "Wolf3D_Hair", - "Wolf3D_Beard", - "Wolf3D_Facewear" - "Wolf3D_Glasses" - "Wolf3D_Headwear" - ]] - outfit: List[Literal[ - "Wolf3D_Body", - "Wolf3D_Outfit_Bottom", - "Wolf3D_Outfit_Footwear", - "Wolf3D_Outfit_Top", - ]] - -MaterialNamesModel = create_model( + +outfit_field = Annotated[ + OutfitMaterialNames, + Field( + json_schema_extra={ + "errorMessage": "Material name should be one of %s. Found ${0} instead." % ", ".join(OutfitMaterialNames) + } + ) +] + +hero_avatar_field = Annotated[ + HeroAvatarMaterialNames, + Field( + json_schema_extra={ + "errorMessage": "Material name should be one of %s. Found ${0} instead." + % ", ".join(HeroAvatarMaterialNames) + } + ) +] + +# Wrap all field validators in a custom error validator. +val_wrap = field_validator("*", mode="wrap")(custom_error_validator) + +MaterialNamesModel: type[PydanticBaseModel] = create_model( "MaterialNames", - beard=(Literal["Wolf3D_Beard"], ...), - facewear=(Literal["Wolf3D_Facewear"], ...), - glasses=(Literal["Wolf3D_Glasses"], ...), - hair=(Literal["Wolf3D_Hair"], ...), - halfBodyShirt=(Literal["Wolf3D_Shirt"], ...), - head=(Literal["Wolf3D_Skin"], ...), - headwear=(Literal["Wolf3D_Headwear"], ...), - modularBottom=(Literal["Wolf3D_Outfit_Bottom"], ...), - modularFootwear=(Literal["Wolf3D_Outfit_Footwear"], ...), - modularTop=(Literal["Wolf3D_Outfit_Top"], ...), - nonCustomizableAvatar=(List[Literal[ - "Wolf3D_Body", - "Wolf3D_Eye", - "Wolf3D_Outfit_Bottom", - "Wolf3D_Outfit_Footwear", - "Wolf3D_Outfit_Top", - "Wolf3D_Skin", - "Wolf3D_Teeth", - "Wolf3D_Hair", - "Wolf3D_Beard", - "Wolf3D_Facewear" - "Wolf3D_Glasses" - "Wolf3D_Headwear" - ]], ...), - outfit=(List[Literal[ - "Wolf3D_Body", - "Wolf3D_Outfit_Bottom", - "Wolf3D_Outfit_Footwear", - "Wolf3D_Outfit_Top", - ]], ...), - __config__=MaterialNamesConfig, + __config__=get_model_config(title="Material Names", validate_default=False), + __validators__={"*": val_wrap}, + **get_material_name_field_definitions(material_names), + outfit=(outfit_field, None), + non_customizable_avatar=(hero_avatar_field, None), ) if __name__ == "__main__": + import json + import logging + + logging.basicConfig(level=logging.DEBUG) # Convert model to JSON schema. - print(MaterialNamesModel.schema_json(indent=2)) + logging.debug(json.dumps(MaterialNamesModel.model_json_schema(), indent=2)) # Example of validation in Python. try: # Test example validation. - MaterialNamesModel(beard="Wrong_Material_Name") - except ValidationError as error: - print("\nValidation Errors:\n", error) + MaterialNamesModel(beard="Wrong_Material_Name", outfit="Other_Wrong_Material_Name") + except (PydanticCustomError, ValidationError, TypeError) as error: + logging.error("\nValidation Errors:\n %s", error) From 0332f20ce40c2729f599acf3f8c60487be2756d2 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Wed, 9 Aug 2023 19:30:02 +0200 Subject: [PATCH 20/66] fix(schema): no validate_default in base config so it does not need to be disabled in each model --- src/readyplayerme/asset_validation_schemas/basemodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index 2263905..33f7681 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -41,7 +41,7 @@ def get_model_config(**kwargs: Any) -> ConfigDict: """ default_dict = { "validate_assignment": True, - "validate_default": True, + "validate_default": False, "strict": True, "populate_by_name": True, "extra": "forbid", From 8994fd2630d0a22ef44d04eb55bb980aaa175521 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Wed, 9 Aug 2023 19:38:59 +0200 Subject: [PATCH 21/66] fix(schema): fix custom validation error --- .../asset_validation_schemas/common_mesh.py | 18 +++---- .../material_names.py | 50 ++++++++++++------- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/common_mesh.py b/src/readyplayerme/asset_validation_schemas/common_mesh.py index ec387a1..434f261 100644 --- a/src/readyplayerme/asset_validation_schemas/common_mesh.py +++ b/src/readyplayerme/asset_validation_schemas/common_mesh.py @@ -10,7 +10,7 @@ field_validator, ) from pydantic.dataclasses import dataclass -from pydantic_core import ErrorDetails, PydanticCustomError +from pydantic_core import PydanticCustomError from readyplayerme.asset_validation_schemas.basemodel import get_model_config @@ -33,7 +33,7 @@ class RenderingMode(str, Enum): MAXIMUM_MESH_SIZE = 512 # Kb render_mode_error = f"Rendering mode must be {RenderingMode.TRIANGLES.value}." -primitives_error = "Number of primitives in the mesh must be 1, or 2 when an additional transparent material is used." +primitives_error = "Number of primitives in the mesh must be 1." indices_error = f"Indices must be '{IntegerType.u16.value}' single-item array." instances_error = "Only 1 instance per mesh is supported." mesh_size_errors = { @@ -42,12 +42,12 @@ class RenderingMode(str, Enum): } -def get_error_type_msg(field_name: str, error: ErrorDetails) -> tuple[str, str] | tuple[None, None]: +def get_error_type_msg(field_name: str, error: dict[str, Any]) -> tuple[str, str] | tuple[None, None]: """Convert the error to a custom error type and message. If the error type is not covered, return a None-tuple. """ - match field_name, error["ctx"]: + match field_name, error: case "mode", _: return "RENDER_MODE", render_mode_error case "primitives", _: @@ -70,10 +70,10 @@ def custom_error_validator(value: Any, handler: ValidatorFunctionWrapHandler, in return handler(value) except ValidationError as error: for err in error.errors(): - error_type, error_msg = get_error_type_msg(info.field_name, err) - if not error_type or error_msg: # We didn't cover this error, so raise default. - raise - raise PydanticCustomError(error_type, error_msg) from error + error_type, error_msg = get_error_type_msg(info.field_name, err["ctx"]) + if error_type and error_msg: + raise PydanticCustomError(error_type, error_msg) from error + raise # We didn't cover this error, so raise default. @dataclass(config=get_model_config(title="Common Mesh Properties")) @@ -89,7 +89,7 @@ class CommonMesh: primitives: int = Field( ..., ge=1, - le=2, + le=1, description="Number of geometry primitives to be rendered with the given material.", json_schema_extra={"errorMessage": primitives_error}, ) diff --git a/src/readyplayerme/asset_validation_schemas/material_names.py b/src/readyplayerme/asset_validation_schemas/material_names.py index 911d7b3..585f590 100644 --- a/src/readyplayerme/asset_validation_schemas/material_names.py +++ b/src/readyplayerme/asset_validation_schemas/material_names.py @@ -1,3 +1,4 @@ +"""Model for allowed names of materials for different asset types.""" from enum import Enum from typing import Annotated, Any, Literal @@ -10,7 +11,6 @@ create_model, field_validator, ) -from pydantic.functional_validators import WrapValidator from pydantic_core import PydanticCustomError from readyplayerme.asset_validation_schemas.basemodel import get_model_config @@ -33,6 +33,8 @@ class OutfitMaterialNames(str, Enum): + """Allowed names of materials for outfit assets.""" + bottom = material_names["bottom"] footwear = material_names["footwear"] top = material_names["top"] @@ -40,6 +42,8 @@ class OutfitMaterialNames(str, Enum): class HeroAvatarMaterialNames(str, Enum): + """Allowed names of materials for hero avatar assets.""" + bottom = material_names["bottom"] footwear = material_names["footwear"] top = material_names["top"] @@ -54,7 +58,7 @@ class HeroAvatarMaterialNames(str, Enum): headwear = material_names["headwear"] -ERROR_CODE = "MATERIAL_NAMES" +ERROR_CODE = "MATERIAL_NAME" DOCS_URL = "https://docs.readyplayer.me/asset-creation-guide/validation/validation-checks/" @@ -65,11 +69,23 @@ def get_error_type_msg(field_name: str, value: Any) -> tuple[str, str] | tuple[N """ match field_name: case key if key in material_names: - return ERROR_CODE, f"Material name should be '{material_names[key]}'. Found '{value}' instead." + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL) + return ( + ERROR_CODE, + f"Material name should be '{material_names[key]}'. Found '{value}' instead." + + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), + ) case "outfit": - return ERROR_CODE, f"Material name should be one of {', '.join(OutfitMaterialNames)}. Found '{value}' instead." + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL) + return ( + ERROR_CODE, + f"Material name should be one of {', '.join(OutfitMaterialNames)}. Found '{value}' instead." + + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), + ) case key if key in ("non_customizable_avatar", "nonCustomizableAvatar"): - return ERROR_CODE, f"Material name should be one of {', '.join(HeroAvatarMaterialNames)}. Found '{value}' instead." + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL) + return ( + ERROR_CODE, + f"Material name should be one of {', '.join(HeroAvatarMaterialNames)}. Found '{value}' instead." + + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), + ) case _: return None, None @@ -84,19 +100,16 @@ def custom_error_validator(value: Any, handler: ValidatorFunctionWrapHandler, in except ValidationError as error: for err in error.errors(): error_type, error_msg = get_error_type_msg(info.field_name, err["input"]) - if not error_type or error_msg: # We didn't cover this error, so raise default. - raise - raise PydanticCustomError(error_type, error_msg) from error + if error_type and error_msg: + raise PydanticCustomError(error_type, error_msg) from error + raise # We didn't cover this error, so raise default. def get_material_name_type(material_name: str) -> Annotated: - """Return a constrained positive integer field type with custom error messages. - - :param le: The (inclusive) maximum of the integer (less-equal). - """ + """Return a constrained positive integer field type with custom error messages.""" return Annotated[ Literal[material_name], - Field(json_schema_extra={"errorMessage": "Material name should be '%s'. Found ${0} instead." % material_name}) + Field(json_schema_extra={"errorMessage": "Material name should be '%s'. Found ${0} instead." % material_name}), ] @@ -111,13 +124,14 @@ def get_material_name_field_definitions(material_names: dict[str, str]) -> Any: } +# Define fields for outfit assets and hero avatar assets. outfit_field = Annotated[ OutfitMaterialNames, Field( json_schema_extra={ "errorMessage": "Material name should be one of %s. Found ${0} instead." % ", ".join(OutfitMaterialNames) } - ) + ), ] hero_avatar_field = Annotated[ @@ -127,16 +141,16 @@ def get_material_name_field_definitions(material_names: dict[str, str]) -> Any: "errorMessage": "Material name should be one of %s. Found ${0} instead." % ", ".join(HeroAvatarMaterialNames) } - ) + ), ] # Wrap all field validators in a custom error validator. -val_wrap = field_validator("*", mode="wrap")(custom_error_validator) +wrapped_validator = field_validator("*", mode="wrap")(custom_error_validator) MaterialNamesModel: type[PydanticBaseModel] = create_model( "MaterialNames", - __config__=get_model_config(title="Material Names", validate_default=False), - __validators__={"*": val_wrap}, + __config__=get_model_config(title="Material Names"), + __validators__={"*": wrapped_validator}, **get_material_name_field_definitions(material_names), outfit=(outfit_field, None), non_customizable_avatar=(hero_avatar_field, None), From f2843a49f1b7eaae5ecb95b07fd1813f05574ea1 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Wed, 9 Aug 2023 20:01:51 +0200 Subject: [PATCH 22/66] fix(schema): unify material names error messages share error message between python and json schema by use of format() --- .../asset_validation_schemas/material_names.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/material_names.py b/src/readyplayerme/asset_validation_schemas/material_names.py index 585f590..03b1c97 100644 --- a/src/readyplayerme/asset_validation_schemas/material_names.py +++ b/src/readyplayerme/asset_validation_schemas/material_names.py @@ -59,6 +59,8 @@ class HeroAvatarMaterialNames(str, Enum): ERROR_CODE = "MATERIAL_NAME" +ERROR_MSG = "Material name should be {material_name}. Found {value} instead." +ERROR_MSG_MULTI = "Material name should be one of {material_names}. Found {value} instead." DOCS_URL = "https://docs.readyplayer.me/asset-creation-guide/validation/validation-checks/" @@ -71,19 +73,19 @@ def get_error_type_msg(field_name: str, value: Any) -> tuple[str, str] | tuple[N case key if key in material_names: return ( ERROR_CODE, - f"Material name should be '{material_names[key]}'. Found '{value}' instead." + ERROR_MSG.format(material_name=material_names[key], value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) case "outfit": return ( ERROR_CODE, - f"Material name should be one of {', '.join(OutfitMaterialNames)}. Found '{value}' instead." + ERROR_MSG_MULTI.format(material_names=", ".join(OutfitMaterialNames), value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) case key if key in ("non_customizable_avatar", "nonCustomizableAvatar"): return ( ERROR_CODE, - f"Material name should be one of {', '.join(HeroAvatarMaterialNames)}. Found '{value}' instead." + ERROR_MSG_MULTI.format(material_names=", ".join(HeroAvatarMaterialNames), value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) case _: @@ -109,7 +111,7 @@ def get_material_name_type(material_name: str) -> Annotated: """Return a constrained positive integer field type with custom error messages.""" return Annotated[ Literal[material_name], - Field(json_schema_extra={"errorMessage": "Material name should be '%s'. Found ${0} instead." % material_name}), + Field(json_schema_extra={"errorMessage": ERROR_MSG.format(material_name=material_name, value="${0}")}), ] @@ -129,7 +131,7 @@ def get_material_name_field_definitions(material_names: dict[str, str]) -> Any: OutfitMaterialNames, Field( json_schema_extra={ - "errorMessage": "Material name should be one of %s. Found ${0} instead." % ", ".join(OutfitMaterialNames) + "errorMessage": ERROR_MSG_MULTI.format(material_names=", ".join(OutfitMaterialNames), value="${0}") } ), ] @@ -138,8 +140,7 @@ def get_material_name_field_definitions(material_names: dict[str, str]) -> Any: HeroAvatarMaterialNames, Field( json_schema_extra={ - "errorMessage": "Material name should be one of %s. Found ${0} instead." - % ", ".join(HeroAvatarMaterialNames) + "errorMessage": ERROR_MSG_MULTI.format(material_names=", ".join(HeroAvatarMaterialNames), value="${0}") } ), ] From b5f7b8ced3adcf2821af022d70fb6939cb076e9c Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Wed, 9 Aug 2023 22:58:17 +0200 Subject: [PATCH 23/66] chore: add mypy type checking pre-commit hook --- .pre-commit-config.yaml | 5 +++++ pyproject.toml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2bc05ba..7d68b84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,3 +35,8 @@ repos: hooks: - id: docstr-coverage args: ["--verbose", "2"] # override the .docstr.yaml to see less output + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.4.1' + hooks: + - id: mypy diff --git a/pyproject.toml b/pyproject.toml index 16f85c0..5a160e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -223,6 +223,10 @@ disallow_any_generics = true check_untyped_defs = true no_implicit_reexport = true +[[tool.mypy.overrides]] +module = "readyplayerme.*" +ignore_missing_imports = true + # for strict mypy: disallow_untyped_defs = false From 691c3ca6a00de573874466328c30fca20054f2f1 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Wed, 9 Aug 2023 23:10:43 +0200 Subject: [PATCH 24/66] chore: add pydantic dependency to mypy pre-commit --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d68b84..55002f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,3 +40,5 @@ repos: rev: 'v1.4.1' hooks: - id: mypy + additional_dependencies: + - "pydantic>=2.1" From a2834fc413056b9de16972143afd88e01a7bda13 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Wed, 9 Aug 2023 23:11:15 +0200 Subject: [PATCH 25/66] fix(linting): ignore ConfigDict kwargs type check --- src/readyplayerme/asset_validation_schemas/basemodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index 33f7681..e9a1382 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -52,7 +52,7 @@ def get_model_config(**kwargs: Any) -> ConfigDict: "frozen": True, } updated_dict = default_dict | kwargs - return ConfigDict(**updated_dict) + return ConfigDict(**updated_dict) # type: ignore[misc] class BaseModel(PydanticBaseModel, abc.ABC): From 5a90c561cd57d8311d51c793a7b4a2d832edee93 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 10 Aug 2023 04:11:34 +0200 Subject: [PATCH 26/66] fix(schema): use dynamically created enums use enum values to create model field types --- .../material_names.py | 95 ++++++++++--------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/material_names.py b/src/readyplayerme/asset_validation_schemas/material_names.py index 03b1c97..b249e7a 100644 --- a/src/readyplayerme/asset_validation_schemas/material_names.py +++ b/src/readyplayerme/asset_validation_schemas/material_names.py @@ -1,6 +1,7 @@ """Model for allowed names of materials for different asset types.""" +from collections.abc import Container, Iterable from enum import Enum -from typing import Annotated, Any, Literal +from typing import Annotated, Any, Literal, cast from pydantic import BaseModel as PydanticBaseModel from pydantic import ( @@ -32,35 +33,37 @@ } -class OutfitMaterialNames(str, Enum): - """Allowed names of materials for outfit assets.""" +# Instead of spelling out members and values for each enum, create classes dynamically. +def create_enum_class(name: str, dictionary: dict[str, str], keys: Container[str] | None = None) -> Enum: + """Create an string-enum class from a dictionary. - bottom = material_names["bottom"] - footwear = material_names["footwear"] - top = material_names["top"] - body = material_names["body"] + If keys are provided, only the keys will be included in the enum class. + """ + + def is_key_set(item: tuple[str, str]) -> bool: + return item[0] in keys if keys else True + if keys is None: + members = dictionary + else: + members = dict(filter(is_key_set, dictionary.items())) + return Enum(name, members, type=str) -class HeroAvatarMaterialNames(str, Enum): - """Allowed names of materials for hero avatar assets.""" - bottom = material_names["bottom"] - footwear = material_names["footwear"] - top = material_names["top"] - body = material_names["body"] - head = material_names["head"] - eye = material_names["eye"] - teeth = material_names["teeth"] - hair = material_names["hair"] - beard = material_names["beard"] - facewear = material_names["facewear"] - glasses = material_names["glasses"] - headwear = material_names["headwear"] +AllMaterialNames = create_enum_class("AllMaterialNames", material_names) + +OutfitMaterialNames = create_enum_class("OutfitMaterialNames", material_names, {"bottom", "footwear", "top", "body"}) + +HeroAvatarMaterialNames = create_enum_class( + "HeroAvatarMaterialNames", + material_names, + {"bottom", "footwear", "top", "body", "head", "eye", "teeth", "hair", "beard", "facewear", "glasses", "headwear"}, +) ERROR_CODE = "MATERIAL_NAME" -ERROR_MSG = "Material name should be {material_name}. Found {value} instead." -ERROR_MSG_MULTI = "Material name should be one of {material_names}. Found {value} instead." +ERROR_MSG = "Material name should be {valid_name}. Found {value} instead." +ERROR_MSG_MULTI = "Material name should be one of {valid_names}. Found {value} instead." DOCS_URL = "https://docs.readyplayer.me/asset-creation-guide/validation/validation-checks/" @@ -70,26 +73,25 @@ def get_error_type_msg(field_name: str, value: Any) -> tuple[str, str] | tuple[N If the error type is not covered, return a None-tuple. """ match field_name: - case key if key in material_names: + case key if key in AllMaterialNames.__members__: # type: ignore[attr-defined] return ( ERROR_CODE, - ERROR_MSG.format(material_name=material_names[key], value=value) + ERROR_MSG.format(valid_name=getattr(AllMaterialNames, key).value, value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) case "outfit": return ( ERROR_CODE, - ERROR_MSG_MULTI.format(material_names=", ".join(OutfitMaterialNames), value=value) + ERROR_MSG_MULTI.format(valid_names=", ".join(cast(Iterable[str], OutfitMaterialNames)), value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) case key if key in ("non_customizable_avatar", "nonCustomizableAvatar"): return ( ERROR_CODE, - ERROR_MSG_MULTI.format(material_names=", ".join(HeroAvatarMaterialNames), value=value) + ERROR_MSG_MULTI.format(valid_names=", ".join(cast(Iterable[str], HeroAvatarMaterialNames)), value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) - case _: - return None, None + return None, None def custom_error_validator(value: Any, handler: ValidatorFunctionWrapHandler, info: FieldValidationInfo) -> Any: @@ -107,40 +109,45 @@ def custom_error_validator(value: Any, handler: ValidatorFunctionWrapHandler, in raise # We didn't cover this error, so raise default. -def get_material_name_type(material_name: str) -> Annotated: - """Return a constrained positive integer field type with custom error messages.""" +def get_const_str_field_type(const: str) -> Any: + """Return a constant-string field type with custom error messages.""" return Annotated[ - Literal[material_name], - Field(json_schema_extra={"errorMessage": ERROR_MSG.format(material_name=material_name, value="${0}")}), + # While this is not really a Literal, since we illegally use a variable, it works as "const" in json schema. + Literal[const], + Field(json_schema_extra={"errorMessage": ERROR_MSG.format(valid_name=const, value="${0}")}), ] -def get_material_name_field_definitions(material_names: dict[str, str]) -> Any: - """Turn simple integer limits into a dict of constrained positive integer field types with custom error messages.""" +def get_field_definitions(field_input: Enum) -> Any: + """Turn a StrEnum into field types of string-constants.""" return { - field_name: ( # Tuple of (type definition, default value). - get_material_name_type(material_name), + member.name: ( # Tuple of (type definition, default value). + get_const_str_field_type(member.value), None, # Default value. ) - for field_name, material_name in material_names.items() + for member in field_input # type: ignore[attr-defined] } # Define fields for outfit assets and hero avatar assets. outfit_field = Annotated[ - OutfitMaterialNames, + OutfitMaterialNames, # type: ignore[valid-type] Field( json_schema_extra={ - "errorMessage": ERROR_MSG_MULTI.format(material_names=", ".join(OutfitMaterialNames), value="${0}") + "errorMessage": ERROR_MSG_MULTI.format( + valid_names=", ".join(cast(Iterable[str], OutfitMaterialNames)), value="${0}" + ) } ), ] hero_avatar_field = Annotated[ - HeroAvatarMaterialNames, + HeroAvatarMaterialNames, # type: ignore[valid-type] Field( json_schema_extra={ - "errorMessage": ERROR_MSG_MULTI.format(material_names=", ".join(HeroAvatarMaterialNames), value="${0}") + "errorMessage": ERROR_MSG_MULTI.format( + valid_names=", ".join(cast(Iterable[str], HeroAvatarMaterialNames)), value="${0}" + ) } ), ] @@ -151,8 +158,8 @@ def get_material_name_field_definitions(material_names: dict[str, str]) -> Any: MaterialNamesModel: type[PydanticBaseModel] = create_model( "MaterialNames", __config__=get_model_config(title="Material Names"), - __validators__={"*": wrapped_validator}, - **get_material_name_field_definitions(material_names), + __validators__={"*": wrapped_validator}, # type: ignore[dict-item] + **get_field_definitions(AllMaterialNames), outfit=(outfit_field, None), non_customizable_avatar=(hero_avatar_field, None), ) From 411847be443be75a62678967d7f6d43b10abd2fa Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 10 Aug 2023 04:26:13 +0200 Subject: [PATCH 27/66] fix(schema): agnostic name for error replacement should increase reusability of functions --- .../asset_validation_schemas/material_names.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/material_names.py b/src/readyplayerme/asset_validation_schemas/material_names.py index b249e7a..e7987af 100644 --- a/src/readyplayerme/asset_validation_schemas/material_names.py +++ b/src/readyplayerme/asset_validation_schemas/material_names.py @@ -62,8 +62,8 @@ def is_key_set(item: tuple[str, str]) -> bool: ERROR_CODE = "MATERIAL_NAME" -ERROR_MSG = "Material name should be {valid_name}. Found {value} instead." -ERROR_MSG_MULTI = "Material name should be one of {valid_names}. Found {value} instead." +ERROR_MSG = "Material name should be {valid_value}. Found {value} instead." +ERROR_MSG_MULTI = "Material name should be one of {valid_values}. Found {value} instead." DOCS_URL = "https://docs.readyplayer.me/asset-creation-guide/validation/validation-checks/" @@ -76,19 +76,21 @@ def get_error_type_msg(field_name: str, value: Any) -> tuple[str, str] | tuple[N case key if key in AllMaterialNames.__members__: # type: ignore[attr-defined] return ( ERROR_CODE, - ERROR_MSG.format(valid_name=getattr(AllMaterialNames, key).value, value=value) + ERROR_MSG.format(valid_value=getattr(AllMaterialNames, key).value, value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) case "outfit": return ( ERROR_CODE, - ERROR_MSG_MULTI.format(valid_names=", ".join(cast(Iterable[str], OutfitMaterialNames)), value=value) + ERROR_MSG_MULTI.format(valid_values=", ".join(cast(Iterable[str], OutfitMaterialNames)), value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) case key if key in ("non_customizable_avatar", "nonCustomizableAvatar"): return ( ERROR_CODE, - ERROR_MSG_MULTI.format(valid_names=", ".join(cast(Iterable[str], HeroAvatarMaterialNames)), value=value) + ERROR_MSG_MULTI.format( + valid_values=", ".join(cast(Iterable[str], HeroAvatarMaterialNames)), value=value + ) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) return None, None @@ -114,7 +116,7 @@ def get_const_str_field_type(const: str) -> Any: return Annotated[ # While this is not really a Literal, since we illegally use a variable, it works as "const" in json schema. Literal[const], - Field(json_schema_extra={"errorMessage": ERROR_MSG.format(valid_name=const, value="${0}")}), + Field(json_schema_extra={"errorMessage": ERROR_MSG.format(valid_value=const, value="${0}")}), ] @@ -135,7 +137,7 @@ def get_field_definitions(field_input: Enum) -> Any: Field( json_schema_extra={ "errorMessage": ERROR_MSG_MULTI.format( - valid_names=", ".join(cast(Iterable[str], OutfitMaterialNames)), value="${0}" + valid_values=", ".join(cast(Iterable[str], OutfitMaterialNames)), value="${0}" ) } ), @@ -146,7 +148,7 @@ def get_field_definitions(field_input: Enum) -> Any: Field( json_schema_extra={ "errorMessage": ERROR_MSG_MULTI.format( - valid_names=", ".join(cast(Iterable[str], HeroAvatarMaterialNames)), value="${0}" + valid_values=", ".join(cast(Iterable[str], HeroAvatarMaterialNames)), value="${0}" ) } ), From b23a48db271b5747d980bf6e66cf49d82c6ae4f7 Mon Sep 17 00:00:00 2001 From: TechyDaniel <117300186+TechyDaniel@users.noreply.github.com> Date: Thu, 10 Aug 2023 18:55:19 +0200 Subject: [PATCH 28/66] feat: Python model fro generating schema for textures Co-authored-by: Olaf Haag Co-authored-by: IvanRPM --- .../common_textures.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/common_textures.py diff --git a/src/readyplayerme/asset_validation_schemas/common_textures.py b/src/readyplayerme/asset_validation_schemas/common_textures.py new file mode 100644 index 0000000..9fffcc8 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/common_textures.py @@ -0,0 +1,105 @@ +""" +Common Texture Validation Schema. + +This module defines a Pydantic model for validating common properties of texture maps. It includes error messages +for validation failures and provides a JSON schema for the model. + +Author: Daniel-Ionut Rancea +Co-authored-by: Olaf Haag +Co-authored-by: Ivan Sanandres Gutierrez +""" +from enum import Enum +from typing import Literal + +from pydantic import ConfigDict, Field, ValidationError + +from readyplayerme.asset_validation_schemas.basemodel import BaseModel + +MAX_FILE_SIZE = 2 # in MB +MAX_GPU_SIZE = 6 # in MB +# TODO: Figure out how to reference other fields in error messages. Maybe use model_validator instead of field_validator +INSTANCE_ERROR_MSG = "Texture map is unused." +MIMETYPE_ERROR_MSG = "Texture map must be encoded as PNG or JPEG. Found {value} instead." +RESOLUTION_ERROR_MSG = "Image resolution must be a power of 2 and square. Maximum {valid_value}. Found {value} instead." +FILE_SIZE_ERROR_MSG = "Texture map exceeds maximum allowed storage size of {valid_value} MB." +GPU_SIZE_ERROR_MSG = "Texture map exceeds maximum allowed GPU size of {valid_value} MB when fully decompressed." + + +class ResolutionType(str, Enum): + """Image resolution data used for textures. Power of 2 and square.""" + + _1x1 = "1x1" + _2x2 = "2x2" + _4x4 = "4x4" + _8x8 = "8x8" + _16x16 = "16x16" + _32x32 = "32x32" + _64x64 = "64x64" + _128x128 = "128x128" + _256x256 = "256x256" + _512x512 = "512x512" + _1024x1024 = "1024x1024" + + def __str__(self): + """ + Get a string representation of the ResolutionType enum value. + + Returns: + str: The string representation of the enum value. + """ + return self.value + + +class CommonTexture(BaseModel): + """Validation schema for common properties of texture maps.""" + + model_config = ConfigDict(title="Common Texture Map Properties") + + name: str + uri: str + instances: int = Field(..., ge=1, json_schema_extra={"errorMessages": {"minimum": INSTANCE_ERROR_MSG}}) + mime_type: Literal["image/png", "image/jpeg"] = Field( + json_schema_extra={"errorMessages": MIMETYPE_ERROR_MSG.format(value="${0}")} + ) + compression: str + resolution: ResolutionType = Field( + ..., + json_schema_extra={ + "errorMessages": RESOLUTION_ERROR_MSG.format(valid_value=str(list(ResolutionType)[-1]), value="${0}") + }, + ) + size: int = Field( + ..., + le=MAX_FILE_SIZE * 1024**2, + json_schema_extra={"errorMessages": {"maximum": FILE_SIZE_ERROR_MSG.format(valid_value=MAX_FILE_SIZE)}}, + ) # Convert to bytes. + gpu_size: int = Field( + ..., + le=MAX_GPU_SIZE * 1024**2, + json_schema_extra={"errorMessages": {"maximum": GPU_SIZE_ERROR_MSG.format(valid_value=MAX_GPU_SIZE)}}, + ) + + +# Print the generated JSON schema with indentation +if __name__ == "__main__": + import json + import logging + + logging.basicConfig(level=logging.DEBUG) + # Convert model to JSON schema. + logging.debug(json.dumps(CommonTexture.model_json_schema(), indent=2)) + + # Example of validation in Python + try: + CommonTexture( + name="normalmap", + uri="path/to/normal.png", + instances=0, + mime_type="image/webP", + compression="default", + resolution="2048x1024", + size=3097152, + gpu_size=20291456, + ) + except ValidationError as error: + logging.debug("\nValidation Errors:\n %s" % error) From 928dae740247e17d525b7f73376478ecc54170f1 Mon Sep 17 00:00:00 2001 From: TechyDaniel <117300186+TechyDaniel@users.noreply.github.com> Date: Thu, 10 Aug 2023 18:56:03 +0200 Subject: [PATCH 29/66] Update: disabled D105 check for ruff --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5a160e6..3ad544e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,8 +151,8 @@ ignore = [ "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", # Exclude self, cls, args, kwargs from annotation, allow dynamically typed expressions (typing.Any) in type annotations "ANN101", "ANN102", "ANN002", "ANN003", "ANN401", - # Don't require documentation for every function parameter. - "D417", "D102", + # Don't require documentation for every function parameter and magic methods. + "D417", "D102", "D105" ] builtins = ["_"] unfixable = [ From f0ab5b6d2028f4410c87c5add4f6d2064c279a59 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 10 Aug 2023 19:27:06 +0200 Subject: [PATCH 30/66] chore: ruff ignore missing doc-string in __init__ --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3ad544e..7411fd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,7 +152,7 @@ ignore = [ # Exclude self, cls, args, kwargs from annotation, allow dynamically typed expressions (typing.Any) in type annotations "ANN101", "ANN102", "ANN002", "ANN003", "ANN401", # Don't require documentation for every function parameter and magic methods. - "D417", "D102", "D105" + "D417", "D102", "D105", "D107" ] builtins = ["_"] unfixable = [ From 12a9c8e9d56ab0fc8738bc9f705464fbd3346e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Sanandr=C3=A9s?= <43515426+Ivan-Sanandres@users.noreply.github.com> Date: Fri, 11 Aug 2023 17:56:09 +0200 Subject: [PATCH 31/66] fix(Schema): revised meshAttributes schema, added skeleton of pydantic V2 model to meshAttributes --- schemas/meshAttributes.schema.json | 4 +- .../mesh_atributes.py | 169 ++++++++++++++---- 2 files changed, 133 insertions(+), 40 deletions(-) diff --git a/schemas/meshAttributes.schema.json b/schemas/meshAttributes.schema.json index 1ae98f0..fd0405d 100644 --- a/schemas/meshAttributes.schema.json +++ b/schemas/meshAttributes.schema.json @@ -16,11 +16,11 @@ "contains": { "type": "string", "pattern": "NORMAL:f32|POSITION:f32|TEXCOORD_0:f32", - "errorMessage": "Mesh ${2/name} requires at least 5 vertex attributes: position, normal, 1 UV set, joint influences, and weights. Found ${1/length} attributes: ${1}." + "errorMessage": "Mesh ${2/name} requires at least 3 vertex attributes: position, normal and 1 UV set. Found ${1/length} attributes: ${1}." }, "minContains": 3, "uniqueItems": true, - "errorMessage": "Mesh ${1/name} requires at least 5 vertex attributes: position, normal, 1 UV set, joint influences, and weights. Found ${0/length} attributes: ${0}." + "errorMessage": "Mesh ${1/name} requires at least 3 vertex attributes: position, normal and 1 UV set. Found ${0/length} attributes: ${0}." }, "skinned": { "description": "Attributes for skinned meshes. Optional tangents.", diff --git a/src/readyplayerme/asset_validation_schemas/mesh_atributes.py b/src/readyplayerme/asset_validation_schemas/mesh_atributes.py index 78d8835..81957b9 100644 --- a/src/readyplayerme/asset_validation_schemas/mesh_atributes.py +++ b/src/readyplayerme/asset_validation_schemas/mesh_atributes.py @@ -1,52 +1,145 @@ -from typing import Any, ClassVar, List, Literal +from typing import Any, Literal, Union -from pydantic import BaseModel as PydanticBaseModel -from pydantic import ValidationError, constr +from pydantic import ValidationError, ValidatorFunctionWrapHandler, FieldValidationInfo, field_validator +from pydantic.dataclasses import dataclass +from pydantic_core import PydanticCustomError -from readyplayerme.asset_validation_schemas.basemodel import SchemaConfig +from readyplayerme.asset_validation_schemas.basemodel import get_model_config +ERROR_CODE = "MESH_ATTRIBUTES" +ERROR_MSG_UNSKINNED = [ + "Mesh {name} error! Allowed attributes are: NORMAL, POSITION, TEXCOORD_0, TANGENT. Found {wrong_value}", + "Mesh {name} requires at least 3 vertex attributes: position, normal and 1 UV set. Found {number} attributes: {attributes}.", +] +ERROR_MSG_SKINNED = [ + "Mesh {name} error! Allowed attributes are: JOINTS_0, NORMAL, POSITION, TEXCOORD_0, TANGENT, WEIGHTS_0. Found {wrong_value}", + "Mesh {name} requires at least 5 vertex attributes: position, normal, 1 UV set, joint influences, and weights. Found {number} attributes: {attributes}.", +] +DOCS_URL = "https://docs.readyplayer.me/asset-creation-guide/validation/validation-checks/" -class MeshAttributesConfig(SchemaConfig): - """Mesh Attributes schema title and error messages.""" +unskinned_possible_attributes = ["NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32", "TANGENT:f32"] +skinned_possible_attributes = [ + "JOINTS_0:u8", + "NORMAL:f32", + "POSITION:f32", + "TEXCOORD_0:f32", + "TANGENT:f32", + "WEIGHTS_0:f32", +] - title = "Mesh Attributes" - error_msg_templates: ClassVar[dict[str, str]] = { - # Prepend the error code to the messages. - "enum": "Mesh ${2/name} error! Allowed attributes are: ${enum_values}. Found ${0}.", - "contains": "Mesh ${2/name} requires at least 5 vertex attributes: position, normal, 1 UV set, joint influences, and weights. Found ${0/length} attributes: ${0}.", - } - @classmethod - def schema_extra(cls, schema: dict[str, Any], model: type[PydanticBaseModel]) -> None: - super().schema_extra(schema, model) - # Add custom ajv-error messages to the schema. - for prop in schema.get("properties", {}).values(): - if "enum" in prop: - prop["errorMessage"] = cls.error_msg_templates["enum"].replace( - "${enum_values}", ", ".join(prop["enum"]) - ) - elif "contains" in prop: - prop["errorMessage"] = cls.error_msg_templates["contains"] +def get_error_type_msg_unskinned(field_name: str, attr: str, items: tuple[Literal]) -> tuple[None, None]: + """Convert the error to a custom error type and message. -class MeshAttributes(SchemaConfig): - """Pydantic model for Mesh Attributes.""" + If the error type is not covered, return a None-tuple. + """ + match field_name: + case attr if attr in unskinned_possible_attributes: + return (ERROR_CODE, ERROR_MSG_UNSKINNED[0].format(wrong_value=attr)) + case items if len(items) < 3: + return (ERROR_CODE, ERROR_MSG_UNSKINNED[1].format(number=len(items), attributes=items)) + case _: + return None, None - unskinned: List[constr( - enum=["NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32", "TANGENT:f32"], - )] - skinned: List[constr( - enum=["JOINTS_0:u8", "NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32", "TANGENT:f32", "WEIGHTS_0:f32"], - )] + +def custom_error_validator_unskinned( + value: Any, handler: ValidatorFunctionWrapHandler, info: FieldValidationInfo +) -> Any: + """Simplify the error message to avoid a gross error stemming from exhaustive checking of all union options.""" + try: + return handler(value) + except ValidationError as error: + for err in error.errors(): + error_type, error_msg = get_error_type_msg_unskinned(info.field_name, err["input"], value) + if error_type and error_msg: + raise PydanticCustomError(error_type, error_msg) from error + raise # We didn't cover this error, so raise default. + + +def get_error_type_msg_skinned(field_name: str, error: dict[str, Any]) -> tuple[str, str] | tuple[None, None]: + """Convert the error to a custom error type and message. + + If the error type is not covered, return a None-tuple. + """ + match field_name, error: + case "items", _: + return "RENDER_MODE", ERROR_MSG_SKINNED[0] + case "contains", _: + return "PRIMITIVES", ERROR_MSG_SKINNED[1] + case _: + return None, None + + +def custom_error_validator_skinned(value: Any, handler: ValidatorFunctionWrapHandler, info: FieldValidationInfo) -> Any: + """Simplify the error message to avoid a gross error stemming from exhaustive checking of all union options.""" + try: + return handler(value) + except ValidationError as error: + for err in error.errors(): + error_type, error_msg = get_error_type_msg_skinned(info.field_name, err["input"]) + if error_type and error_msg: + raise PydanticCustomError(error_type, error_msg) from error + raise # We didn't cover this error, so raise default. + + +@dataclass(config=get_model_config(title="Unskinned Mesh Attributes")) +class Unskinned: + """Validation schema for unskinned mesh attributes.""" + + items: tuple[Literal["NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32", "TANGENT:f32"], ...] = Field( + ..., + json_schema_extra={ + "errorMessage": ERROR_MSG_UNSKINNED[0], + }, + ) + contains: str = Field( + pattern=r"NORMAL:f32|POSITION:f32|TEXCOORD_0:f32", + json_schema_extra={ + "errorMessage": ERROR_MSG_UNSKINNED[1], + }, + ) + val_wrap = field_validator("*", mode="wrap")(custom_error_validator_unskinned) + + +@dataclass(config=get_model_config(title="Skinned Mesh Attributes")) +class Skinned: + """Validation schema for skinned mesh attributes.""" + + items: tuple[ + Literal["JOINTS_0:u8", "NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32", "TANGENT:f32", "WEIGHTS_0:f32"], ... + ] = Field( + ..., + json_schema_extra={ + "errorMessage": ERROR_MSG_SKINNED[0], + }, + ) + contains: str = Field( + pattern=r"JOINTS_0:u8|NORMAL:f32|POSITION:f32|TEXCOORD_0:f32|WEIGHTS_0:f32", + json_schema_extra={ + "errorMessage": ERROR_MSG_UNSKINNED[1], + }, + ) + val_wrap = field_validator("*", mode="wrap")(custom_error_validator_skinned) + + +@dataclass(config=get_model_config(title="Mesh Attributes")) +class MeshAttributes: + """Data attributes for mesh vertices. Includes the vertex position, normal, tangent, texture coordinates, influencing joints, and skin weights.""" + + MeshAttributes: Union[Unskinned, Skinned] if __name__ == "__main__": - # Example of validation in Python + import json + import logging + + from pydantic import TypeAdapter + + logging.basicConfig(level=logging.DEBUG) + # Convert model to JSON schema. + logging.debug(json.dumps(TypeAdapter(MeshAttributes).json_schema(), indent=2)) + try: - # Test example validation. - data = { - "unskinned": ["NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32"], - "skinned": ["JOINTS_0:u8", "NORMAL:f32", "POSITION:f32", "TEXCOORD_0:f32", "WEIGHTS_0:f32"], - } - MeshAttributes(**data) + model = MeshAttributes(MeshAttributes=Unskinned(items=("WRONG_ATTR", "TEXCOORD_0:f32"))) except ValidationError as error: print("\nValidation Errors:\n", error) From 8cfe26e41dbda03f085821dcca26c28e49930066 Mon Sep 17 00:00:00 2001 From: TechyDaniel <117300186+TechyDaniel@users.noreply.github.com> Date: Mon, 14 Aug 2023 17:04:15 +0200 Subject: [PATCH 32/66] feat: MVP Co-authored-by: Olaf Haag --- .../common_textures.py | 90 ++++++++++++------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/common_textures.py b/src/readyplayerme/asset_validation_schemas/common_textures.py index 9fffcc8..6c724b0 100644 --- a/src/readyplayerme/asset_validation_schemas/common_textures.py +++ b/src/readyplayerme/asset_validation_schemas/common_textures.py @@ -9,9 +9,10 @@ Co-authored-by: Ivan Sanandres Gutierrez """ from enum import Enum -from typing import Literal +from typing import Literal, TypeAlias from pydantic import ConfigDict, Field, ValidationError +from pydantic.json_schema import models_json_schema from readyplayerme.asset_validation_schemas.basemodel import BaseModel @@ -25,28 +26,31 @@ GPU_SIZE_ERROR_MSG = "Texture map exceeds maximum allowed GPU size of {valid_value} MB when fully decompressed." -class ResolutionType(str, Enum): - """Image resolution data used for textures. Power of 2 and square.""" +ResolutionType: TypeAlias = Literal[ + "1x1", + "2x2", + "4x4", + "8x8", + "16x16", + "32x32", + "64x64", + "128x128", + "256x256", + "512x512", + "1024x1024", +] - _1x1 = "1x1" - _2x2 = "2x2" - _4x4 = "4x4" - _8x8 = "8x8" - _16x16 = "16x16" - _32x32 = "32x32" - _64x64 = "64x64" - _128x128 = "128x128" - _256x256 = "256x256" - _512x512 = "512x512" - _1024x1024 = "1024x1024" - def __str__(self): - """ - Get a string representation of the ResolutionType enum value. +class TextureSlot(str, Enum): + """Available texture inputs for materials.""" + + normal_texture = "normalTexture" + base_color_texture = "baseColorTexture" + emissive_texture = "emissiveTexture" + metallic_roughness_texture = "metallicRoughnessTexture" + occlusion_texture = "occlusionTexture" - Returns: - str: The string representation of the enum value. - """ + def __str__(self): return self.value @@ -64,9 +68,7 @@ class CommonTexture(BaseModel): compression: str resolution: ResolutionType = Field( ..., - json_schema_extra={ - "errorMessages": RESOLUTION_ERROR_MSG.format(valid_value=str(list(ResolutionType)[-1]), value="${0}") - }, + description="Image resolution data used for textures. Power of 2 and square.", ) size: int = Field( ..., @@ -80,26 +82,54 @@ class CommonTexture(BaseModel): ) +class FullPBRTextureSet(CommonTexture): + """Accepting any texture type.""" + + slots: list[TextureSlot] = Field(..., min_items=1, max_items=5) + + +class NormalMapTextureSet(CommonTexture): + """Accepting only normal and occlusion textures.""" + + slots: list[Literal[TextureSlot.normal_texture, TextureSlot.occlusion_texture]] = Field( + ..., min_items=1, max_items=1 + ) + + +class NormalMap(BaseModel): + """Normal map validation schema.""" + + properties: list[NormalMapTextureSet] + + +class FullPBR(BaseModel): + """Full PBR validation schema.""" + + properties: list[FullPBRTextureSet] + + +_, top_level_schema = models_json_schema([(NormalMap, "validation"), (FullPBR, "validation")]) + # Print the generated JSON schema with indentation if __name__ == "__main__": import json import logging - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(filename=".temp/commonTexture.log", filemode="w", encoding="utf-8", level=logging.DEBUG) # Convert model to JSON schema. - logging.debug(json.dumps(CommonTexture.model_json_schema(), indent=2)) + logging.debug(json.dumps(top_level_schema, indent=2)) # Example of validation in Python try: CommonTexture( name="normalmap", uri="path/to/normal.png", - instances=0, - mime_type="image/webP", + instances=1, + mime_type="image/png", compression="default", - resolution="2048x1024", - size=3097152, - gpu_size=20291456, + resolution="1024x1024", + size=1097152, + gpu_size=1291456, ) except ValidationError as error: logging.debug("\nValidation Errors:\n %s" % error) From 68f700ca69ae5e451a7795469c6905c881fae59d Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Tue, 15 Aug 2023 17:13:57 +0200 Subject: [PATCH 33/66] chore(pre-commit): add git msg linting for conventional commits lint commit messages to follow conventional commits. --- .pre-commit-config.yaml | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55002f7..e40a1c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,3 +42,11 @@ repos: - id: mypy additional_dependencies: - "pydantic>=2.1" + + - repo: https://github.com/espressif/conventional-precommit-linter + rev: v1.2.1 + hooks: + - id: conventional-precommit-linter + stages: [commit-msg] + args: + - --types=feat,fix,docs,style,refactor,perf,test,build,ci,chore,revert diff --git a/pyproject.toml b/pyproject.toml index 7411fd4..47f06e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ post-install-commands = [ ] [tool.hatch.envs.default.scripts] -install-precommit = "pre-commit install" +install-precommit = "pre-commit install -t pre-commit -t commit-msg -t pre-push" test = "pytest {args:tests}" test-cov = "coverage run -m pytest {args:tests}" cov-report = [ From 8b9460c7c30f51f96bfe02e7bf336f68cc5857b0 Mon Sep 17 00:00:00 2001 From: TechyDaniel <117300186+TechyDaniel@users.noreply.github.com> Date: Tue, 15 Aug 2023 22:08:17 +0200 Subject: [PATCH 34/66] feat(Pydantic): :sparkles: Added Asset_Glass Model for Json Schema -MVP with some empty fields """ Glasses Validation Schema. This module defines a Pydantic model for validating an asset of type Glasses. It includes error messages for validation failures and provides a JSON schema for the model. Task: SRV-483 --- .../asset_validation_schemas/asset_glasses.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/asset_glasses.py diff --git a/src/readyplayerme/asset_validation_schemas/asset_glasses.py b/src/readyplayerme/asset_validation_schemas/asset_glasses.py new file mode 100644 index 0000000..2180a11 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/asset_glasses.py @@ -0,0 +1,82 @@ +""" +Glasses Validation Schema. + +This module defines a Pydantic model for validating an asset of type Glasses. It includes error messages +for validation failures and provides a JSON schema for the model. + +Author: Daniel-Ionut Rancea +Co-authored-by: Olaf Haag +Co-authored-by: Ivan Sanandres Gutierrez +""" + + +from pydantic import BaseModel as PydanticBaseModel +from pydantic import ConfigDict, TypeAdapter, ValidationError + +from readyplayerme.asset_validation_schemas import common_mesh, common_textures +from readyplayerme.asset_validation_schemas.basemodel import BaseModel + +# Defining constants +# TODO: Figure out how to reference other fields in error messages. Maybe use model_validator instead of field_validator + + +class Mesh(PydanticBaseModel): + """ + Mesh validation schema. + + This is a hack to create a 'properties' object. + inspect() creates a 'properties' object. Do not confuse with the 'properties' keyword. + """ + + properties: common_mesh.CommonMesh + + +class AssetGlasses(BaseModel): + """Validation schema for asset of type Glasses.""" + + model_config = ConfigDict(title="Glasses Asset") + + scenes: str + meshes: Mesh + materials: str + animations: str | None = None + textures: common_textures.FullPBR + + +# Print the generated JSON schema with indentation +if __name__ == "__main__": + import json + import logging + + logging.basicConfig(filename=".temp/commonTexture.log", filemode="w", encoding="utf-8", level=logging.DEBUG) + # Convert model to JSON schema. + logging.debug(json.dumps(TypeAdapter(AssetGlasses).json_schema(), indent=2)) + + # Example of validation in Python + try: + AssetGlasses( + scenes="glass_scene", + meshes=Mesh( + properties=common_mesh.CommonMesh( + mode=("LINES",), primitives=3, indices=("u8",), instances=2, size=int(1e7), extra_prop="no!" + ), + ), + materials="glass_material", + animations=None, + textures=common_textures.FullPBR( + properties=common_textures.CommonTexture( + name="normalmap", + uri="path/to/normal.asf", + instances=1, + mime_type="image/png", + compression="default", + resolution="1024x1024", + size=1097152, + gpu_size=1291456, + slots=["normalTexture", "baseColorTexture"], + ), + ), + ) + + except ValidationError as error: + logging.debug("\nValidation Errors:\n %s" % error) From b0198188620b8ac522f4556fed2e958f7d8ad471 Mon Sep 17 00:00:00 2001 From: TechyDaniel <117300186+TechyDaniel@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:36:55 +0200 Subject: [PATCH 35/66] fix(Schema): :bug: Fixing validation for glasses and minor changes to the naming IN the asset_glasses we decided that the minimum size should start at 16x16, and we fixed the slots issue. --- .../asset_validation_schemas/asset_glasses.py | 61 +++++++++---------- .../common_textures.py | 21 ++++--- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/asset_glasses.py b/src/readyplayerme/asset_validation_schemas/asset_glasses.py index 2180a11..fad929d 100644 --- a/src/readyplayerme/asset_validation_schemas/asset_glasses.py +++ b/src/readyplayerme/asset_validation_schemas/asset_glasses.py @@ -9,8 +9,6 @@ Co-authored-by: Ivan Sanandres Gutierrez """ - -from pydantic import BaseModel as PydanticBaseModel from pydantic import ConfigDict, TypeAdapter, ValidationError from readyplayerme.asset_validation_schemas import common_mesh, common_textures @@ -20,15 +18,10 @@ # TODO: Figure out how to reference other fields in error messages. Maybe use model_validator instead of field_validator -class Mesh(PydanticBaseModel): - """ - Mesh validation schema. - - This is a hack to create a 'properties' object. - inspect() creates a 'properties' object. Do not confuse with the 'properties' keyword. - """ +class Mesh(BaseModel): + """inspect() creates a 'properties' object. Do not confuse with the 'properties' keyword.""" - properties: common_mesh.CommonMesh + properties: list[common_mesh.CommonMesh] class AssetGlasses(BaseModel): @@ -55,27 +48,33 @@ class AssetGlasses(BaseModel): # Example of validation in Python try: AssetGlasses( - scenes="glass_scene", - meshes=Mesh( - properties=common_mesh.CommonMesh( - mode=("LINES",), primitives=3, indices=("u8",), instances=2, size=int(1e7), extra_prop="no!" - ), - ), - materials="glass_material", - animations=None, - textures=common_textures.FullPBR( - properties=common_textures.CommonTexture( - name="normalmap", - uri="path/to/normal.asf", - instances=1, - mime_type="image/png", - compression="default", - resolution="1024x1024", - size=1097152, - gpu_size=1291456, - slots=["normalTexture", "baseColorTexture"], - ), - ), + **{ + "scenes": "glass_scene", + "meshes": { + "properties": [ + common_mesh.CommonMesh( + mode=("TRIANGLES",), primitives=1, indices=("u16",), instances=1, size=500 + ) + ] + }, + "materials": "glass_material", + "animations": None, + "textures": { + "properties": [ + { + "name": ("normalmap"), + "uri": ("path/to/normal.asf"), + "instances": 1, + "mime_type": "image/png", + "compression": "default", + "resolution": "1024x1024", + "size": 1097152, + "gpu_size": 1291456, + "slots": ["normalTexture"], + } + ] + }, + } ) except ValidationError as error: diff --git a/src/readyplayerme/asset_validation_schemas/common_textures.py b/src/readyplayerme/asset_validation_schemas/common_textures.py index 6c724b0..d48b458 100644 --- a/src/readyplayerme/asset_validation_schemas/common_textures.py +++ b/src/readyplayerme/asset_validation_schemas/common_textures.py @@ -27,10 +27,6 @@ ResolutionType: TypeAlias = Literal[ - "1x1", - "2x2", - "4x4", - "8x8", "16x16", "32x32", "64x64", @@ -85,10 +81,18 @@ class CommonTexture(BaseModel): class FullPBRTextureSet(CommonTexture): """Accepting any texture type.""" - slots: list[TextureSlot] = Field(..., min_items=1, max_items=5) + slots: list[ + Literal[ + TextureSlot.normal_texture, + TextureSlot.base_color_texture, + TextureSlot.emissive_texture, + TextureSlot.metallic_roughness_texture, + TextureSlot.occlusion_texture, + ] + ] = Field(..., min_items=1, max_items=5) -class NormalMapTextureSet(CommonTexture): +class NormalOcclusionMapTextureSet(CommonTexture): """Accepting only normal and occlusion textures.""" slots: list[Literal[TextureSlot.normal_texture, TextureSlot.occlusion_texture]] = Field( @@ -99,7 +103,7 @@ class NormalMapTextureSet(CommonTexture): class NormalMap(BaseModel): """Normal map validation schema.""" - properties: list[NormalMapTextureSet] + properties: list[NormalOcclusionMapTextureSet] class FullPBR(BaseModel): @@ -121,7 +125,7 @@ class FullPBR(BaseModel): # Example of validation in Python try: - CommonTexture( + FullPBRTextureSet( name="normalmap", uri="path/to/normal.png", instances=1, @@ -130,6 +134,7 @@ class FullPBR(BaseModel): resolution="1024x1024", size=1097152, gpu_size=1291456, + slots=["metallicRoughnessTexture", "occlusionTexture", "specularMap"], ) except ValidationError as error: logging.debug("\nValidation Errors:\n %s" % error) From 1760f8aa01e8ebc9884c983c37d1bd4a88f2aa99 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 17 Aug 2023 12:31:38 +0200 Subject: [PATCH 36/66] feat(validators): add custom error handler A class that can be used for wrapping validators to raise custom errors --- .../asset_validation_schemas/validators.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/validators.py diff --git a/src/readyplayerme/asset_validation_schemas/validators.py b/src/readyplayerme/asset_validation_schemas/validators.py new file mode 100644 index 0000000..1563865 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/validators.py @@ -0,0 +1,55 @@ +"""Custom reusable validators.""" + +from collections.abc import Callable +from typing import Any, TypeAlias + +from pydantic import ( + FieldValidationInfo, + ValidationError, + ValidatorFunctionWrapHandler, +) +from pydantic_core import ErrorDetails, PydanticCustomError + +# Make a TypeAlias for the inner function signature +ErrorMsgReturnType: TypeAlias = tuple[str, str] | tuple[None, None] +GetErrorMsgFunc: TypeAlias = Callable[[str, ErrorDetails], ErrorMsgReturnType] # input: field_name, error_details + + +class CustomValidator: + """Provides a wrapper-function for validators to raise custom error types and messages. + + The purpose of this class is to reuse its custom_error_validator method to wrap validations for different fields. + To raise a custom pydantic error, the class needs to be instantiated with a function that takes the field name + and error details as input, and returns a tuple of (error_type: str, error_msg: str). + + Usage: + def error_msg_func(field_name: str, error_details: ErrorDetails) -> ErrorMsgReturnType: + if field_name == "a": + return "MyCustomErrorType", f"A custom error message. Wrong value: {error_details['value']}" + return None, None # Makes custom_error_validator raise the original error! + + class MyModel(BaseModel): + a: int + b: str + + val_wrap = field_validator("*", mode="wrap")(CustomValidator(error_msg_func).custom_error_validator) + """ + + def __init__(self, error_msg_fn: GetErrorMsgFunc): + self._get_error_msg = error_msg_fn + + def custom_error_validator( + self, value: Any, handler: ValidatorFunctionWrapHandler, info: FieldValidationInfo + ) -> Any: + """Wrap a validator function to raise a custom error type and message. + + If error type and message returned by the error message func evaluate to False, the original error is raised. + """ + try: + return handler(value) + except ValidationError as error: + for err in error.errors(): + error_type, error_msg = self._get_error_msg(info.field_name, err) + if error_type and error_msg: + raise PydanticCustomError(error_type, error_msg) from error + raise # We didn't cover this error, so raise the original error. From beb1e57b33b7e4d6d1819dafb56892ab21432e5c Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 17 Aug 2023 14:55:01 +0200 Subject: [PATCH 37/66] feat(schema): add animation sub-schema a pydantic model that asserts that there are no animations Closes SRV-531 --- .../asset_validation_schemas/animation.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/animation.py diff --git a/src/readyplayerme/asset_validation_schemas/animation.py b/src/readyplayerme/asset_validation_schemas/animation.py new file mode 100644 index 0000000..350a88a --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/animation.py @@ -0,0 +1,71 @@ +"""Sub-schemas for animation validation.""" +from pydantic import Field, ValidationError, field_validator +from pydantic.dataclasses import dataclass +from pydantic.json_schema import GenerateJsonSchema, JsonSchemaMode, JsonSchemaValue +from pydantic_core import CoreSchema, ErrorDetails + +from readyplayerme.asset_validation_schemas.basemodel import remove_keywords_from_properties +from readyplayerme.asset_validation_schemas.validators import CustomValidator, ErrorMsgReturnType + +ANIMATION_ERROR = "AnimationError" +ANIMATION_ERROR_MSG = "Animation is currently not supported." + + +class GenerateAnimationJsonSchema(GenerateJsonSchema): + """Generate the animation model JSON schema.""" + + def generate(self, schema: CoreSchema, mode: JsonSchemaMode = "validation") -> JsonSchemaValue: + _schema = super().generate(schema, mode) + remove_keywords_from_properties(_schema, ["title", "default"]) + _schema.pop("title", None) + return _schema + + +def error_msg_func(field_name: str, error_details: ErrorDetails) -> ErrorMsgReturnType: # noqa: ARG001 + """Return a custom error type and message for the animation model.""" + return ANIMATION_ERROR, ANIMATION_ERROR_MSG + + +@dataclass +class NoAnimation: + """Empty animation data.""" + + properties: list[object] = Field( + ..., + max_length=0, + description="List of animations.", + json_schema_extra={ + "errorMessage": ANIMATION_ERROR_MSG, + "$comment": ( + "gltf-transform's inspect() creates a 'properties' object. " + "Do not confuse with the 'properties' keyword." + ), + }, + ) + validation_wrapper = field_validator("*", mode="wrap")(CustomValidator(error_msg_func).custom_error_validator) + + +if __name__ == "__main__": + import json + import logging + from pathlib import Path + + from pydantic.json_schema import model_json_schema + from pydantic_core import PydanticCustomError + + Path(".temp").mkdir(exist_ok=True) + logging.basicConfig( + filename=f".temp/{Path(__file__).stem}.log", filemode="w", encoding="utf-8", level=logging.DEBUG + ) + # Demonstrate alternative way to convert a model to custom JSON schema. + top_level_schema = model_json_schema( + NoAnimation, schema_generator=GenerateAnimationJsonSchema # type: ignore[arg-type] + ) + logging.debug(json.dumps(top_level_schema, indent=2)) + + # Example of validation in Python. + try: + # Multiple checks at once. Test non-existent field as well. + NoAnimation(**{"properties": [1]}) # type: ignore[arg-type] + except (PydanticCustomError, ValidationError, TypeError) as error: + logging.error("\nValidation Errors:\n %s", error) From ab236f5d7d27f38d7798688bec315b3a889eebbf Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Fri, 18 Aug 2023 18:22:34 +0200 Subject: [PATCH 38/66] feat(io): add write json function can write json schema to a default location using the caller's file name --- .../asset_validation_schemas/schema_io.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/schema_io.py diff --git a/src/readyplayerme/asset_validation_schemas/schema_io.py b/src/readyplayerme/asset_validation_schemas/schema_io.py new file mode 100644 index 0000000..451d988 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/schema_io.py @@ -0,0 +1,22 @@ +"""Utilities to generate custom schemas and read/write JSON files.""" +import inspect +import json +from pathlib import Path +from typing import Any + + +def write_json(json_obj: Any, path: Path | None = None) -> None: + """Write JSON file. + + If no path is provided, the file will be written to a .temp folder in the current working directory. + The file will then have the name of the python file that called this function. + + :param json_obj: JSON object to write. + :param path: Path to write the JSON file to. + """ + if not path: + Path(".temp").mkdir(exist_ok=True) + caller_filename = inspect.stack()[1].filename + path = Path(".temp") / Path(caller_filename).with_suffix(".json").name + with path.open("w", encoding="UTF-8") as target: + json.dump(json_obj, target, indent=2) From 63fd72c5829cf7ea3a59e5b7ce3a31477944d294 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Fri, 18 Aug 2023 18:25:29 +0200 Subject: [PATCH 39/66] feat(io): use write_json func to write schema use the new function instead of the logger to output schema to json Task: SRV-531 --- .../asset_validation_schemas/animation.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/animation.py b/src/readyplayerme/asset_validation_schemas/animation.py index 350a88a..9e20e7a 100644 --- a/src/readyplayerme/asset_validation_schemas/animation.py +++ b/src/readyplayerme/asset_validation_schemas/animation.py @@ -46,22 +46,20 @@ class NoAnimation: if __name__ == "__main__": - import json import logging - from pathlib import Path from pydantic.json_schema import model_json_schema from pydantic_core import PydanticCustomError - Path(".temp").mkdir(exist_ok=True) - logging.basicConfig( - filename=f".temp/{Path(__file__).stem}.log", filemode="w", encoding="utf-8", level=logging.DEBUG - ) + from readyplayerme.asset_validation_schemas.schema_io import write_json + + logging.basicConfig(encoding="utf-8", level=logging.DEBUG) + # Demonstrate alternative way to convert a model to custom JSON schema. top_level_schema = model_json_schema( NoAnimation, schema_generator=GenerateAnimationJsonSchema # type: ignore[arg-type] ) - logging.debug(json.dumps(top_level_schema, indent=2)) + write_json(top_level_schema) # Example of validation in Python. try: From 6c18055ca831f4b5ae3283af6b1a6b75a64616aa Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sat, 19 Aug 2023 02:22:21 +0200 Subject: [PATCH 40/66] fix(io): use $id property as schema file name use caller's name only as a backup --- .../asset_validation_schemas/schema_io.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/schema_io.py b/src/readyplayerme/asset_validation_schemas/schema_io.py index 451d988..9d2f57d 100644 --- a/src/readyplayerme/asset_validation_schemas/schema_io.py +++ b/src/readyplayerme/asset_validation_schemas/schema_io.py @@ -16,7 +16,11 @@ def write_json(json_obj: Any, path: Path | None = None) -> None: """ if not path: Path(".temp").mkdir(exist_ok=True) - caller_filename = inspect.stack()[1].filename - path = Path(".temp") / Path(caller_filename).with_suffix(".json").name - with path.open("w", encoding="UTF-8") as target: + try: + file_name = json_obj["$id"] + except (TypeError, KeyError): + # Use caller's file name as a backup. + file_name = Path(inspect.stack()[1].filename).with_suffix(".json").name + path = Path(".temp") / file_name + with path.open("w", encoding="UTF-8") as target: # type: ignore[union-attr] # We made sure path is set. json.dump(json_obj, target, indent=2) From a8200fcb80f4441d8263d1f9b1c96caebb3ed8c1 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sat, 19 Aug 2023 02:28:37 +0200 Subject: [PATCH 41/66] feat(io): add writing common_mesh schema to json --- .../asset_validation_schemas/common_mesh.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/common_mesh.py b/src/readyplayerme/asset_validation_schemas/common_mesh.py index 434f261..f2cfca8 100644 --- a/src/readyplayerme/asset_validation_schemas/common_mesh.py +++ b/src/readyplayerme/asset_validation_schemas/common_mesh.py @@ -123,18 +123,22 @@ class CommonMesh: if __name__ == "__main__": - import json import logging from pydantic import TypeAdapter + from readyplayerme.asset_validation_schemas.schema_io import write_json + logging.basicConfig(level=logging.DEBUG) # Convert model to JSON schema. - logging.debug(json.dumps(TypeAdapter(CommonMesh).json_schema(), indent=2)) + top_level_schema = TypeAdapter(CommonMesh).json_schema() + write_json(top_level_schema) # Example of validation in Python. try: # Multiple checks at once. Test non-existent field as well. - model = CommonMesh(mode=("LINES",), primitives=3, indices=("u8",), instances=2, size=int(1e7), extra_prop="no!") + model = CommonMesh( + mode=("LINES",), primitives=3, indices=("u8",), instances=2, size=int(1e7), extra_prop="no!" # type: ignore + ) except ValidationError as error: logging.debug("\nValidation Errors:\n %s", error) From 3702b4fcdb18c9d7ff1c9f9edb93b000f6ebc60e Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sat, 19 Aug 2023 12:23:30 +0200 Subject: [PATCH 42/66] refactor(schema): move json schema generator move GenerateAnimationJsonSchema to basemodel as SchemaNoTitleAndDefault --- .../asset_validation_schemas/animation.py | 17 +++-------------- .../asset_validation_schemas/basemodel.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/animation.py b/src/readyplayerme/asset_validation_schemas/animation.py index 9e20e7a..91cfcee 100644 --- a/src/readyplayerme/asset_validation_schemas/animation.py +++ b/src/readyplayerme/asset_validation_schemas/animation.py @@ -1,26 +1,14 @@ """Sub-schemas for animation validation.""" from pydantic import Field, ValidationError, field_validator from pydantic.dataclasses import dataclass -from pydantic.json_schema import GenerateJsonSchema, JsonSchemaMode, JsonSchemaValue -from pydantic_core import CoreSchema, ErrorDetails +from pydantic_core import ErrorDetails -from readyplayerme.asset_validation_schemas.basemodel import remove_keywords_from_properties from readyplayerme.asset_validation_schemas.validators import CustomValidator, ErrorMsgReturnType ANIMATION_ERROR = "AnimationError" ANIMATION_ERROR_MSG = "Animation is currently not supported." -class GenerateAnimationJsonSchema(GenerateJsonSchema): - """Generate the animation model JSON schema.""" - - def generate(self, schema: CoreSchema, mode: JsonSchemaMode = "validation") -> JsonSchemaValue: - _schema = super().generate(schema, mode) - remove_keywords_from_properties(_schema, ["title", "default"]) - _schema.pop("title", None) - return _schema - - def error_msg_func(field_name: str, error_details: ErrorDetails) -> ErrorMsgReturnType: # noqa: ARG001 """Return a custom error type and message for the animation model.""" return ANIMATION_ERROR, ANIMATION_ERROR_MSG @@ -51,13 +39,14 @@ class NoAnimation: from pydantic.json_schema import model_json_schema from pydantic_core import PydanticCustomError + from readyplayerme.asset_validation_schemas.basemodel import SchemaNoTitleAndDefault from readyplayerme.asset_validation_schemas.schema_io import write_json logging.basicConfig(encoding="utf-8", level=logging.DEBUG) # Demonstrate alternative way to convert a model to custom JSON schema. top_level_schema = model_json_schema( - NoAnimation, schema_generator=GenerateAnimationJsonSchema # type: ignore[arg-type] + NoAnimation, schema_generator=SchemaNoTitleAndDefault # type: ignore[arg-type] ) write_json(top_level_schema) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index e9a1382..f679337 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -5,7 +5,8 @@ from pydantic import BaseModel as PydanticBaseModel from pydantic import ConfigDict from pydantic.alias_generators import to_camel -from pydantic.json_schema import GenerateJsonSchema +from pydantic.json_schema import GenerateJsonSchema, JsonSchemaMode, JsonSchemaValue +from pydantic_core import CoreSchema def add_metaschema(schema: dict[str, Any]) -> None: @@ -61,6 +62,16 @@ class BaseModel(PydanticBaseModel, abc.ABC): model_config = get_model_config(title="Base Model", defer_build=True) +class SchemaNoTitleAndDefault(GenerateJsonSchema): + """Generator for a JSON schema without titles and default values.""" + + def generate(self, schema: CoreSchema, mode: JsonSchemaMode = "validation") -> JsonSchemaValue: + _schema = super().generate(schema, mode) + remove_keywords_from_properties(_schema, ["title", "default"]) + _schema.pop("title", None) + return _schema + + if __name__ == "__main__": import json import logging From 249b669b19d1b8cc756b0a5669b0a00eaaea599a Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sat, 19 Aug 2023 12:58:25 +0200 Subject: [PATCH 43/66] style: remove ellipses from Field call --- src/readyplayerme/asset_validation_schemas/animation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/readyplayerme/asset_validation_schemas/animation.py b/src/readyplayerme/asset_validation_schemas/animation.py index 91cfcee..ee0909d 100644 --- a/src/readyplayerme/asset_validation_schemas/animation.py +++ b/src/readyplayerme/asset_validation_schemas/animation.py @@ -19,7 +19,6 @@ class NoAnimation: """Empty animation data.""" properties: list[object] = Field( - ..., max_length=0, description="List of animations.", json_schema_extra={ From 87a8dac0b95ffc244aeb7d9d2d7a3b6e645a1d86 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sun, 20 Aug 2023 13:55:47 +0200 Subject: [PATCH 44/66] refactor(schema): reusable properties comment comment for a json schema to avoid confusion between object and keyword --- src/readyplayerme/asset_validation_schemas/animation.py | 6 ++---- src/readyplayerme/asset_validation_schemas/schema_io.py | 4 ++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/animation.py b/src/readyplayerme/asset_validation_schemas/animation.py index ee0909d..c5b34c3 100644 --- a/src/readyplayerme/asset_validation_schemas/animation.py +++ b/src/readyplayerme/asset_validation_schemas/animation.py @@ -3,6 +3,7 @@ from pydantic.dataclasses import dataclass from pydantic_core import ErrorDetails +from readyplayerme.asset_validation_schemas.schema_io import properties_comment from readyplayerme.asset_validation_schemas.validators import CustomValidator, ErrorMsgReturnType ANIMATION_ERROR = "AnimationError" @@ -23,10 +24,7 @@ class NoAnimation: description="List of animations.", json_schema_extra={ "errorMessage": ANIMATION_ERROR_MSG, - "$comment": ( - "gltf-transform's inspect() creates a 'properties' object. " - "Do not confuse with the 'properties' keyword." - ), + "$comment": properties_comment, }, ) validation_wrapper = field_validator("*", mode="wrap")(CustomValidator(error_msg_func).custom_error_validator) diff --git a/src/readyplayerme/asset_validation_schemas/schema_io.py b/src/readyplayerme/asset_validation_schemas/schema_io.py index 9d2f57d..ea6a75c 100644 --- a/src/readyplayerme/asset_validation_schemas/schema_io.py +++ b/src/readyplayerme/asset_validation_schemas/schema_io.py @@ -4,6 +4,10 @@ from pathlib import Path from typing import Any +properties_comment = ( + "gltf-transform's inspect() creates a 'properties' object. Do not confuse with the 'properties' keyword." +) + def write_json(json_obj: Any, path: Path | None = None) -> None: """Write JSON file. From 18d9baa6d8761ce2f197502c8d81b74950ecfc45 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sun, 20 Aug 2023 15:54:58 +0200 Subject: [PATCH 45/66] fix(schema): use enum values instead of names populate models with the value of enums, rather than the raw enum --- src/readyplayerme/asset_validation_schemas/basemodel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index f679337..a00af60 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -45,6 +45,7 @@ def get_model_config(**kwargs: Any) -> ConfigDict: "validate_default": False, "strict": True, "populate_by_name": True, + "use_enum_values": True, "extra": "forbid", "hide_input_in_errors": True, "alias_generator": to_camel, From 953c9fd80b450e96eb3b202df3f7457f0e062de9 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sun, 20 Aug 2023 15:59:16 +0200 Subject: [PATCH 46/66] refactor: dynamic enum class func Move create_enum_class to separate module for re-use --- .../material_names.py | 29 +++++-------------- .../asset_validation_schemas/types.py | 20 +++++++++++++ 2 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 src/readyplayerme/asset_validation_schemas/types.py diff --git a/src/readyplayerme/asset_validation_schemas/material_names.py b/src/readyplayerme/asset_validation_schemas/material_names.py index e7987af..96a16b4 100644 --- a/src/readyplayerme/asset_validation_schemas/material_names.py +++ b/src/readyplayerme/asset_validation_schemas/material_names.py @@ -1,5 +1,5 @@ """Model for allowed names of materials for different asset types.""" -from collections.abc import Container, Iterable +from collections.abc import Iterable from enum import Enum from typing import Annotated, Any, Literal, cast @@ -15,6 +15,7 @@ from pydantic_core import PydanticCustomError from readyplayerme.asset_validation_schemas.basemodel import get_model_config +from readyplayerme.asset_validation_schemas.types import create_enum_class material_names = { "beard": "Wolf3D_Beard", @@ -32,24 +33,6 @@ "top": "Wolf3D_Outfit_Top", } - -# Instead of spelling out members and values for each enum, create classes dynamically. -def create_enum_class(name: str, dictionary: dict[str, str], keys: Container[str] | None = None) -> Enum: - """Create an string-enum class from a dictionary. - - If keys are provided, only the keys will be included in the enum class. - """ - - def is_key_set(item: tuple[str, str]) -> bool: - return item[0] in keys if keys else True - - if keys is None: - members = dictionary - else: - members = dict(filter(is_key_set, dictionary.items())) - return Enum(name, members, type=str) - - AllMaterialNames = create_enum_class("AllMaterialNames", material_names) OutfitMaterialNames = create_enum_class("OutfitMaterialNames", material_names, {"bottom", "footwear", "top", "body"}) @@ -73,7 +56,7 @@ def get_error_type_msg(field_name: str, value: Any) -> tuple[str, str] | tuple[N If the error type is not covered, return a None-tuple. """ match field_name: - case key if key in AllMaterialNames.__members__: # type: ignore[attr-defined] + case key if key in AllMaterialNames.__members__: return ( ERROR_CODE, ERROR_MSG.format(valid_value=getattr(AllMaterialNames, key).value, value=value) @@ -168,12 +151,14 @@ def get_field_definitions(field_input: Enum) -> Any: if __name__ == "__main__": - import json import logging + from readyplayerme.asset_validation_schemas.schema_io import write_json + logging.basicConfig(level=logging.DEBUG) # Convert model to JSON schema. - logging.debug(json.dumps(MaterialNamesModel.model_json_schema(), indent=2)) + write_json(MaterialNamesModel.model_json_schema()) + # Example of validation in Python. try: # Test example validation. diff --git a/src/readyplayerme/asset_validation_schemas/types.py b/src/readyplayerme/asset_validation_schemas/types.py new file mode 100644 index 0000000..6a6efaa --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/types.py @@ -0,0 +1,20 @@ +"""Custom types and helper functions for creating types.""" +from collections.abc import Container +from enum import Enum + + +# Instead of spelling out members and values for each enum, create classes dynamically. +def create_enum_class(name: str, dictionary: dict[str, str], keys: Container[str] | None = None) -> Enum: + """Create an string-enum class from a dictionary. + + If keys are provided, only the keys will be included in the enum class. + """ + + def is_key_set(item: tuple[str, str]) -> bool: + return item[0] in keys if keys else True + + if keys is None: + members = dictionary + else: + members = dict(filter(is_key_set, dictionary.items())) + return Enum(name, members, type=str) From b29aef1199559d8f28c8c3df369aa451575de4c4 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sun, 20 Aug 2023 16:59:03 +0200 Subject: [PATCH 47/66] refactor(schema): common textures properties dynamic enums instead of Literals, new model names, error messages Closes SRV-528 --- .../common_textures.py | 222 +++++++++++------- .../asset_validation_schemas/types.py | 8 +- 2 files changed, 148 insertions(+), 82 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/common_textures.py b/src/readyplayerme/asset_validation_schemas/common_textures.py index d48b458..00ddb71 100644 --- a/src/readyplayerme/asset_validation_schemas/common_textures.py +++ b/src/readyplayerme/asset_validation_schemas/common_textures.py @@ -1,131 +1,191 @@ -""" -Common Texture Validation Schema. - -This module defines a Pydantic model for validating common properties of texture maps. It includes error messages -for validation failures and provides a JSON schema for the model. - -Author: Daniel-Ionut Rancea -Co-authored-by: Olaf Haag -Co-authored-by: Ivan Sanandres Gutierrez -""" +"""Validation models for properties of image textures.""" +from collections.abc import Iterable from enum import Enum -from typing import Literal, TypeAlias +from typing import Annotated, Any, Literal, cast -from pydantic import ConfigDict, Field, ValidationError -from pydantic.json_schema import models_json_schema +from pydantic import ConfigDict, Field +from pydantic.alias_generators import to_snake -from readyplayerme.asset_validation_schemas.basemodel import BaseModel +from readyplayerme.asset_validation_schemas.basemodel import PydanticBaseModel +from readyplayerme.asset_validation_schemas.schema_io import properties_comment +from readyplayerme.asset_validation_schemas.types import create_enum_class, get_enum_length MAX_FILE_SIZE = 2 # in MB MAX_GPU_SIZE = 6 # in MB -# TODO: Figure out how to reference other fields in error messages. Maybe use model_validator instead of field_validator +# TODO: Figure out how to reference other fields in Python error messages. Maybe use model_validator instead of field_validator INSTANCE_ERROR_MSG = "Texture map is unused." MIMETYPE_ERROR_MSG = "Texture map must be encoded as PNG or JPEG. Found {value} instead." RESOLUTION_ERROR_MSG = "Image resolution must be a power of 2 and square. Maximum {valid_value}. Found {value} instead." FILE_SIZE_ERROR_MSG = "Texture map exceeds maximum allowed storage size of {valid_value} MB." GPU_SIZE_ERROR_MSG = "Texture map exceeds maximum allowed GPU size of {valid_value} MB when fully decompressed." +MIN_MAP_COUNT_ERROR_MSG = ( + "Too few texture maps ({value})! This Asset type must have at least one base color texture map." +) +MAX_MAP_COUNT_ERROR_MSG = "Too many texture maps ({value})! Allowed: {valid_value}." +SLOTS_ERROR_MSG = "This texture can only be used for slots: {valid_value}. Found {value} instead." +MIN_SLOTS_ERROR_MSG = ( + "Too few material slots ({value}) occupied by this texture! " + "It must be used in at least {valid_value} material slots." +) +MAX_SLOTS_ERROR_MSG = "Texture map used for too many slots ({value}). Allowed: {valid_value}." + +Resolution: Enum = create_enum_class( + "resolution", + { + f"_{res}": res + for res in [ + "16x16", + "32x32", + "64x64", + "128x128", + "256x256", + "512x512", + "1024x1024", + ] + }, +) +# Create enum classes for material slots a texture map can be used in. +texture_slots = { + to_snake(tex): tex + for tex in ["normalTexture", "baseColorTexture", "emissiveTexture", "metallicRoughnessTexture", "occlusionTexture"] +} - -ResolutionType: TypeAlias = Literal[ - "16x16", - "32x32", - "64x64", - "128x128", - "256x256", - "512x512", - "1024x1024", -] - - -class TextureSlot(str, Enum): - """Available texture inputs for materials.""" - - normal_texture = "normalTexture" - base_color_texture = "baseColorTexture" - emissive_texture = "emissiveTexture" - metallic_roughness_texture = "metallicRoughnessTexture" - occlusion_texture = "occlusionTexture" - - def __str__(self): - return self.value +TextureSlotStandard: Enum = create_enum_class("textureSlotStandard", texture_slots) +TextureSlotNormalOcclusion: Enum = create_enum_class( + "textureSlotNormalOcclusion", texture_slots, {"normal_texture", "occlusion_texture"} +) -class CommonTexture(BaseModel): +class CommonTextureProperties(PydanticBaseModel): """Validation schema for common properties of texture maps.""" - model_config = ConfigDict(title="Common Texture Map Properties") + model_config = ConfigDict(title="Common Texture Map Properties", use_enum_values=True) name: str uri: str - instances: int = Field(..., ge=1, json_schema_extra={"errorMessages": {"minimum": INSTANCE_ERROR_MSG}}) + instances: int = Field(ge=1, json_schema_extra={"errorMessage": {"minimum": INSTANCE_ERROR_MSG}}) mime_type: Literal["image/png", "image/jpeg"] = Field( - json_schema_extra={"errorMessages": MIMETYPE_ERROR_MSG.format(value="${0}")} + json_schema_extra={"errorMessage": MIMETYPE_ERROR_MSG.format(value="${0}")} ) compression: str - resolution: ResolutionType = Field( - ..., - description="Image resolution data used for textures. Power of 2 and square.", - ) + resolution: Annotated[ # type: ignore[valid-type] # Resolution is an Enum, not a regular variable. + Resolution, + Field( + description="Image resolution data used for textures. Power of 2 and square.", + json_schema_extra={ + "errorMessage": RESOLUTION_ERROR_MSG.format(valid_value=next(reversed(Resolution)).value, value="${0}") + }, + ), + ] size: int = Field( - ..., le=MAX_FILE_SIZE * 1024**2, - json_schema_extra={"errorMessages": {"maximum": FILE_SIZE_ERROR_MSG.format(valid_value=MAX_FILE_SIZE)}}, + json_schema_extra={"errorMessage": {"maximum": FILE_SIZE_ERROR_MSG.format(valid_value=MAX_FILE_SIZE)}}, ) # Convert to bytes. gpu_size: int = Field( - ..., le=MAX_GPU_SIZE * 1024**2, - json_schema_extra={"errorMessages": {"maximum": GPU_SIZE_ERROR_MSG.format(valid_value=MAX_GPU_SIZE)}}, + json_schema_extra={"errorMessage": {"maximum": GPU_SIZE_ERROR_MSG.format(valid_value=MAX_GPU_SIZE)}}, ) -class FullPBRTextureSet(CommonTexture): - """Accepting any texture type.""" - - slots: list[ - Literal[ - TextureSlot.normal_texture, - TextureSlot.base_color_texture, - TextureSlot.emissive_texture, - TextureSlot.metallic_roughness_texture, - TextureSlot.occlusion_texture, - ] - ] = Field(..., min_items=1, max_items=5) - - -class NormalOcclusionMapTextureSet(CommonTexture): - """Accepting only normal and occlusion textures.""" - - slots: list[Literal[TextureSlot.normal_texture, TextureSlot.occlusion_texture]] = Field( - ..., min_items=1, max_items=1 +def get_slot_field(slots: Enum) -> Any: + """Annotate the slots enumeration itself, which is a single item in the slots array, by a custom error message.""" + valid = ", ".join(cast(Iterable[str], slots)) # Use cast to make type checker happy. + return Annotated[ + slots, + Field(json_schema_extra={"errorMessage": SLOTS_ERROR_MSG.format(valid_value=valid, value="${0}")}), + ] + + +class TexturePropertiesStandard(CommonTextureProperties): + """Texture can occupy any material slot supported by the standard PBR shader.""" + + # Having an expression in a type hint is considered invalid, but works for now. + slots: list[get_slot_field(TextureSlotStandard)] = Field( # type: ignore[valid-type] + description="Material inputs that this texture is used for.", + min_length=1, + max_length=get_enum_length(TextureSlotStandard), + # Now we add custom errors to the array. + json_schema_extra={ + "errorMessage": { + "minItems": MIN_SLOTS_ERROR_MSG.format(valid_value=1, value="${0/length}"), + "maxItems": MAX_SLOTS_ERROR_MSG.format( + valid_value=get_enum_length(TextureSlotStandard), value="${0/length}" + ), + } + }, ) -class NormalMap(BaseModel): - """Normal map validation schema.""" +class TextureSchemaStandard(PydanticBaseModel): + """Texture schema for a single- or multi-mesh asset with standard PBR map support.""" - properties: list[NormalOcclusionMapTextureSet] + properties: list[TexturePropertiesStandard] = Field( + min_length=1, + json_schema_extra={ + "errorMessage": {"minItems": MIN_MAP_COUNT_ERROR_MSG.format(value="${0/length}")}, + "$comment": properties_comment, + }, + ) -class FullPBR(BaseModel): - """Full PBR validation schema.""" +class TexturePropertiesNormalOcclusion(CommonTextureProperties): + """Texture is only allowed to occupy normal and occlusion material slots.""" + + # Having an expression in a type hint is considered invalid, but works for now. + slots: list[get_slot_field(TextureSlotNormalOcclusion)] = Field( # type: ignore[valid-type] + description="Material input slots that this texture is used for.", + min_length=1, + max_length=get_enum_length(TextureSlotNormalOcclusion), + # Now we add custom errors to the array. + json_schema_extra={ + "errorMessage": { + "minItems": MIN_SLOTS_ERROR_MSG.format(valid_value=1, value="${0/length}"), + "maxItems": MAX_SLOTS_ERROR_MSG.format( + valid_value=get_enum_length(TextureSlotNormalOcclusion), value="${0/length}" + ), + } + }, + ) - properties: list[FullPBRTextureSet] +class TextureSchemaNormalOcclusion(PydanticBaseModel): + """Texture schema for a single-mesh asset with only normal and occlusion map support.""" + + properties: list[TexturePropertiesNormalOcclusion] = Field( + # Assets that use this texture schema (body, hair, beard) only have a single material. + # So we can't have more textures than we allow slots in a single material. + max_length=get_enum_length(TextureSlotNormalOcclusion), + json_schema_extra={ + "errorMessage": { + "maxItems": MAX_MAP_COUNT_ERROR_MSG.format( + valid_value=get_enum_length(TextureSlotNormalOcclusion), value="${0/length}" + ) + }, + "$comment": properties_comment, + }, + ) -_, top_level_schema = models_json_schema([(NormalMap, "validation"), (FullPBR, "validation")]) -# Print the generated JSON schema with indentation if __name__ == "__main__": - import json import logging - logging.basicConfig(filename=".temp/commonTexture.log", filemode="w", encoding="utf-8", level=logging.DEBUG) + from pydantic import ValidationError + from pydantic.json_schema import models_json_schema + + from readyplayerme.asset_validation_schemas.basemodel import SchemaNoTitleAndDefault + from readyplayerme.asset_validation_schemas.schema_io import write_json + + logging.basicConfig(encoding="utf-8", level=logging.DEBUG) # Convert model to JSON schema. - logging.debug(json.dumps(top_level_schema, indent=2)) + _, schema = models_json_schema( + [(TextureSchemaNormalOcclusion, "validation"), (TextureSchemaStandard, "validation")], + schema_generator=SchemaNoTitleAndDefault, + ) + write_json(schema) # Example of validation in Python try: - FullPBRTextureSet( + TexturePropertiesStandard( name="normalmap", uri="path/to/normal.png", instances=1, diff --git a/src/readyplayerme/asset_validation_schemas/types.py b/src/readyplayerme/asset_validation_schemas/types.py index 6a6efaa..35ad20c 100644 --- a/src/readyplayerme/asset_validation_schemas/types.py +++ b/src/readyplayerme/asset_validation_schemas/types.py @@ -1,6 +1,7 @@ """Custom types and helper functions for creating types.""" -from collections.abc import Container +from collections.abc import Container, Sized from enum import Enum +from typing import cast # Instead of spelling out members and values for each enum, create classes dynamically. @@ -18,3 +19,8 @@ def is_key_set(item: tuple[str, str]) -> bool: else: members = dict(filter(is_key_set, dictionary.items())) return Enum(name, members, type=str) + + +def get_enum_length(enum: Enum) -> int: + """Get the length of an enum.""" + return len(cast(Sized, enum)) From bdc476f9c3b20f700bf82ae04643e6330a6ca81e Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Sun, 20 Aug 2023 17:02:43 +0200 Subject: [PATCH 48/66] fix(schema): glasses schema wip empty objects instead of str use object as types to allow anything Closes SRV-483 --- .../asset_validation_schemas/asset_glasses.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/asset_glasses.py b/src/readyplayerme/asset_validation_schemas/asset_glasses.py index fad929d..5683b13 100644 --- a/src/readyplayerme/asset_validation_schemas/asset_glasses.py +++ b/src/readyplayerme/asset_validation_schemas/asset_glasses.py @@ -9,13 +9,12 @@ Co-authored-by: Ivan Sanandres Gutierrez """ -from pydantic import ConfigDict, TypeAdapter, ValidationError +from pydantic import ConfigDict, ValidationError -from readyplayerme.asset_validation_schemas import common_mesh, common_textures +from readyplayerme.asset_validation_schemas import common_mesh +from readyplayerme.asset_validation_schemas.animation import NoAnimation from readyplayerme.asset_validation_schemas.basemodel import BaseModel - -# Defining constants -# TODO: Figure out how to reference other fields in error messages. Maybe use model_validator instead of field_validator +from readyplayerme.asset_validation_schemas.common_textures import TextureSchemaStandard class Mesh(BaseModel): @@ -29,27 +28,28 @@ class AssetGlasses(BaseModel): model_config = ConfigDict(title="Glasses Asset") - scenes: str + scenes: object meshes: Mesh - materials: str - animations: str | None = None - textures: common_textures.FullPBR + materials: object + animations: NoAnimation + textures: TextureSchemaStandard # Print the generated JSON schema with indentation if __name__ == "__main__": - import json import logging - logging.basicConfig(filename=".temp/commonTexture.log", filemode="w", encoding="utf-8", level=logging.DEBUG) + from readyplayerme.asset_validation_schemas.schema_io import write_json + + logging.basicConfig(encoding="utf-8", level=logging.DEBUG) # Convert model to JSON schema. - logging.debug(json.dumps(TypeAdapter(AssetGlasses).json_schema(), indent=2)) + write_json(AssetGlasses.model_json_schema()) # Example of validation in Python try: AssetGlasses( **{ - "scenes": "glass_scene", + "scenes": {}, "meshes": { "properties": [ common_mesh.CommonMesh( @@ -57,8 +57,8 @@ class AssetGlasses(BaseModel): ) ] }, - "materials": "glass_material", - "animations": None, + "materials": {}, + "animations": {"properties": []}, "textures": { "properties": [ { From a0d716cc60d5e02b71b11538f2b4a6729673aea6 Mon Sep 17 00:00:00 2001 From: TechyDaniel <117300186+TechyDaniel@users.noreply.github.com> Date: Tue, 22 Aug 2023 10:50:16 +0200 Subject: [PATCH 49/66] feat: Added error message wrapper --- .../common_textures.py | 78 +++++++++++++++++-- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/common_textures.py b/src/readyplayerme/asset_validation_schemas/common_textures.py index 00ddb71..ef926c3 100644 --- a/src/readyplayerme/asset_validation_schemas/common_textures.py +++ b/src/readyplayerme/asset_validation_schemas/common_textures.py @@ -3,16 +3,19 @@ from enum import Enum from typing import Annotated, Any, Literal, cast -from pydantic import ConfigDict, Field +from pydantic import ConfigDict, Field, ValidationError, field_validator from pydantic.alias_generators import to_snake +from pydantic_core import ErrorDetails -from readyplayerme.asset_validation_schemas.basemodel import PydanticBaseModel +from readyplayerme.asset_validation_schemas.basemodel import PydanticBaseModel, get_model_config from readyplayerme.asset_validation_schemas.schema_io import properties_comment from readyplayerme.asset_validation_schemas.types import create_enum_class, get_enum_length +from readyplayerme.asset_validation_schemas.validators import CustomValidator, ErrorMsgReturnType MAX_FILE_SIZE = 2 # in MB MAX_GPU_SIZE = 6 # in MB # TODO: Figure out how to reference other fields in Python error messages. Maybe use model_validator instead of field_validator +TEXTURE_ERROR = "TextureError" INSTANCE_ERROR_MSG = "Texture map is unused." MIMETYPE_ERROR_MSG = "Texture map must be encoded as PNG or JPEG. Found {value} instead." RESOLUTION_ERROR_MSG = "Image resolution must be a power of 2 and square. Maximum {valid_value}. Found {value} instead." @@ -22,7 +25,7 @@ "Too few texture maps ({value})! This Asset type must have at least one base color texture map." ) MAX_MAP_COUNT_ERROR_MSG = "Too many texture maps ({value})! Allowed: {valid_value}." -SLOTS_ERROR_MSG = "This texture can only be used for slots: {valid_value}. Found {value} instead." +SLOTS_ERROR_MSG = "This texture can only be used for slots: {valid_value}. Found '{value}' instead." MIN_SLOTS_ERROR_MSG = ( "Too few material slots ({value}) occupied by this texture! " "It must be used in at least {valid_value} material slots." @@ -56,6 +59,51 @@ ) +def error_msg_func_common(field_name: str, error_details: ErrorDetails) -> ErrorMsgReturnType: + """Return a custom error type and message for the texture model.""" + match field_name: + case "instances": + return TEXTURE_ERROR, INSTANCE_ERROR_MSG + case "mime_type": + return TEXTURE_ERROR, MIMETYPE_ERROR_MSG.format(value=error_details.get("value")) + case "resolution": + return ( + TEXTURE_ERROR, + RESOLUTION_ERROR_MSG.format( + valid_value=next(reversed(Resolution)).value, # type: ignore[call-overload] # Enum is reversible. + value=error_details.get("value"), + ), + ) + case "size": + return TEXTURE_ERROR, FILE_SIZE_ERROR_MSG.format(valid_value=MAX_FILE_SIZE) + case "gpu_size": + return TEXTURE_ERROR, GPU_SIZE_ERROR_MSG.format(valid_value=MAX_GPU_SIZE) + + return None, None + + +def error_msg_func_slots(field_name: str, error_details: ErrorDetails) -> ErrorMsgReturnType: # noqa: ARG001 + """Return a custom error type and message for the texture model.""" + match error_details: + case {"type": "enum"}: + expected = error_details.get("ctx", {}).get("expected") + return TEXTURE_ERROR, SLOTS_ERROR_MSG.format(valid_value=expected, value=error_details.get("input")) + case {"type": "too_long"}: + return generate_length_error_msg(error_details, "max_length", MAX_SLOTS_ERROR_MSG) + case {"type": "too_short"}: + return generate_length_error_msg(error_details, "min_length", MIN_SLOTS_ERROR_MSG) + return None, None + + +def generate_length_error_msg(error_details, length_type, error_msg_template): + """Generate an error message related to the length of an input value.""" + expected = error_details.get("ctx", {}).get(length_type) + input_value = error_details.get("input") + if input_value is not None: + actual_length = len(input_value) + return TEXTURE_ERROR, error_msg_template.format(valid_value=expected, value=actual_length) + + class CommonTextureProperties(PydanticBaseModel): """Validation schema for common properties of texture maps.""" @@ -85,6 +133,9 @@ class CommonTextureProperties(PydanticBaseModel): le=MAX_GPU_SIZE * 1024**2, json_schema_extra={"errorMessage": {"maximum": GPU_SIZE_ERROR_MSG.format(valid_value=MAX_GPU_SIZE)}}, ) + validation_wrapper = field_validator("*", mode="wrap")( + CustomValidator(error_msg_func_common).custom_error_validator + ) def get_slot_field(slots: Enum) -> Any: @@ -114,11 +165,16 @@ class TexturePropertiesStandard(CommonTextureProperties): } }, ) + validation_wrapper = field_validator("*", mode="wrap")(CustomValidator(error_msg_func_slots).custom_error_validator) class TextureSchemaStandard(PydanticBaseModel): """Texture schema for a single- or multi-mesh asset with standard PBR map support.""" + model_config = get_model_config( + json_schema_extra={}, + defer_build=True, + ) properties: list[TexturePropertiesStandard] = Field( min_length=1, json_schema_extra={ @@ -146,6 +202,7 @@ class TexturePropertiesNormalOcclusion(CommonTextureProperties): } }, ) + validation_wrapper = field_validator("*", mode="wrap")(CustomValidator(error_msg_func_slots).custom_error_validator) class TextureSchemaNormalOcclusion(PydanticBaseModel): @@ -169,7 +226,6 @@ class TextureSchemaNormalOcclusion(PydanticBaseModel): if __name__ == "__main__": import logging - from pydantic import ValidationError from pydantic.json_schema import models_json_schema from readyplayerme.asset_validation_schemas.basemodel import SchemaNoTitleAndDefault @@ -185,16 +241,24 @@ class TextureSchemaNormalOcclusion(PydanticBaseModel): # Example of validation in Python try: - TexturePropertiesStandard( + TexturePropertiesNormalOcclusion( name="normalmap", uri="path/to/normal.png", instances=1, mime_type="image/png", compression="default", resolution="1024x1024", - size=1097152, + size=197152, gpu_size=1291456, - slots=["metallicRoughnessTexture", "occlusionTexture", "specularMap"], + slots=[ + "baseColor", + "test", + "asd", + "gasad", + "normalTexture", + "normalTexture", + "normalTexture", + ], ) except ValidationError as error: logging.debug("\nValidation Errors:\n %s" % error) From c01ffc09fe8a6eacfc066b7468e9429fc4b733ff Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Wed, 23 Aug 2023 19:51:02 +0200 Subject: [PATCH 50/66] chore: bump pydantic to 2.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 47f06e0..9a414af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "pydantic>=2.1" + "pydantic>=2.2" ] [project.optional-dependencies] From 45f31737703f1567a8fa8583bc255db3847574cc Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Wed, 23 Aug 2023 20:16:53 +0200 Subject: [PATCH 51/66] feat(schema): error msg func for length of input inject values from the ErrorDetails into a custom template message co-authored by: @TechyDaniel --- .../asset_validation_schemas/validators.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/readyplayerme/asset_validation_schemas/validators.py b/src/readyplayerme/asset_validation_schemas/validators.py index 1563865..fd8687b 100644 --- a/src/readyplayerme/asset_validation_schemas/validators.py +++ b/src/readyplayerme/asset_validation_schemas/validators.py @@ -53,3 +53,25 @@ def custom_error_validator( if error_type and error_msg: raise PydanticCustomError(error_type, error_msg) from error raise # We didn't cover this error, so raise the original error. + + +def get_length_error_msg(error_details: ErrorDetails, length_type: str, error_msg_template: str) -> str: + """Generate an error message related to the length type of an input value. + + :param error_details: Error details from Pydantic's ValidationError. + :param length_type: Type of length error, e.g. "min_length" or "max_length". + :param error_msg_template: Template for the error message. Must have "{valid_value}" and "{value}" placeholders. + :return: Error message. + """ + error_ctx = error_details.get("ctx", {}) + expected = error_ctx.get(length_type, f"") + input_value = error_details.get("input") + try: + # Getting length of the input is more accurate than "actual_length" in error context. + actual_length = str(len(input_value)) # type: ignore[arg-type] + except TypeError: + actual_length = str(error_ctx.get("actual_length", "")) + try: + return error_msg_template.format(valid_value=expected, value=actual_length) + except KeyError: + return error_msg_template From e80bb3d0e63abefc9e271a5cbd4075afc54654da Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 17:27:51 +0200 Subject: [PATCH 52/66] chore: bump pydantic to 2.3, mypy to 1.5.1 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9a414af..00fc4fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "pydantic>=2.2" + "pydantic>=2.3" ] [project.optional-dependencies] @@ -56,7 +56,7 @@ type = "virtual" path = ".venv" dependencies = [ "black>=23.3.0", - "mypy>=1.3.0", + "mypy>=1.5.1", "ruff>=0.0.275", "coverage[toml]>=6.5", "pytest", From 965b8adcb3eb5d7d8489b89491119731912e9244 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 17:43:14 +0200 Subject: [PATCH 53/66] refactor(schema): custom field module reusable field definitions --- .../asset_validation_schemas/fields.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 src/readyplayerme/asset_validation_schemas/fields.py diff --git a/src/readyplayerme/asset_validation_schemas/fields.py b/src/readyplayerme/asset_validation_schemas/fields.py new file mode 100644 index 0000000..7e65aa0 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/fields.py @@ -0,0 +1,34 @@ +"""Reusable field definitions for use in validation schemas.""" +from enum import Enum +from typing import Annotated, Any, Literal + +from pydantic import Field + + +def get_const_str_field_type(const: str, error_msg_template: str) -> Any: + """Return a constant-string field type with custom error messages. + + :param const: The constant string value. + :param error_msg_template: The error message template. Must contain a {valid_value} and {value} placeholder. + """ + return Annotated[ + # While this is not really a Literal, since we illegally use a variable, it works as "const" in json schema. + Literal[const], + Field(json_schema_extra={"errorMessage": error_msg_template.format(valid_value=const, value="${0}")}), + ] + + +def get_enum_field_definitions(field_input: Enum, error_msg_template: str) -> Any: + """Turn a StrEnum into field types of string-constants. + + :param field_input: The StrEnum to convert to fields. + :param error_msg_template: The error message template. Must contain a {valid_value} and {value} placeholder. + :return: A dictionary of field definitions. + """ + return { + member.name: ( # Tuple of (type definition, default value). + get_const_str_field_type(member.value, error_msg_template), + None, # Default value. + ) + for member in field_input # type: ignore[attr-defined] + } From 5c9529c10ae3c1559797083068fe8b0a4c6db553 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 17:48:55 +0200 Subject: [PATCH 54/66] refactor(schema): smaller basemodel module move extra functions and classes related to json schemas to schema_io --- .../asset_validation_schemas/basemodel.py | 38 +---- .../asset_validation_schemas/schema_io.py | 135 +++++++++++++++++- 2 files changed, 134 insertions(+), 39 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index a00af60..820a0a8 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -5,34 +5,8 @@ from pydantic import BaseModel as PydanticBaseModel from pydantic import ConfigDict from pydantic.alias_generators import to_camel -from pydantic.json_schema import GenerateJsonSchema, JsonSchemaMode, JsonSchemaValue -from pydantic_core import CoreSchema - -def add_metaschema(schema: dict[str, Any]) -> None: - """Add the JSON schema metaschema to a schema.""" - schema["$schema"] = GenerateJsonSchema.schema_dialect - - -def add_schema_id(schema: dict[str, Any], model: type["PydanticBaseModel"]) -> None: - """Add the JSON schema id based on the model name to a schema.""" - schema["$id"] = f"{(name := model.__name__)[0].lower() + name[1:]}.schema.json" - - -def remove_keywords_from_properties(schema: dict[str, Any], keywords: list[str]) -> None: - """Remove given keywords from properties of a schema.""" - for prop in schema.get("properties", {}).values(): - for kw in keywords: - prop.pop(kw, None) - - -def json_schema_extra(schema: dict[str, Any], model: type["PydanticBaseModel"]) -> None: - """Provide extra JSON schema properties.""" - # Add metaschema and id. - add_metaschema(schema) - add_schema_id(schema, model) - # Remove "title" & "default" from properties. - remove_keywords_from_properties(schema, ["title", "default"]) +from readyplayerme.asset_validation_schemas.schema_io import json_schema_extra def get_model_config(**kwargs: Any) -> ConfigDict: @@ -63,16 +37,6 @@ class BaseModel(PydanticBaseModel, abc.ABC): model_config = get_model_config(title="Base Model", defer_build=True) -class SchemaNoTitleAndDefault(GenerateJsonSchema): - """Generator for a JSON schema without titles and default values.""" - - def generate(self, schema: CoreSchema, mode: JsonSchemaMode = "validation") -> JsonSchemaValue: - _schema = super().generate(schema, mode) - remove_keywords_from_properties(_schema, ["title", "default"]) - _schema.pop("title", None) - return _schema - - if __name__ == "__main__": import json import logging diff --git a/src/readyplayerme/asset_validation_schemas/schema_io.py b/src/readyplayerme/asset_validation_schemas/schema_io.py index ea6a75c..fdbc2c3 100644 --- a/src/readyplayerme/asset_validation_schemas/schema_io.py +++ b/src/readyplayerme/asset_validation_schemas/schema_io.py @@ -1,8 +1,23 @@ """Utilities to generate custom schemas and read/write JSON files.""" import inspect import json +from collections.abc import Sequence from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any, ClassVar, cast + +from pydantic import BaseModel +from pydantic.json_schema import ( + DEFAULT_REF_TEMPLATE, + DefsRef, + GenerateJsonSchema, + JsonSchemaKeyT, + JsonSchemaMode, + JsonSchemaValue, +) +from pydantic_core import CoreSchema, core_schema + +if TYPE_CHECKING: + from pydantic._internal._dataclasses import PydanticDataclass # Didn't find another way of type hinting dataclass. properties_comment = ( "gltf-transform's inspect() creates a 'properties' object. Do not confuse with the 'properties' keyword." @@ -24,7 +39,123 @@ def write_json(json_obj: Any, path: Path | None = None) -> None: file_name = json_obj["$id"] except (TypeError, KeyError): # Use caller's file name as a backup. - file_name = Path(inspect.stack()[1].filename).with_suffix(".json").name + file_name = Path(inspect.stack()[1].filename).with_suffix(".schema.json").name path = Path(".temp") / file_name with path.open("w", encoding="UTF-8") as target: # type: ignore[union-attr] # We made sure path is set. json.dump(json_obj, target, indent=2) + + +class NoDirectInstantiation(type): + """Metaclass to prevent direct instantiation of a class.""" + + def __call__(cls, *args, **kwargs): + msg = ( + f"{cls.__name__} cannot be instantiated directly. " + "Use the with_keys class method instead to create a new class." + ) + raise TypeError(msg) + + +class GenerateJsonSchemaWithoutKeys(GenerateJsonSchema): + """Generator for a JSON schema without given keys.""" + + _keys: ClassVar[list[str]] = [] + + @classmethod + def with_keys(cls, keys: list[str]) -> type["GenerateJsonSchemaWithoutKeys"]: + """Return a new class with given keys.""" + return type(cls.__name__ + "".join(k.title() for k in keys), (cls,), {"_keys": keys}) + + def generate(self, schema: CoreSchema, mode: JsonSchemaMode = "validation") -> JsonSchemaValue: + _schema = super().generate(schema, mode) + remove_keys(_schema, self._keys) + return _schema + + def generate_definitions( + self, inputs: Sequence[tuple[JsonSchemaKeyT, JsonSchemaMode, core_schema.CoreSchema]] + ) -> tuple[dict[tuple[JsonSchemaKeyT, JsonSchemaMode], JsonSchemaValue], dict[DefsRef, JsonSchemaValue]]: + json_schemas_map, defs = super().generate_definitions(inputs) + remove_keys(cast(dict[str, Any], defs), self._keys) + return json_schemas_map, defs + + +def remove_keys(dict_: dict[str, Any], keys: list[str]) -> None: + """Remove given keywords from a dict and its nested dicts.""" + if isinstance(dict_, dict): + for keyword in keys: + dict_.pop(keyword, None) + for prop in dict_.values(): + remove_keys(prop, keys) + + +def remove_keys_from_schema(schema: dict[str, Any], keys: list[str]) -> None: + """Remove given keywords from properties of a schema.""" + for value in schema.get("$defs", {}).values(): + remove_keys(value, keys=keys) + for value in schema.get("properties", {}).values(): + remove_keys(value, keys=keys) + + +def add_schema_id(schema: dict[str, Any], model: type["BaseModel"]) -> None: + """Add the JSON schema id based on the model name to a schema.""" + schema["$id"] = f"{(name := model.__name__)[0].lower() + name[1:]}.schema.json" + + +def add_metaschema(schema: dict[str, Any], schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema) -> None: + """Add the JSON schema metaschema to a schema.""" + schema["$schema"] = schema_generator.schema_dialect + + +def json_schema_extra(schema: dict[str, Any], model: type["BaseModel"]) -> None: + """Provide extra JSON schema properties.""" + # Add metaschema and id. + add_metaschema(schema) + add_schema_id(schema, model) + # Remove "title" & "default" from properties. + remove_keys_from_schema(schema, ["title", "default"]) + + +def models_definitions_json_schema( + models: Sequence[type[BaseModel] | type["PydanticDataclass"]], + *, + by_alias: bool = True, + id_: str | None = None, + title: str | None = None, + description: str | None = None, + ref_template: str = DEFAULT_REF_TEMPLATE, + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, + mode: JsonSchemaMode = "validation", +) -> JsonSchemaValue: + """Generate a JSON Schema for multiple models. + + Clone of pydantic's models_json_schema function with the following changes: + Does only return the JSON schema definitions without the mapping. + The mode is set for all models instead of individually. + Includes the metaschema ($schema) and ID ($id) in the generated JSON Schema's top-level. + + :param models: A sequence of tuples of the form (model, mode). + :param by_alias: Whether field aliases should be used as keys in the generated JSON Schema. + :param id_: The $id of the generated JSON Schema. + :param title: The title of the generated JSON Schema. + :param description: The description of the generated JSON Schema. + :param ref_template: The reference template to use for generating JSON Schema references. + :param schema_generator: The schema generator to use for generating the JSON Schema. + + :return: A JSON schema containing all definitions along with the optional title and description keys. + """ + instance = schema_generator(by_alias=by_alias, ref_template=ref_template) + inputs = [(m, mode, m.__pydantic_core_schema__) for m in models] + _, definitions = instance.generate_definitions(inputs) + + json_schema: dict[str, Any] = {} + add_metaschema(json_schema, schema_generator) + if id_: + json_schema["$id"] = id_ + if title: + json_schema["title"] = title + if description: + json_schema["description"] = description + if definitions: + json_schema["$defs"] = definitions + + return json_schema From 74415718c3a3ccfe2184203438f88b457aebd557 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 17:52:48 +0200 Subject: [PATCH 55/66] refactor(schema): animation use moved functions that were moved earlier --- .../asset_validation_schemas/animation.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/animation.py b/src/readyplayerme/asset_validation_schemas/animation.py index c5b34c3..63ebb16 100644 --- a/src/readyplayerme/asset_validation_schemas/animation.py +++ b/src/readyplayerme/asset_validation_schemas/animation.py @@ -1,9 +1,14 @@ """Sub-schemas for animation validation.""" +from pathlib import Path +from typing import Any + from pydantic import Field, ValidationError, field_validator +from pydantic.alias_generators import to_camel from pydantic.dataclasses import dataclass from pydantic_core import ErrorDetails -from readyplayerme.asset_validation_schemas.schema_io import properties_comment +from readyplayerme.asset_validation_schemas.basemodel import get_model_config +from readyplayerme.asset_validation_schemas.schema_io import add_metaschema, properties_comment, remove_keys_from_schema from readyplayerme.asset_validation_schemas.validators import CustomValidator, ErrorMsgReturnType ANIMATION_ERROR = "AnimationError" @@ -15,7 +20,15 @@ def error_msg_func(field_name: str, error_details: ErrorDetails) -> ErrorMsgRetu return ANIMATION_ERROR, ANIMATION_ERROR_MSG -@dataclass +def json_schema_extra(schema: dict[str, Any]) -> None: + """Provide extra JSON schema properties.""" + # Add metaschema and id. + add_metaschema(schema) + schema["$id"] = f"{to_camel(Path(__file__).stem)}.schema.json" + remove_keys_from_schema(schema, ["title", "default"]) + + +@dataclass(config=get_model_config(title="Animation", json_schema_extra=json_schema_extra)) class NoAnimation: """Empty animation data.""" @@ -36,16 +49,13 @@ class NoAnimation: from pydantic.json_schema import model_json_schema from pydantic_core import PydanticCustomError - from readyplayerme.asset_validation_schemas.basemodel import SchemaNoTitleAndDefault from readyplayerme.asset_validation_schemas.schema_io import write_json logging.basicConfig(encoding="utf-8", level=logging.DEBUG) # Demonstrate alternative way to convert a model to custom JSON schema. - top_level_schema = model_json_schema( - NoAnimation, schema_generator=SchemaNoTitleAndDefault # type: ignore[arg-type] - ) - write_json(top_level_schema) + schema = model_json_schema(NoAnimation) # type: ignore[arg-type] + write_json(schema) # Example of validation in Python. try: From 746446359b9c7423e256ee69315755b0ec6ccbd9 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 17:55:44 +0200 Subject: [PATCH 56/66] refactor(schema): material names errror messages, fields --- .../material_names.py | 62 +++++-------------- 1 file changed, 15 insertions(+), 47 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/material_names.py b/src/readyplayerme/asset_validation_schemas/material_names.py index 96a16b4..dca00cc 100644 --- a/src/readyplayerme/asset_validation_schemas/material_names.py +++ b/src/readyplayerme/asset_validation_schemas/material_names.py @@ -1,21 +1,20 @@ """Model for allowed names of materials for different asset types.""" from collections.abc import Iterable -from enum import Enum -from typing import Annotated, Any, Literal, cast +from typing import Annotated, cast from pydantic import BaseModel as PydanticBaseModel from pydantic import ( Field, - FieldValidationInfo, ValidationError, - ValidatorFunctionWrapHandler, create_model, field_validator, ) -from pydantic_core import PydanticCustomError +from pydantic_core import ErrorDetails, PydanticCustomError from readyplayerme.asset_validation_schemas.basemodel import get_model_config +from readyplayerme.asset_validation_schemas.fields import get_enum_field_definitions from readyplayerme.asset_validation_schemas.types import create_enum_class +from readyplayerme.asset_validation_schemas.validators import CustomValidator, ErrorMsgReturnType material_names = { "beard": "Wolf3D_Beard", @@ -50,16 +49,19 @@ DOCS_URL = "https://docs.readyplayer.me/asset-creation-guide/validation/validation-checks/" -def get_error_type_msg(field_name: str, value: Any) -> tuple[str, str] | tuple[None, None]: +def get_error_type_msg(field_name: str, error: ErrorDetails) -> ErrorMsgReturnType: """Convert the error to a custom error type and message. If the error type is not covered, return a None-tuple. """ + error_ctx = error.get("ctx", {}) + expected = error_ctx.get("expected") + value = error.get("input") match field_name: case key if key in AllMaterialNames.__members__: return ( ERROR_CODE, - ERROR_MSG.format(valid_value=getattr(AllMaterialNames, key).value, value=value) + ERROR_MSG.format(valid_value=expected, value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) case "outfit": @@ -68,7 +70,7 @@ def get_error_type_msg(field_name: str, value: Any) -> tuple[str, str] | tuple[N ERROR_MSG_MULTI.format(valid_values=", ".join(cast(Iterable[str], OutfitMaterialNames)), value=value) + f"\n\tFor further information visit {DOCS_URL}.".expandtabs(4) * bool(DOCS_URL), ) - case key if key in ("non_customizable_avatar", "nonCustomizableAvatar"): + case key if key in ("hero_avatar", "heroAvatar"): return ( ERROR_CODE, ERROR_MSG_MULTI.format( @@ -79,41 +81,6 @@ def get_error_type_msg(field_name: str, value: Any) -> tuple[str, str] | tuple[N return None, None -def custom_error_validator(value: Any, handler: ValidatorFunctionWrapHandler, info: FieldValidationInfo) -> Any: - """Wrap the field validation function to raise custom error types. - - Return the validated value if no error occurred. - """ - try: - return handler(value) - except ValidationError as error: - for err in error.errors(): - error_type, error_msg = get_error_type_msg(info.field_name, err["input"]) - if error_type and error_msg: - raise PydanticCustomError(error_type, error_msg) from error - raise # We didn't cover this error, so raise default. - - -def get_const_str_field_type(const: str) -> Any: - """Return a constant-string field type with custom error messages.""" - return Annotated[ - # While this is not really a Literal, since we illegally use a variable, it works as "const" in json schema. - Literal[const], - Field(json_schema_extra={"errorMessage": ERROR_MSG.format(valid_value=const, value="${0}")}), - ] - - -def get_field_definitions(field_input: Enum) -> Any: - """Turn a StrEnum into field types of string-constants.""" - return { - member.name: ( # Tuple of (type definition, default value). - get_const_str_field_type(member.value), - None, # Default value. - ) - for member in field_input # type: ignore[attr-defined] - } - - # Define fields for outfit assets and hero avatar assets. outfit_field = Annotated[ OutfitMaterialNames, # type: ignore[valid-type] @@ -137,14 +104,15 @@ def get_field_definitions(field_input: Enum) -> Any: ), ] -# Wrap all field validators in a custom error validator. -wrapped_validator = field_validator("*", mode="wrap")(custom_error_validator) +# Wrapped validator to use for custom error messages. +wrapped_validator = field_validator("*", mode="wrap")(CustomValidator(get_error_type_msg).custom_error_validator) +# We don't really have the need for a model, since we can use the defined fields directly in other schemas. MaterialNamesModel: type[PydanticBaseModel] = create_model( "MaterialNames", __config__=get_model_config(title="Material Names"), - __validators__={"*": wrapped_validator}, # type: ignore[dict-item] - **get_field_definitions(AllMaterialNames), + __validators__={"*": wrapped_validator}, + **get_enum_field_definitions(AllMaterialNames, ERROR_MSG), outfit=(outfit_field, None), non_customizable_avatar=(hero_avatar_field, None), ) From 79276b83550f99f1864abdaefe415f5ba61a9ce7 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 20:31:20 +0200 Subject: [PATCH 57/66] feat(schema): custom StrEnum type Python <3.11 doesn't yet support it in the stdlib --- .../asset_validation_schemas/types.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/types.py b/src/readyplayerme/asset_validation_schemas/types.py index 35ad20c..d3c48c7 100644 --- a/src/readyplayerme/asset_validation_schemas/types.py +++ b/src/readyplayerme/asset_validation_schemas/types.py @@ -4,8 +4,21 @@ from typing import cast +class StrEnum(str, Enum): + """Enum with string values. + + This is a workaround for Python 3.10 not yet supporting StrEnum. + """ + + def __str__(self) -> str: + return self.value + + def __repr__(self) -> str: + return str.__repr__(self.value) + + # Instead of spelling out members and values for each enum, create classes dynamically. -def create_enum_class(name: str, dictionary: dict[str, str], keys: Container[str] | None = None) -> Enum: +def create_enum_class(name: str, dictionary: dict[str, str], keys: Container[str] | None = None) -> StrEnum: """Create an string-enum class from a dictionary. If keys are provided, only the keys will be included in the enum class. @@ -18,7 +31,7 @@ def is_key_set(item: tuple[str, str]) -> bool: members = dictionary else: members = dict(filter(is_key_set, dictionary.items())) - return Enum(name, members, type=str) + return StrEnum(name, members) # type: ignore[call-overload] def get_enum_length(enum: Enum) -> int: From d2b5a3d5fcf84ab73872268fb0f8352ba88c8904 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 20:32:26 +0200 Subject: [PATCH 58/66] fix(schema): disable strict mode it was blocking passing str to enum fields --- src/readyplayerme/asset_validation_schemas/basemodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index 820a0a8..516d741 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -17,7 +17,7 @@ def get_model_config(**kwargs: Any) -> ConfigDict: default_dict = { "validate_assignment": True, "validate_default": False, - "strict": True, + "strict": False, # Otherwise, enum values don't pass validation as strings. "populate_by_name": True, "use_enum_values": True, "extra": "forbid", From ac7771341a3dd96ffc835484e2f7cdf4c113a602 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 20:46:34 +0200 Subject: [PATCH 59/66] chore: bump mypy, ruff versions --- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e40a1c0..90ba05d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: # Code style and formatting - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.0.282 + rev: v0.0.285 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -37,7 +37,7 @@ repos: args: ["--verbose", "2"] # override the .docstr.yaml to see less output - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.4.1' + rev: 'v1.5.1' hooks: - id: mypy additional_dependencies: diff --git a/pyproject.toml b/pyproject.toml index 00fc4fe..7e9322c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ path = ".venv" dependencies = [ "black>=23.3.0", "mypy>=1.5.1", - "ruff>=0.0.275", + "ruff>=0.0.285", "coverage[toml]>=6.5", "pytest", ] From d015b4f1d64d1f94f63b372597745c4e6cde0ecd Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 20:52:28 +0200 Subject: [PATCH 60/66] fix(schema): common texture rename, error messages remove plural from name fix error messages make wrap validators not override base refactor for latest changes json schema generation fixed Task: SRV-528 --- .../{common_textures.py => common_texture.py} | 95 ++++++++++--------- 1 file changed, 48 insertions(+), 47 deletions(-) rename src/readyplayerme/asset_validation_schemas/{common_textures.py => common_texture.py} (78%) diff --git a/src/readyplayerme/asset_validation_schemas/common_textures.py b/src/readyplayerme/asset_validation_schemas/common_texture.py similarity index 78% rename from src/readyplayerme/asset_validation_schemas/common_textures.py rename to src/readyplayerme/asset_validation_schemas/common_texture.py index ef926c3..665d733 100644 --- a/src/readyplayerme/asset_validation_schemas/common_textures.py +++ b/src/readyplayerme/asset_validation_schemas/common_texture.py @@ -1,16 +1,15 @@ """Validation models for properties of image textures.""" from collections.abc import Iterable -from enum import Enum from typing import Annotated, Any, Literal, cast from pydantic import ConfigDict, Field, ValidationError, field_validator from pydantic.alias_generators import to_snake from pydantic_core import ErrorDetails -from readyplayerme.asset_validation_schemas.basemodel import PydanticBaseModel, get_model_config +from readyplayerme.asset_validation_schemas.basemodel import BaseModel from readyplayerme.asset_validation_schemas.schema_io import properties_comment -from readyplayerme.asset_validation_schemas.types import create_enum_class, get_enum_length -from readyplayerme.asset_validation_schemas.validators import CustomValidator, ErrorMsgReturnType +from readyplayerme.asset_validation_schemas.types import StrEnum, create_enum_class, get_enum_length +from readyplayerme.asset_validation_schemas.validators import CustomValidator, ErrorMsgReturnType, get_length_error_msg MAX_FILE_SIZE = 2 # in MB MAX_GPU_SIZE = 6 # in MB @@ -32,7 +31,7 @@ ) MAX_SLOTS_ERROR_MSG = "Texture map used for too many slots ({value}). Allowed: {valid_value}." -Resolution: Enum = create_enum_class( +Resolution: StrEnum = create_enum_class( "resolution", { f"_{res}": res @@ -53,9 +52,9 @@ for tex in ["normalTexture", "baseColorTexture", "emissiveTexture", "metallicRoughnessTexture", "occlusionTexture"] } -TextureSlotStandard: Enum = create_enum_class("textureSlotStandard", texture_slots) -TextureSlotNormalOcclusion: Enum = create_enum_class( - "textureSlotNormalOcclusion", texture_slots, {"normal_texture", "occlusion_texture"} +TextureSlotStandard: StrEnum = create_enum_class("TextureSlotStandard", texture_slots) +TextureSlotNormalOcclusion: StrEnum = create_enum_class( + "TextureSlotNormalOcclusion", texture_slots, {"normal_texture", "occlusion_texture"} ) @@ -65,13 +64,13 @@ def error_msg_func_common(field_name: str, error_details: ErrorDetails) -> Error case "instances": return TEXTURE_ERROR, INSTANCE_ERROR_MSG case "mime_type": - return TEXTURE_ERROR, MIMETYPE_ERROR_MSG.format(value=error_details.get("value")) + return TEXTURE_ERROR, MIMETYPE_ERROR_MSG.format(value=error_details.get("input")) case "resolution": return ( TEXTURE_ERROR, RESOLUTION_ERROR_MSG.format( - valid_value=next(reversed(Resolution)).value, # type: ignore[call-overload] # Enum is reversible. - value=error_details.get("value"), + valid_value=next(reversed(Resolution)).value, + value=error_details.get("input"), ), ) case "size": @@ -85,29 +84,29 @@ def error_msg_func_common(field_name: str, error_details: ErrorDetails) -> Error def error_msg_func_slots(field_name: str, error_details: ErrorDetails) -> ErrorMsgReturnType: # noqa: ARG001 """Return a custom error type and message for the texture model.""" match error_details: + # In case strict mode is on, the error type is "is_instance_of" instead of "enum". + case {"type": "is_instance_of"}: + # Identify the enum class. + try: + cls = globals()[error_details.get("ctx", {}).get("class")] # type: ignore[index] + except KeyError: + return None, None + expected = ", ".join(cast(Iterable[str], cls)) + return TEXTURE_ERROR, SLOTS_ERROR_MSG.format(valid_value=expected, value=error_details.get("input")) case {"type": "enum"}: - expected = error_details.get("ctx", {}).get("expected") + expected = error_details.get("ctx", {}).get("expected", f"") return TEXTURE_ERROR, SLOTS_ERROR_MSG.format(valid_value=expected, value=error_details.get("input")) case {"type": "too_long"}: - return generate_length_error_msg(error_details, "max_length", MAX_SLOTS_ERROR_MSG) + return TEXTURE_ERROR, get_length_error_msg(error_details, "max_length", MAX_SLOTS_ERROR_MSG) case {"type": "too_short"}: - return generate_length_error_msg(error_details, "min_length", MIN_SLOTS_ERROR_MSG) + return TEXTURE_ERROR, get_length_error_msg(error_details, "min_length", MIN_SLOTS_ERROR_MSG) return None, None -def generate_length_error_msg(error_details, length_type, error_msg_template): - """Generate an error message related to the length of an input value.""" - expected = error_details.get("ctx", {}).get(length_type) - input_value = error_details.get("input") - if input_value is not None: - actual_length = len(input_value) - return TEXTURE_ERROR, error_msg_template.format(valid_value=expected, value=actual_length) - - -class CommonTextureProperties(PydanticBaseModel): +class CommonTextureProperties(BaseModel): """Validation schema for common properties of texture maps.""" - model_config = ConfigDict(title="Common Texture Map Properties", use_enum_values=True) + model_config = ConfigDict(title="Common Texture Map Properties") name: str uri: str @@ -116,7 +115,7 @@ class CommonTextureProperties(PydanticBaseModel): json_schema_extra={"errorMessage": MIMETYPE_ERROR_MSG.format(value="${0}")} ) compression: str - resolution: Annotated[ # type: ignore[valid-type] # Resolution is an Enum, not a regular variable. + resolution: Annotated[ Resolution, Field( description="Image resolution data used for textures. Power of 2 and square.", @@ -138,7 +137,7 @@ class CommonTextureProperties(PydanticBaseModel): ) -def get_slot_field(slots: Enum) -> Any: +def get_slot_field(slots: StrEnum) -> Any: """Annotate the slots enumeration itself, which is a single item in the slots array, by a custom error message.""" valid = ", ".join(cast(Iterable[str], slots)) # Use cast to make type checker happy. return Annotated[ @@ -165,16 +164,14 @@ class TexturePropertiesStandard(CommonTextureProperties): } }, ) - validation_wrapper = field_validator("*", mode="wrap")(CustomValidator(error_msg_func_slots).custom_error_validator) + validation_wrapper_slots = field_validator("slots", mode="wrap")( + CustomValidator(error_msg_func_slots).custom_error_validator + ) -class TextureSchemaStandard(PydanticBaseModel): +class TextureSchemaStandard(BaseModel): """Texture schema for a single- or multi-mesh asset with standard PBR map support.""" - model_config = get_model_config( - json_schema_extra={}, - defer_build=True, - ) properties: list[TexturePropertiesStandard] = Field( min_length=1, json_schema_extra={ @@ -202,10 +199,12 @@ class TexturePropertiesNormalOcclusion(CommonTextureProperties): } }, ) - validation_wrapper = field_validator("*", mode="wrap")(CustomValidator(error_msg_func_slots).custom_error_validator) + validation_wrapper_slots = field_validator("slots", mode="wrap")( + CustomValidator(error_msg_func_slots).custom_error_validator + ) -class TextureSchemaNormalOcclusion(PydanticBaseModel): +class TextureSchemaNormalOcclusion(BaseModel): """Texture schema for a single-mesh asset with only normal and occlusion map support.""" properties: list[TexturePropertiesNormalOcclusion] = Field( @@ -225,17 +224,24 @@ class TextureSchemaNormalOcclusion(PydanticBaseModel): if __name__ == "__main__": import logging + from pathlib import Path - from pydantic.json_schema import models_json_schema + from pydantic.alias_generators import to_camel - from readyplayerme.asset_validation_schemas.basemodel import SchemaNoTitleAndDefault - from readyplayerme.asset_validation_schemas.schema_io import write_json + from readyplayerme.asset_validation_schemas.schema_io import ( + GenerateJsonSchemaWithoutKeys, + models_definitions_json_schema, + write_json, + ) logging.basicConfig(encoding="utf-8", level=logging.DEBUG) # Convert model to JSON schema. - _, schema = models_json_schema( - [(TextureSchemaNormalOcclusion, "validation"), (TextureSchemaStandard, "validation")], - schema_generator=SchemaNoTitleAndDefault, + schema = models_definitions_json_schema( + [TextureSchemaStandard, TextureSchemaNormalOcclusion], + schema_generator=GenerateJsonSchemaWithoutKeys.with_keys(["title", "default", "$id", "$schema"]), + id_=f"{to_camel(Path(__file__).stem)}.schema.json", + title="Common Texture Map Properties", + description="Validation schema for common properties of texture maps.", ) write_json(schema) @@ -251,13 +257,8 @@ class TextureSchemaNormalOcclusion(PydanticBaseModel): size=197152, gpu_size=1291456, slots=[ - "baseColor", + "baseColorTexture", "test", - "asd", - "gasad", - "normalTexture", - "normalTexture", - "normalTexture", ], ) except ValidationError as error: From b5583748ed57ff0cb81f958da12fb9ac68ff551b Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 20:54:58 +0200 Subject: [PATCH 61/66] fix(schema): asset glasses common_texture import need to change after common_texture renaming Task: SRV-483 --- src/readyplayerme/asset_validation_schemas/asset_glasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readyplayerme/asset_validation_schemas/asset_glasses.py b/src/readyplayerme/asset_validation_schemas/asset_glasses.py index 5683b13..65c8239 100644 --- a/src/readyplayerme/asset_validation_schemas/asset_glasses.py +++ b/src/readyplayerme/asset_validation_schemas/asset_glasses.py @@ -14,7 +14,7 @@ from readyplayerme.asset_validation_schemas import common_mesh from readyplayerme.asset_validation_schemas.animation import NoAnimation from readyplayerme.asset_validation_schemas.basemodel import BaseModel -from readyplayerme.asset_validation_schemas.common_textures import TextureSchemaStandard +from readyplayerme.asset_validation_schemas.common_texture import TextureSchemaStandard class Mesh(BaseModel): From 6283b07d9b7993187acceb6f96b97ee6eee65c8d Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Thu, 24 Aug 2023 20:59:13 +0200 Subject: [PATCH 62/66] style(schema): change type ignore issue probably introduced through mypy update --- src/readyplayerme/asset_validation_schemas/basemodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index 516d741..f31f77e 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -28,7 +28,7 @@ def get_model_config(**kwargs: Any) -> ConfigDict: "frozen": True, } updated_dict = default_dict | kwargs - return ConfigDict(**updated_dict) # type: ignore[misc] + return ConfigDict(**updated_dict) # type: ignore[typeddict-item] class BaseModel(PydanticBaseModel, abc.ABC): From 306cd806c1ce38d86e01922dd62fd2a4094e9d62 Mon Sep 17 00:00:00 2001 From: TechyDaniel <117300186+TechyDaniel@users.noreply.github.com> Date: Fri, 25 Aug 2023 14:50:23 +0200 Subject: [PATCH 63/66] feat: added empty keys to match the Json instance --- .../asset_validation_schemas/asset_glasses.py | 264 +++++++++++++++++- .../asset_validation_schemas/common_mesh.py | 4 + .../common_texture.py | 2 +- 3 files changed, 255 insertions(+), 15 deletions(-) diff --git a/src/readyplayerme/asset_validation_schemas/asset_glasses.py b/src/readyplayerme/asset_validation_schemas/asset_glasses.py index 65c8239..29744c9 100644 --- a/src/readyplayerme/asset_validation_schemas/asset_glasses.py +++ b/src/readyplayerme/asset_validation_schemas/asset_glasses.py @@ -21,6 +21,8 @@ class Mesh(BaseModel): """inspect() creates a 'properties' object. Do not confuse with the 'properties' keyword.""" properties: list[common_mesh.CommonMesh] + has_morph_targets: bool + total_triangle_count: int class AssetGlasses(BaseModel): @@ -28,6 +30,10 @@ class AssetGlasses(BaseModel): model_config = ConfigDict(title="Glasses Asset") + asset_type: str + transforms: object + joints: object + gltf_errors: object scenes: object meshes: Mesh materials: object @@ -49,31 +55,261 @@ class AssetGlasses(BaseModel): try: AssetGlasses( **{ - "scenes": {}, + "assetType": "outfit", + "scenes": { + "properties": [ + { + "name": "Validation", + "rootName": "Armature", + "bboxMin": [-0.46396, 0.00172, -0.12935], + "bboxMax": [0.46396, 1.5212, 0.19108], + } + ], + "hasDefaultScene": True, + }, "meshes": { "properties": [ - common_mesh.CommonMesh( - mode=("TRIANGLES",), primitives=1, indices=("u16",), instances=1, size=500 - ) - ] + { + "name": "Wolf3D_Body", + "mode": ["TRIANGLES"], + "primitives": 1, + "glPrimitives": 1982, + "vertices": 1266, + "indices": ["u16"], + "attributes": [ + "JOINTS_0:u8", + "NORMAL:f32", + "POSITION:f32", + "TEXCOORD_0:f32", + "WEIGHTS_0:f32", + ], + "instances": 1, + "size": 77724, + }, + { + "name": "Wolf3D_Outfit_Bottom", + "mode": ["TRIANGLES"], + "primitives": 1, + "glPrimitives": 1790, + "vertices": 1295, + "indices": ["u16"], + "attributes": [ + "JOINTS_0:u8", + "NORMAL:f32", + "POSITION:f32", + "TEXCOORD_0:f32", + "WEIGHTS_0:f32", + ], + "instances": 1, + "size": 78080, + }, + { + "name": "Wolf3D_Outfit_Footwear", + "mode": ["TRIANGLES"], + "primitives": 1, + "glPrimitives": 2000, + "vertices": 1566, + "indices": ["u16"], + "attributes": [ + "JOINTS_0:u8", + "NORMAL:f32", + "POSITION:f32", + "TEXCOORD_0:f32", + "WEIGHTS_0:f32", + ], + "instances": 1, + "size": 93432, + }, + { + "name": "Wolf3D_Outfit_Top", + "mode": ["TRIANGLES"], + "primitives": 1, + "glPrimitives": 2998, + "vertices": 1794, + "indices": ["u16"], + "attributes": [ + "JOINTS_0:u8", + "NORMAL:f32", + "POSITION:f32", + "TEXCOORD_0:f32", + "WEIGHTS_0:f32", + ], + "instances": 1, + "size": 111276, + }, + ], + "hasMorphTargets": False, + "totalTriangleCount": 8770, + }, + "materials": { + "properties": [ + { + "name": "Wolf3D_Body", + "instances": 1, + "textures": ["normalTexture"], + "alphaMode": "OPAQUE", + "doubleSided": False, + }, + { + "name": "Wolf3D_Outfit_Bottom", + "instances": 1, + "textures": ["baseColorTexture"], + "alphaMode": "OPAQUE", + "doubleSided": False, + }, + { + "name": "Wolf3D_Outfit_Footwear", + "instances": 1, + "textures": ["baseColorTexture"], + "alphaMode": "OPAQUE", + "doubleSided": False, + }, + { + "name": "Wolf3D_Outfit_Top", + "instances": 1, + "textures": ["baseColorTexture", "normalTexture"], + "alphaMode": "OPAQUE", + "doubleSided": False, + }, + ], + "drawCallCount": 4, }, - "materials": {}, - "animations": {"properties": []}, "textures": { "properties": [ { - "name": ("normalmap"), - "uri": ("path/to/normal.asf"), + "name": "Wolf3D-fullbody-f-N-1024", + "uri": "", + "slots": ["normalTexture"], "instances": 1, - "mime_type": "image/png", - "compression": "default", + "mimeType": "image/jpeg", + "compression": "", "resolution": "1024x1024", - "size": 1097152, - "gpu_size": 1291456, + "size": 450233, + "gpuSize": 5592404, + }, + { + "name": "skirt_LOGO TIALSXBFF-C", + "uri": "", + "slots": ["baseColorTexture"], + "instances": 1, + "mimeType": "image/png", + "compression": "", + "resolution": "750x750", + "size": 61815, + "gpuSize": 2998156, + }, + { + "name": "boots_LOGO TIALSXBFF-C", + "uri": "", + "slots": ["baseColorTexture"], + "instances": 1, + "mimeType": "image/jpeg", + "compression": "", + "resolution": "838x303", + "size": 24138, + "gpuSize": 1351776, + }, + { + "name": "Shirt_N.png", + "uri": "", "slots": ["normalTexture"], - } + "instances": 1, + "mimeType": "image/png", + "compression": "", + "resolution": "1024x1024", + "size": 1108322, + "gpuSize": 5592404, + }, + { + "name": "Shirt_BC-C", + "uri": "", + "slots": ["baseColorTexture"], + "instances": 1, + "mimeType": "image/jpeg", + "compression": "", + "resolution": "1024x1024", + "size": 167644, + "gpuSize": 5592404, + }, ] }, + "animations": {"properties": []}, + "transforms": { + "Wolf3D_Body": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1], + "Wolf3D_Outfit_Bottom": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1], + "Wolf3D_Outfit_Footwear": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1], + "Wolf3D_Outfit_Top": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1], + "Armature": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1], + }, + "joints": { + "Hips": "Armature", + "Spine": "Hips", + "Spine1": "Spine", + "Spine2": "Spine1", + "Neck": "Spine2", + "Head": "Neck", + "HeadTop_End": "Head", + "LeftShoulder": "Spine2", + "LeftArm": "LeftShoulder", + "LeftForeArm": "LeftArm", + "LeftHand": "LeftForeArm", + "LeftHandThumb1": "LeftHand", + "LeftHandThumb2": "LeftHandThumb1", + "LeftHandThumb3": "LeftHandThumb2", + "LeftHandThumb4": "LeftHandThumb3", + "LeftHandIndex1": "LeftHand", + "LeftHandIndex2": "LeftHandIndex1", + "LeftHandIndex3": "LeftHandIndex2", + "LeftHandIndex4": "LeftHandIndex3", + "LeftHandMiddle1": "LeftHand", + "LeftHandMiddle2": "LeftHandMiddle1", + "LeftHandMiddle3": "LeftHandMiddle2", + "LeftHandMiddle4": "LeftHandMiddle3", + "LeftHandRing1": "LeftHand", + "LeftHandRing2": "LeftHandRing1", + "LeftHandRing3": "LeftHandRing2", + "LeftHandRing4": "LeftHandRing3", + "LeftHandPinky1": "LeftHand", + "LeftHandPinky2": "LeftHandPinky1", + "LeftHandPinky3": "LeftHandPinky2", + "LeftHandPinky4": "LeftHandPinky3", + "RightShoulder": "Spine2", + "RightArm": "RightShoulder", + "RightForeArm": "RightArm", + "RightHand": "RightForeArm", + "RightHandThumb1": "RightHand", + "RightHandThumb2": "RightHandThumb1", + "RightHandThumb3": "RightHandThumb2", + "RightHandThumb4": "RightHandThumb3", + "RightHandIndex1": "RightHand", + "RightHandIndex2": "RightHandIndex1", + "RightHandIndex3": "RightHandIndex2", + "RightHandIndex4": "RightHandIndex3", + "RightHandMiddle1": "RightHand", + "RightHandMiddle2": "RightHandMiddle1", + "RightHandMiddle3": "RightHandMiddle2", + "RightHandMiddle4": "RightHandMiddle3", + "RightHandRing1": "RightHand", + "RightHandRing2": "RightHandRing1", + "RightHandRing3": "RightHandRing2", + "RightHandRing4": "RightHandRing3", + "RightHandPinky1": "RightHand", + "RightHandPinky2": "RightHandPinky1", + "RightHandPinky3": "RightHandPinky2", + "RightHandPinky4": "RightHandPinky3", + "LeftUpLeg": "Hips", + "LeftLeg": "LeftUpLeg", + "LeftFoot": "LeftLeg", + "LeftToeBase": "LeftFoot", + "LeftToe_End": "LeftToeBase", + "RightUpLeg": "Hips", + "RightLeg": "RightUpLeg", + "RightFoot": "RightLeg", + "RightToeBase": "RightFoot", + "RightToe_End": "RightToeBase", + "neutral_bone": "Armature", + }, + "gltfErrors": ["IMAGE_NPOT_DIMENSIONS"], } ) diff --git a/src/readyplayerme/asset_validation_schemas/common_mesh.py b/src/readyplayerme/asset_validation_schemas/common_mesh.py index f2cfca8..af4955d 100644 --- a/src/readyplayerme/asset_validation_schemas/common_mesh.py +++ b/src/readyplayerme/asset_validation_schemas/common_mesh.py @@ -80,6 +80,10 @@ def custom_error_validator(value: Any, handler: ValidatorFunctionWrapHandler, in class CommonMesh: """Validation schema for common properties of meshes.""" + name: str + gl_primitives: int + attributes: object + vertices: int mode: tuple[Literal[RenderingMode.TRIANGLES]] = Field( ..., description=f"The rendering mode of the mesh. Only {RenderingMode.TRIANGLES.value} are supported.", diff --git a/src/readyplayerme/asset_validation_schemas/common_texture.py b/src/readyplayerme/asset_validation_schemas/common_texture.py index 665d733..77c92d5 100644 --- a/src/readyplayerme/asset_validation_schemas/common_texture.py +++ b/src/readyplayerme/asset_validation_schemas/common_texture.py @@ -94,7 +94,7 @@ def error_msg_func_slots(field_name: str, error_details: ErrorDetails) -> ErrorM expected = ", ".join(cast(Iterable[str], cls)) return TEXTURE_ERROR, SLOTS_ERROR_MSG.format(valid_value=expected, value=error_details.get("input")) case {"type": "enum"}: - expected = error_details.get("ctx", {}).get("expected", f"") + expected = error_details.get("ctx", {}).get("expected", "") return TEXTURE_ERROR, SLOTS_ERROR_MSG.format(valid_value=expected, value=error_details.get("input")) case {"type": "too_long"}: return TEXTURE_ERROR, get_length_error_msg(error_details, "max_length", MAX_SLOTS_ERROR_MSG) From 547b91a35e580d2fe4191bb9ce5013f040c0480d Mon Sep 17 00:00:00 2001 From: TechyDaniel <117300186+TechyDaniel@users.noreply.github.com> Date: Fri, 25 Aug 2023 16:59:54 +0200 Subject: [PATCH 64/66] feat: added scenes placeholder Remove $id and $schema from subschemas --- .../asset_validation_schemas/animation.py | 15 ++---------- .../asset_validation_schemas/asset_glasses.py | 17 ++++++++++---- .../asset_validation_schemas/basemodel.py | 4 ++-- .../asset_validation_schemas/scenes.py | 23 +++++++++++++++++++ 4 files changed, 40 insertions(+), 19 deletions(-) create mode 100644 src/readyplayerme/asset_validation_schemas/scenes.py diff --git a/src/readyplayerme/asset_validation_schemas/animation.py b/src/readyplayerme/asset_validation_schemas/animation.py index 63ebb16..f89ecb5 100644 --- a/src/readyplayerme/asset_validation_schemas/animation.py +++ b/src/readyplayerme/asset_validation_schemas/animation.py @@ -1,14 +1,11 @@ """Sub-schemas for animation validation.""" -from pathlib import Path -from typing import Any from pydantic import Field, ValidationError, field_validator -from pydantic.alias_generators import to_camel from pydantic.dataclasses import dataclass from pydantic_core import ErrorDetails from readyplayerme.asset_validation_schemas.basemodel import get_model_config -from readyplayerme.asset_validation_schemas.schema_io import add_metaschema, properties_comment, remove_keys_from_schema +from readyplayerme.asset_validation_schemas.schema_io import properties_comment from readyplayerme.asset_validation_schemas.validators import CustomValidator, ErrorMsgReturnType ANIMATION_ERROR = "AnimationError" @@ -20,15 +17,7 @@ def error_msg_func(field_name: str, error_details: ErrorDetails) -> ErrorMsgRetu return ANIMATION_ERROR, ANIMATION_ERROR_MSG -def json_schema_extra(schema: dict[str, Any]) -> None: - """Provide extra JSON schema properties.""" - # Add metaschema and id. - add_metaschema(schema) - schema["$id"] = f"{to_camel(Path(__file__).stem)}.schema.json" - remove_keys_from_schema(schema, ["title", "default"]) - - -@dataclass(config=get_model_config(title="Animation", json_schema_extra=json_schema_extra)) +@dataclass(config=get_model_config(title="Animation")) class NoAnimation: """Empty animation data.""" diff --git a/src/readyplayerme/asset_validation_schemas/asset_glasses.py b/src/readyplayerme/asset_validation_schemas/asset_glasses.py index 29744c9..01803f2 100644 --- a/src/readyplayerme/asset_validation_schemas/asset_glasses.py +++ b/src/readyplayerme/asset_validation_schemas/asset_glasses.py @@ -9,12 +9,14 @@ Co-authored-by: Ivan Sanandres Gutierrez """ -from pydantic import ConfigDict, ValidationError +from pydantic import ValidationError from readyplayerme.asset_validation_schemas import common_mesh from readyplayerme.asset_validation_schemas.animation import NoAnimation -from readyplayerme.asset_validation_schemas.basemodel import BaseModel +from readyplayerme.asset_validation_schemas.basemodel import BaseModel, get_model_config from readyplayerme.asset_validation_schemas.common_texture import TextureSchemaStandard +from readyplayerme.asset_validation_schemas.scenes import SceneProperties +from readyplayerme.asset_validation_schemas.schema_io import json_schema_extra class Mesh(BaseModel): @@ -25,16 +27,23 @@ class Mesh(BaseModel): total_triangle_count: int +class Scenes(BaseModel): + """inspect() creates a 'properties' object. Do not confuse with the 'properties' keyword.""" + + properties: list[SceneProperties] + has_default_scene: bool + + class AssetGlasses(BaseModel): """Validation schema for asset of type Glasses.""" - model_config = ConfigDict(title="Glasses Asset") + model_config = get_model_config(title="Glasses Asset", json_schema_extra=json_schema_extra) asset_type: str transforms: object joints: object gltf_errors: object - scenes: object + scenes: Scenes meshes: Mesh materials: object animations: NoAnimation diff --git a/src/readyplayerme/asset_validation_schemas/basemodel.py b/src/readyplayerme/asset_validation_schemas/basemodel.py index f31f77e..aef860d 100644 --- a/src/readyplayerme/asset_validation_schemas/basemodel.py +++ b/src/readyplayerme/asset_validation_schemas/basemodel.py @@ -6,7 +6,7 @@ from pydantic import ConfigDict from pydantic.alias_generators import to_camel -from readyplayerme.asset_validation_schemas.schema_io import json_schema_extra +from readyplayerme.asset_validation_schemas.schema_io import remove_keys_from_schema def get_model_config(**kwargs: Any) -> ConfigDict: @@ -24,7 +24,7 @@ def get_model_config(**kwargs: Any) -> ConfigDict: "hide_input_in_errors": True, "alias_generator": to_camel, "str_strip_whitespace": False, - "json_schema_extra": json_schema_extra, + "json_schema_extra": lambda schema: remove_keys_from_schema(schema, ["title", "default"]), "frozen": True, } updated_dict = default_dict | kwargs diff --git a/src/readyplayerme/asset_validation_schemas/scenes.py b/src/readyplayerme/asset_validation_schemas/scenes.py new file mode 100644 index 0000000..c4d27c4 --- /dev/null +++ b/src/readyplayerme/asset_validation_schemas/scenes.py @@ -0,0 +1,23 @@ +"""Validation models for properties of Scenes.""" +from pydantic import Field + +from readyplayerme.asset_validation_schemas.basemodel import BaseModel + + +class SceneProperties(BaseModel): + """Validation schema for Scenes.""" + + name: str + root_name: str + bbox_min: tuple[float, float, float] = Field(..., min_items=3, max_items=3) + bbox_max: tuple[float, float, float] = Field(..., min_items=3, max_items=3) + + +if __name__ == "__main__": + import logging + + from readyplayerme.asset_validation_schemas.schema_io import write_json + + logging.basicConfig(encoding="utf-8", level=logging.DEBUG) + # Convert model to JSON schema. + write_json(SceneProperties.model_json_schema()) From 6bcb3fe04a3465bd282b1f17f20932f5f8fcabbe Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Mon, 28 Aug 2023 13:18:24 +0200 Subject: [PATCH 65/66] docs(contributing): update pre-commit install command when not using hatch, the pre-commit install command needs to be more --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e08cf4d..3ca6b6d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,7 +105,7 @@ Unfortunately, there are no pre-built binaries for hatch, and hatch on its own c If you decided against using hatch, we still recommend installing the pre-commit hooks. - ```pre-commit install``` + ```pre-commit install -t pre-commit -t commit-msg -t pre-push``` ### Branch Off & Make Your Changes From 7a7f1dca26c4339496d19d090b150c2559bf7bd9 Mon Sep 17 00:00:00 2001 From: Olaf Haag Date: Mon, 28 Aug 2023 13:23:12 +0200 Subject: [PATCH 66/66] docs(contributing): consistent code blocks make code blocks copyable in github UI --- CONTRIBUTING.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3ca6b6d..b37f987 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,11 +82,15 @@ Unfortunately, there are no pre-built binaries for hatch, and hatch on its own c 3. Activate the hatch environment. - ```mamba activate hatch``` + ```powershell + mamba activate hatch + ``` OR if you're using Powershell (see [issue](https://github.com/mamba-org/mamba/issues/1717)): - ```conda activate hatch``` + ```powershell + conda activate hatch + ``` 4. Prepare the environment for development. Once you setup hatch, navigate to the cloned repository, and execute: @@ -101,11 +105,15 @@ Unfortunately, there are no pre-built binaries for hatch, and hatch on its own c Alternatively, you can get the new environment path and add it to your IDE as a Python interpreter for this repository with: - ```hatch run python -c "import sys;print(sys.executable)"``` + ```powershell + hatch run python -c "import sys;print(sys.executable)" + ``` If you decided against using hatch, we still recommend installing the pre-commit hooks. - ```pre-commit install -t pre-commit -t commit-msg -t pre-push``` + ```powershell + pre-commit install -t pre-commit -t commit-msg -t pre-push + ``` ### Branch Off & Make Your Changes