diff --git a/pyairtable/models/_base.py b/pyairtable/models/_base.py index 526723a0..0db55ffb 100644 --- a/pyairtable/models/_base.py +++ b/pyairtable/models/_base.py @@ -24,7 +24,7 @@ class AirtableModel(pydantic.BaseModel): populate_by_name=True, ) - _raw: Any = pydantic.PrivateAttr() + _raw: Dict[str, Any] = pydantic.PrivateAttr() def __init__(self, **data: Any) -> None: raw = data.copy() diff --git a/pyairtable/models/comment.py b/pyairtable/models/comment.py index e9566320..8cd15238 100644 --- a/pyairtable/models/comment.py +++ b/pyairtable/models/comment.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Dict, Optional +from typing import Dict, List, Optional import pydantic @@ -62,6 +62,12 @@ class Comment( #: Users or groups that were mentioned in the text. mentioned: Dict[str, "Mentioned"] = pydantic.Field(default_factory=dict) + #: The comment ID of the parent comment, if this comment is a threaded reply. + parent_comment_id: Optional[str] = None + + #: List of reactions to this comment. + reactions: List["Reaction"] = pydantic.Field(default_factory=list) + class Mentioned(AirtableModel): """ @@ -88,4 +94,28 @@ class Mentioned(AirtableModel): email: Optional[str] = None +class Reaction(AirtableModel): + """ + A reaction to a comment. + """ + + class EmojiInfo(AirtableModel): + unicode_character: str + + class ReactingUser(AirtableModel): + user_id: str + email: Optional[str] = None + name: Optional[str] = None + + emoji_info: EmojiInfo = pydantic.Field(alias="emoji") + reacting_user: ReactingUser + + @property + def emoji(self) -> str: + """ + The emoji character used for the reaction. + """ + return chr(int(self.emoji_info.unicode_character, 16)) + + rebuild_models(vars()) diff --git a/pyairtable/models/schema.py b/pyairtable/models/schema.py index 61d02565..4ac071bc 100644 --- a/pyairtable/models/schema.py +++ b/pyairtable/models/schema.py @@ -168,6 +168,7 @@ class BaseCollaborators(_Collaborators, url="meta/bases/{base.id}"): id: str name: str + created_time: datetime permission_level: str workspace_id: str interfaces: Dict[str, "BaseCollaborators.InterfaceCollaborators"] = _FD() @@ -179,6 +180,8 @@ class InterfaceCollaborators( _Collaborators, url="meta/bases/{base.id}/interfaces/{key}", ): + id: str + name: str created_time: datetime first_publish_time: Optional[datetime] = None group_collaborators: List["GroupCollaborator"] = _FL() @@ -219,6 +222,7 @@ class Info( created_time: datetime share_id: str type: str + can_be_synced: Optional[bool] = None is_password_protected: bool block_installation_id: Optional[str] = None restricted_to_email_domains: List[str] = _FL() @@ -434,6 +438,8 @@ class EnterpriseInfo(AirtableModel): user_ids: List[str] workspace_ids: List[str] email_domains: List["EnterpriseInfo.EmailDomain"] + root_enterprise_id: str = pydantic.Field(alias="rootEnterpriseAccountId") + descendant_enterprise_ids: List[str] = _FL(alias="descendantEnterpriseAccountIds") class EmailDomain(AirtableModel): email_domain: str @@ -559,6 +565,7 @@ class UserInfo( name: str email: str state: str + is_service_account: bool is_sso_required: bool is_two_factor_auth_enabled: bool last_activity_time: Optional[datetime] = None diff --git a/pyairtable/models/webhook.py b/pyairtable/models/webhook.py index 9a93284c..e457d2c0 100644 --- a/pyairtable/models/webhook.py +++ b/pyairtable/models/webhook.py @@ -2,7 +2,7 @@ from datetime import datetime from functools import partial from hmac import HMAC -from typing import Any, Callable, Dict, Iterator, List, Optional, Union +from typing import Any, Callable, Dict, Iterator, List, Literal, Optional, Union import pydantic from typing_extensions import Self as SelfType @@ -270,13 +270,17 @@ class Filters(AirtableModel): watch_schemas_of_field_ids: List[str] = FL() class SourceOptions(AirtableModel): - form_submission: Optional["WebhookSpecification.FormSubmission"] = None + form_submission: Optional["FormSubmission"] = None + form_page_submission: Optional["FormPageSubmission"] = None - class FormSubmission(AirtableModel): - view_id: str + class FormSubmission(AirtableModel): + view_id: str + + class FormPageSubmission(AirtableModel): + page_id: str class Includes(AirtableModel): - include_cell_values_in_field_ids: List[str] = FL() + include_cell_values_in_field_ids: Union[None, List[str], Literal["all"]] = None include_previous_cell_values: bool = False include_previous_field_definitions: bool = False diff --git a/scripts/find_model_changes.py b/scripts/find_model_changes.py new file mode 100644 index 00000000..2988d7f6 --- /dev/null +++ b/scripts/find_model_changes.py @@ -0,0 +1,302 @@ +""" +Scans the API documentation on airtable.com and compares it to the models in pyAirtable. +Attempts to flag any places where the library is missing fields or has extra undocumented fields. +""" + +import importlib +import json +import re +from functools import cached_property +from operator import attrgetter +from typing import Any, Dict, Iterator, List, Type + +import requests + +from pyairtable.models._base import AirtableModel + +API_PREFIX = "https://airtable.com/developers/web/api" +API_INTRO = f"{API_PREFIX}/introduction" +INITDATA_RE = r"]*>\s*window\.initData = (\{.*\})\s*" + +SCAN_MODELS = { + "pyairtable.models.audit:AuditLogResponse": "operations:audit-log-events:response:schema", + "pyairtable.models.audit:AuditLogEvent": "operations:audit-log-events:response:schema:@events:items", + "pyairtable.models.audit:AuditLogEvent.Context": "operations:audit-log-events:response:schema:@events:items:@context", + "pyairtable.models.audit:AuditLogEvent.Origin": "operations:audit-log-events:response:schema:@events:items:@origin", + "pyairtable.models.audit:AuditLogActor": "schemas:audit-log-actor", + "pyairtable.models.audit:AuditLogActor.UserInfo": "schemas:audit-log-actor:@user", + "pyairtable.models.collaborator:Collaborator": "operations:list-comments:response:schema:@comments:items:@author", + "pyairtable.models.comment:Comment": "operations:list-comments:response:schema:@comments:items", + "pyairtable.models.comment:Reaction": "operations:list-comments:response:schema:@comments:items:@reactions:items", + "pyairtable.models.comment:Reaction.EmojiInfo": "operations:list-comments:response:schema:@comments:items:@reactions:items:@emoji", + "pyairtable.models.comment:Reaction.ReactingUser": "operations:list-comments:response:schema:@comments:items:@reactions:items:@reactingUser", + "pyairtable.models.comment:Mentioned": "schemas:user-mentioned", + "pyairtable.models.schema:BaseSchema": "operations:get-base-schema:response:schema", + "pyairtable.models.schema:TableSchema": "schemas:table-model", + "pyairtable.models.schema:Bases": "operations:list-bases:response:schema", + "pyairtable.models.schema:Bases.Info": "operations:list-bases:response:schema:@bases:items", + "pyairtable.models.schema:BaseCollaborators": "operations:get-base-collaborators:response:schema", + "pyairtable.models.schema:BaseCollaborators.IndividualCollaborators": "operations:get-base-collaborators:response:schema:@individualCollaborators", + "pyairtable.models.schema:BaseCollaborators.GroupCollaborators": "operations:get-base-collaborators:response:schema:@groupCollaborators", + "pyairtable.models.schema:BaseCollaborators.InterfaceCollaborators": "operations:get-base-collaborators:response:schema:@interfaces:additionalProperties", + "pyairtable.models.schema:BaseCollaborators.InviteLinks": "operations:get-base-collaborators:response:schema:@inviteLinks", + "pyairtable.models.schema:BaseShares": "operations:list-shares:response:schema", + "pyairtable.models.schema:BaseShares.Info": "operations:list-shares:response:schema:@shares:items", + "pyairtable.models.schema:ViewSchema": "operations:get-view-metadata:response:schema", + "pyairtable.models.schema:InviteLink": "schemas:invite-link", + "pyairtable.models.schema:WorkspaceInviteLink": "schemas:invite-link", + "pyairtable.models.schema:InterfaceInviteLink": "schemas:invite-link", + "pyairtable.models.schema:EnterpriseInfo": "operations:get-enterprise:response:schema", + "pyairtable.models.schema:EnterpriseInfo.EmailDomain": "operations:get-enterprise:response:schema:@emailDomains:items", + "pyairtable.models.schema:WorkspaceCollaborators": "operations:get-workspace-collaborators:response:schema", + "pyairtable.models.schema:WorkspaceCollaborators.Restrictions": "operations:get-workspace-collaborators:response:schema:@workspaceRestrictions", + "pyairtable.models.schema:WorkspaceCollaborators.GroupCollaborators": "operations:get-workspace-collaborators:response:schema:@groupCollaborators", + "pyairtable.models.schema:WorkspaceCollaborators.IndividualCollaborators": "operations:get-workspace-collaborators:response:schema:@individualCollaborators", + "pyairtable.models.schema:WorkspaceCollaborators.InviteLinks": "operations:get-workspace-collaborators:response:schema:@inviteLinks", + "pyairtable.models.schema:GroupCollaborator": "schemas:group-collaborator", + "pyairtable.models.schema:IndividualCollaborator": "schemas:individual-collaborator", + "pyairtable.models.schema:BaseGroupCollaborator": "schemas:base-group-collaborator", + "pyairtable.models.schema:BaseIndividualCollaborator": "schemas:base-individual-collaborator", + "pyairtable.models.schema:BaseInviteLink": "schemas:base-invite-link", + "pyairtable.models.schema:Collaborations": "schemas:collaborations", + "pyairtable.models.schema:Collaborations.BaseCollaboration": "schemas:collaborations:@baseCollaborations:items", + "pyairtable.models.schema:Collaborations.InterfaceCollaboration": "schemas:collaborations:@interfaceCollaborations:items", + "pyairtable.models.schema:Collaborations.WorkspaceCollaboration": "schemas:collaborations:@workspaceCollaborations:items", + "pyairtable.models.schema:UserInfo": "operations:get-user-by-id:response:schema", + "pyairtable.models.schema:UserGroup": "operations:get-user-group:response:schema", + "pyairtable.models.schema:UserGroup.Member": "operations:get-user-group:response:schema:@members:items", + "pyairtable.models.webhook:Webhook": "operations:list-webhooks:response:schema:@webhooks:items", + "pyairtable.models.webhook:WebhookNotificationResult": "schemas:webhooks-notification", + "pyairtable.models.webhook:WebhookError": "schemas:webhooks-notification:@error", + "pyairtable.models.webhook:WebhookPayloads": "operations:list-webhook-payloads:response:schema", + "pyairtable.models.webhook:WebhookPayload": "schemas:webhooks-payload", + "pyairtable.models.webhook:WebhookPayload.ActionMetadata": "schemas:webhooks-action", + "pyairtable.models.webhook:WebhookPayload.FieldChanged": "schemas:webhooks-table-changed:@changedFieldsById:additionalProperties", + "pyairtable.models.webhook:WebhookPayload.FieldInfo": "schemas:webhooks-table-changed:@changedFieldsById:additionalProperties:@current", + "pyairtable.models.webhook:WebhookPayload.RecordChanged": "schemas:webhooks-changed-record:additionalProperties", + "pyairtable.models.webhook:WebhookPayload.RecordCreated": "schemas:webhooks-created-record:additionalProperties", + "pyairtable.models.webhook:WebhookPayload.TableChanged": "schemas:webhooks-table-changed", + "pyairtable.models.webhook:WebhookPayload.TableChanged.ChangedMetadata": "schemas:webhooks-table-changed:@changedMetadata", + "pyairtable.models.webhook:WebhookPayload.TableInfo": "schemas:webhooks-table-changed:@changedMetadata:@current", + "pyairtable.models.webhook:WebhookPayload.TableCreated": "schemas:webhooks-table-created", + "pyairtable.models.webhook:WebhookPayload.ViewChanged": "schemas:webhooks-table-changed:@changedViewsById:additionalProperties", + "pyairtable.models.webhook:CreateWebhook": "operations:create-a-webhook:request:schema", + "pyairtable.models.webhook:CreateWebhookResponse": "operations:create-a-webhook:response:schema", + "pyairtable.models.webhook:WebhookSpecification": "operations:create-a-webhook:request:schema:@specification", + "pyairtable.models.webhook:WebhookSpecification.Options": "schemas:webhooks-specification", + "pyairtable.models.webhook:WebhookSpecification.Includes": "schemas:webhooks-specification:@includes", + "pyairtable.models.webhook:WebhookSpecification.Filters": "schemas:webhooks-specification:@filters", + "pyairtable.models.webhook:WebhookSpecification.SourceOptions": "schemas:webhooks-specification:@filters:@sourceOptions", + "pyairtable.models.webhook:WebhookSpecification.SourceOptions.FormSubmission": "schemas:webhooks-specification:@filters:@sourceOptions:@formSubmission", + "pyairtable.models.webhook:WebhookSpecification.SourceOptions.FormPageSubmission": "schemas:webhooks-specification:@filters:@sourceOptions:@formPageSubmission", +} + +IGNORED = [ + "pyairtable.models.audit.AuditLogResponse.Pagination", # pagination, not exposed + "pyairtable.models.schema.NestedId", # internal + "pyairtable.models.schema.NestedFieldId", # internal + "pyairtable.models.schema.Bases.offset", # pagination, not exposed + "pyairtable.models.schema.BaseCollaborators.collaborators", # deprecated + "pyairtable.models.schema.WorkspaceCollaborators.collaborators", # deprecated + "pyairtable.models.webhook.WebhookPayload.cursor", # pyAirtable provides this + "pyairtable.models.schema.BaseShares.Info.shareTokenPrefix", # deprecated + "pyairtable.models.webhook.WebhookPayload.CellValuesByFieldId", # undefined in schema + "pyairtable.models.webhook.WebhookNotification", # undefined in schema +] + + +def main() -> None: + initdata = get_api_data() + issues: List[str] = [] + + # Find missing/extra fields + for model_path, initdata_path in SCAN_MODELS.items(): + modname, clsname = model_path.split(":", 1) + model_module = importlib.import_module(modname) + model_cls = attrgetter(clsname)(model_module) + initdata_path = initdata_path.replace(":@", ":properties:") + issues.extend(scan_schema(model_cls, initdata.get_nested(initdata_path))) + + if not issues: + print("No missing/extra fields found in scanned classes") + else: + for issue in issues: + print(issue) + + # Find unscanned model classes + issues.clear() + modules = sorted({model_path.split(":")[0] for model_path in SCAN_MODELS}) + for modname in modules: + if not ignore_name(modname): + mod = importlib.import_module(modname) + issues.extend(scan_missing(mod, prefix=(modname + ":"))) + + if not issues: + print("No unscanned classes found in scanned modules") + else: + for issue in issues: + print(issue) + + +def ignore_name(name: str) -> bool: + if "." in name and any(ignore_name(n) for n in name.split(".")): + return True + return ( + name in IGNORED + or name.startswith("_") + or name.endswith("FieldConfig") + or name.endswith("FieldOptions") + or name.endswith("FieldSchema") + ) + + +class ApiData(Dict[str, Any]): + """ + Wrapper around ``dict`` that adds convenient behavior for reading the API definition. + """ + + def __getitem__(self, key: str) -> Any: + # handy shortcuts + if key == "operations": + return self.by_operation + if key == "schemas": + return self.by_model_name + return super().__getitem__(key) + + def get_nested(self, path: str, separator: str = ":") -> Any: + """ + Retrieves nested objects with a path-like syntax. + """ + get_from = self + traversed = [] + while separator in path: + next_key, path = path.split(separator, 1) + traversed.append(next_key) + try: + get_from = get_from[next_key] + except KeyError: + raise KeyError(*traversed) + return get_from[path] + + @cached_property + def by_operation(self) -> Dict[str, Dict[str, Any]]: + """ + Simplifies traversal of request/response information for defined web API operations, + grouping them by the operation name instead of path/method. + """ + result: Dict[str, Dict[str, Any]] = {} + paths: Dict[str, Dict[str, Any]] = self["openApi"]["paths"] + methodinfo_dicts = [ + methodinfo + for pathinfo in paths.values() + for methodinfo in pathinfo.values() + if isinstance(methodinfo, dict) + ] + for methodinfo in methodinfo_dicts: + methodname = str(methodinfo["operationId"]).lower() + r = result[methodname] = {} + try: + r["response"] = methodinfo["responses"]["200"]["content"]["application/json"] # fmt: skip + except KeyError: + pass + try: + r["request"] = methodinfo["requestBody"]["content"]["application/json"] # fmt: skip + except KeyError: + pass + + return result + + @cached_property + def by_model_name(self) -> Dict[str, Dict[str, Any]]: + """ + Simplifies traversal of schema information by preemptively collapsing + anyOf models + """ + return { + key: self.collapse_schema(self.get_model(name)) + for name in self["openApi"]["components"]["schemas"] + for key in (str(name), str(name).lower()) + } + + def get_model(self, name: str) -> Dict[str, Any]: + """ + Retrieve a model schema by name. + """ + return self.collapse_schema( + self.get_nested(f"openApi:components:schemas:{name}") + ) + + def collapse_schema(self, schema: Dict[str, Any]) -> Dict[str, Any]: + """ + Merge together properties of all entries in anyOf or allOf schemas. + This is acceptable for our use case, but a bad idea in most other cases. + """ + if set(schema) == {"$ref"}: + if (ref := schema["$ref"]).startswith("#/components/schemas/"): + return self.collapse_schema(self.get_model(ref.split("/")[-1])) + raise ValueError(f"unhandled $ref: {ref}") + + for key in ("anyOf", "allOf"): + if key not in schema: + continue + collected_properties = {} + subschema: Dict[str, Any] + for subschema in list(schema[key]): + if subschema.get("type") == "object" or "$ref" in subschema: + collected_properties.update( + self.collapse_schema(subschema).get("properties", {}) + ) + return {"properties": collected_properties} + + return schema + + +def get_api_data() -> ApiData: + """ + Retrieve API information. + """ + response = requests.get(API_INTRO) + response.raise_for_status() + match = re.search(INITDATA_RE, response.text) + if not match: + raise RuntimeError(f"could not find {INITDATA_RE!r} in {API_INTRO}") + return ApiData(json.loads(match.group(1))) + + +def scan_schema(cls: Type[AirtableModel], schema: Dict[str, Any]) -> Iterator[str]: + """ + Yield error messages for missing or undocumented fields. + """ + + name = f"{cls.__module__}.{cls.__qualname__}" + model_aliases = {f.alias for f in cls.model_fields.values() if f.alias} + api_properties = set(schema["properties"]) + missing_keys = api_properties - model_aliases + extra_keys = model_aliases - api_properties + for missing_key in missing_keys: + if not ignore_name(f"{name}.{missing_key}"): + yield f"{name} is missing field: {missing_key}" + for extra_key in extra_keys: + if not ignore_name(f"{name}.{extra_key}"): + yield (f"{name} has undocumented field: {extra_key}") + + +def scan_missing(container: Any, prefix: str) -> Iterator[str]: + """ + Yield error messages for models within the given container which were not scanned. + """ + for name, obj in vars(container).items(): + if not isinstance(obj, type) or not issubclass(obj, AirtableModel): + continue + # ignore imported models in other modules + if not prefix.startswith(obj.__module__): + continue + if ignore_name(f"{obj.__module__}.{obj.__qualname__}"): + continue + if (subpath := f"{prefix}{name}") not in SCAN_MODELS: + yield f"{subpath} was not scanned" + yield from scan_missing(obj, prefix=(subpath + ".")) + + +if __name__ == "__main__": + main() diff --git a/tests/sample_data/Comment.json b/tests/sample_data/Comment.json index b5bc4dcc..21336ae0 100644 --- a/tests/sample_data/Comment.json +++ b/tests/sample_data/Comment.json @@ -14,5 +14,15 @@ "email": "alice@example.com", "type": "user" } - } + }, + "parentCommentId": "comkNDICXNqxSDhGL", + "reactions": [ + { + "emoji": {"unicodeCharacter": "1f44d"}, + "reactingUser": { + "userId": "usr0000000reacted", + "email": "carol@example.com" + } + } + ] } diff --git a/tests/sample_data/EnterpriseInfo.json b/tests/sample_data/EnterpriseInfo.json index 02ebde50..4ff1caa8 100644 --- a/tests/sample_data/EnterpriseInfo.json +++ b/tests/sample_data/EnterpriseInfo.json @@ -11,6 +11,7 @@ "ugpR8ZT9KtIgp8Bh3" ], "id": "entUBq2RGdihxl3vU", + "rootEnterpriseAccountId": "entUBq2RGdihxl3vU", "userIds": [ "usrL2PNC5o3H4lBEi", "usrsOEchC9xuwRgKk", diff --git a/tests/sample_data/UserInfo.json b/tests/sample_data/UserInfo.json index 2097ffe3..359dee0b 100644 --- a/tests/sample_data/UserInfo.json +++ b/tests/sample_data/UserInfo.json @@ -39,6 +39,7 @@ "id": "usrL2PNC5o3H4lBEi", "invitedToAirtableByUserId": "usrsOEchC9xuwRgKk", "isManaged": true, + "isServiceAccount": false, "isSsoRequired": true, "isTwoFactorAuthEnabled": false, "lastActivityTime": "2019-01-03T12:33:12.421Z", diff --git a/tests/sample_data/Webhook.json b/tests/sample_data/Webhook.json index 9185589c..839917e2 100644 --- a/tests/sample_data/Webhook.json +++ b/tests/sample_data/Webhook.json @@ -20,7 +20,24 @@ "options": { "filters": { "dataTypes": ["tableData"], - "recordChangeScope": "tbltp8DGLhqbUmjK1" + "changeTypes": ["add", "remove", "update"], + "fromSources": ["client"], + "recordChangeScope": "tbltp8DGLhqbUmjK1", + "sourceOptions": { + "formPageSubmission": { + "pageId": "pbdLkNDICXNqxSDhG" + }, + "formSubmission": { + "viewId": "viwLkNDICXNqxSDhG" + } + }, + "watchDataInFieldIds": ["fldLkNDICXNqxSDhG"], + "watchSchemasOfFieldIds": ["fldLkNDICXNqxSDhG"] + }, + "includes": { + "includeCellValuesInFieldIds": "all", + "includePreviousCellValues": false, + "includePreviousFieldDefinitions": false } } } diff --git a/tests/test_api_base.py b/tests/test_api_base.py index 307d070c..def883c0 100644 --- a/tests/test_api_base.py +++ b/tests/test_api_base.py @@ -186,6 +186,7 @@ def test_name(api, base, requests_mock): base.urls.meta, json={ "id": base.id, + "createdTime": "2021-01-01T00:00:00.000Z", "name": "Mocked Base Name", "permissionLevel": "create", "workspaceId": "wspFake", diff --git a/tests/test_models_comment.py b/tests/test_models_comment.py index 4895ae57..0cb2487a 100644 --- a/tests/test_models_comment.py +++ b/tests/test_models_comment.py @@ -25,10 +25,12 @@ def comments_url(base, table): return f"https://api.airtable.com/v0/{base.id}/{table.name}/{RECORD_ID}/comments" -def test_parse(comment_json): - c = Comment.model_validate(comment_json) - assert isinstance(c.created_time, datetime.datetime) - assert isinstance(c.last_updated_time, datetime.datetime) +def test_parse(comment): + assert isinstance(comment.created_time, datetime.datetime) + assert isinstance(comment.last_updated_time, datetime.datetime) + assert comment.author.id == "usrLkNDICXNqxSDhG" + assert comment.mentioned["usr00000mentioned"].display_name == "Alice Doe" + assert comment.reactions[0].emoji == "👍" def test_missing_attributes(comment_json): diff --git a/tox.ini b/tox.ini index e112cc7e..b292d812 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,7 @@ basepython = py312: python3.12 py313: python3.13 deps = -r requirements-dev.txt -commands = mypy --strict pyairtable tests/test_typing.py +commands = mypy --strict pyairtable scripts tests/test_typing.py [testenv:integration] commands =