update tests

dlt-hub · May 22, 2024 · e471686 · e471686
1 parent c9544db
commit e471686
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 11 deletions.
diff --git a/dlt_init_openapi/parser/endpoints.py b/dlt_init_openapi/parser/endpoints.py
@@ -96,15 +96,15 @@ def render_pagination_args(self) -> Optional[Dict[str, Union[str, int]]]:
         if self.detected_pagination == self.detected_global_pagination:
             return None
         return self.detected_pagination.paginator_config if self.detected_pagination else None
-    
+
     @property
     def render_auto_paginator(self) -> Optional[Dict[str, Union[str, int]]]:
         """if we could not figure out the paginator, set it to auto"""
         return not (self.detected_pagination)
 
     @property
     def data_json_path(self) -> str:
-        return self.payload.json_path if self.payload else "$"
+        return self.payload.json_path if self.payload else None
 
     @property
     def transformer(self) -> Optional[TransformerSetting]:

diff --git a/dlt_init_openapi/renderer/default/templates/source.py.j2 b/dlt_init_openapi/renderer/default/templates/source.py.j2
@@ -51,7 +51,9 @@ def {{ source_name }}(
             {% endif %}
             {% endif %}
             "endpoint": {
+                {% if endpoint.data_json_path %}
                 "data_selector": "{{ endpoint.data_json_path }}",
+                {% endif %}
                 "path": "{{endpoint.path }}",
                 {% if endpoint.transformer or endpoint.unresolvable_path_param_names or endpoint.unresolvable_query_param_names %}
                 "params": {

diff --git a/tests/integration/basics/test_paginator.py b/tests/integration/basics/test_paginator.py
@@ -73,7 +73,7 @@ def test_global_paginator() -> None:
 
     # check endpoint pagination settings
     resources: Any = {entry["name"]: entry for entry in api_dict["resources"]}  # type: ignore
-    assert not resources["item_endpoint"]["endpoint"].get("paginator")
+    assert resources["item_endpoint"]["endpoint"].get("paginator") == "auto"  # nothing detected
     assert not resources["collection_1"]["endpoint"].get("paginator")
     assert not resources["collection_2"]["endpoint"].get("paginator")
 

diff --git a/tests/integration/basics/test_transformer.py b/tests/integration/basics/test_transformer.py
@@ -17,7 +17,11 @@ def test_simple_transformer(resources: Dict[str, Any]) -> None:
         "table_name": "collection",
         "primary_key": "id",
         "write_disposition": "merge",
-        "endpoint": {"data_selector": "$", "path": "/collection/"},
+        "endpoint": {
+            "data_selector": "$",
+            "path": "/collection/",
+            "paginator": "auto",
+        },
     }
     assert resources["single_collection"] == {
         "name": "single_collection",
@@ -26,6 +30,7 @@ def test_simple_transformer(resources: Dict[str, Any]) -> None:
         "write_disposition": "merge",
         "endpoint": {
             "data_selector": "$",
+            "paginator": "auto",
             "path": "/collection/{collection_id}",
             "params": {"collection_id": {"type": "resolve", "resource": "collections", "field": "id"}},
         },
@@ -47,7 +52,7 @@ def test_simple_transformer_with_deselected_parent() -> None:
         "primary_key": "id",
         "write_disposition": "merge",
         "selected": False,
-        "endpoint": {"data_selector": "$", "path": "/collection/"},
+        "endpoint": {"data_selector": "$", "path": "/collection/", "paginator": "auto"},
     }
     assert resources["single_collection"] == {
         "name": "single_collection",
@@ -58,6 +63,7 @@ def test_simple_transformer_with_deselected_parent() -> None:
             "data_selector": "$",
             "path": "/collection/{collection_id}",
             "params": {"collection_id": {"type": "resolve", "resource": "collections", "field": "id"}},
+            "paginator": "auto",
         },
     }
 
@@ -68,7 +74,7 @@ def test_match_by_path_var_only(resources: Dict[str, Any]) -> None:
         "table_name": "user",
         "primary_key": "user_id",
         "write_disposition": "merge",
-        "endpoint": {"data_selector": "$", "path": "/users/"},
+        "endpoint": {"data_selector": "$", "path": "/users/", "paginator": "auto"},
     }
     assert resources["single_user"] == {
         "name": "single_user",
@@ -77,6 +83,7 @@ def test_match_by_path_var_only(resources: Dict[str, Any]) -> None:
         "write_disposition": "merge",
         "endpoint": {
             "data_selector": "$",
+            "paginator": "auto",
             "path": "/users/{user_id}",
             "params": {"user_id": {"type": "resolve", "resource": "users", "field": "user_id"}},
         },
@@ -89,16 +96,23 @@ def test_match_singularized_path(resources: Dict[str, Any]) -> None:
         "table_name": "invoice",
         "primary_key": "invoice_id",
         "write_disposition": "merge",
-        "endpoint": {"data_selector": "$", "path": "/invoices/"},
+        "endpoint": {"data_selector": "$", "path": "/invoices/", "paginator": "auto"},
     }
     assert resources["single_invoice"] == {
         "name": "single_invoice",
         "table_name": "invoice",
         "primary_key": "invoice_id",
         "write_disposition": "merge",
         "endpoint": {
+            "paginator": "auto",
             "data_selector": "$",
             "path": "/invoice/{invoice_id}",
-            "params": {"invoice_id": {"type": "resolve", "resource": "invoices", "field": "invoice_id"}},
+            "params": {
+                "invoice_id": {
+                    "type": "resolve",
+                    "resource": "invoices",
+                    "field": "invoice_id",
+                }
+            },
         },
     }
diff --git a/tests/integration/extracted/test_dotastats_cases.py b/tests/integration/extracted/test_dotastats_cases.py
@@ -21,6 +21,5 @@ def test_simple_dotastats_load() -> None:
         "table_name": "team",
         "endpoint": {
             "path": "/teams",
-            "data_selector": "$",
         },
     }
diff --git a/tests/integration/extracted/test_pokemon_cases.py b/tests/integration/extracted/test_pokemon_cases.py
@@ -11,7 +11,7 @@ def test_simple_poke_load() -> None:
     assert source["resources"][0] == {
         "name": "pokemon",
         "table_name": "pokemon",
-        "endpoint": {"path": "/api/v2/pokemon/", "data_selector": "$"},
+        "endpoint": {"path": "/api/v2/pokemon/", "paginator": "auto"},
     }
 
 
@@ -72,6 +72,7 @@ def test_simple_child_table_poke_load() -> None:
         "primary_key": "id",
         "write_disposition": "merge",
         "endpoint": {
+            "paginator": "auto",
             "path": "/api/v2/pokemon/{name}/",
             "data_selector": "$",
             "params": {

diff --git a/tests/integration/extracted/test_quotesapi_cases.py b/tests/integration/extracted/test_quotesapi_cases.py
@@ -23,7 +23,6 @@ def test_simple_quotesapi_load() -> None:
         "name": "list",
         "table_name": "list",
         "endpoint": {
-            "data_selector": "$",
             "path": "/quote/list",
         },
     }