Add SDK pytests

Restructure pytest files
MeltanoLabs · Oct 26, 2023 · f5149db · f5149db
1 parent a99fe7f
commit f5149db
Show file tree

Hide file tree

Showing 3 changed files with 154 additions and 53 deletions.
diff --git a/target_postgres/tests/core.py b/target_postgres/tests/core.py
@@ -0,0 +1,59 @@
+""" Config and base values for target-postgres testing """
+# flake8: noqa
+import sqlalchemy
+
+from target_postgres.target import TargetPostgres
+
+
+def postgres_config():
+    return {
+        "dialect+driver": "postgresql+psycopg2",
+        "host": "localhost",
+        "user": "postgres",
+        "password": "postgres",
+        "database": "postgres",
+        "port": 5432,
+        "ssl_enable": True,
+        "ssl_client_certificate_enable": True,
+        "ssl_mode": "verify-full",
+        "ssl_certificate_authority": "./ssl/root.crt",
+        "ssl_client_certificate": "./ssl/cert.crt",
+        "ssl_client_private_key": "./ssl/pkey.key",
+        "ssl_storage_directory": ".secrets",
+        "add_record_metadata": True,
+        "hard_delete": False,
+        "default_target_schema": "melty",
+    }
+
+
+def postgres_config_no_ssl():
+    return {
+        "dialect+driver": "postgresql+psycopg2",
+        "host": "localhost",
+        "user": "postgres",
+        "password": "postgres",
+        "database": "postgres",
+        "port": 5433,
+        "add_record_metadata": True,
+        "hard_delete": False,
+        "default_target_schema": "melty",
+    }
+
+
+def postgres_config_ssh_tunnel():
+    return {
+        "sqlalchemy_url": "postgresql://postgres:[email protected]:5432/main",
+        "ssh_tunnel": {
+            "enable": True,
+            "host": "127.0.0.1",
+            "port": 2223,
+            "username": "melty",
+            "private_key": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn\nNhAAAAAwEAAQAAAYEAvIGU0pRpThhIcaSPrg2+v7cXl+QcG0icb45hfD44yrCoXkpJp7nh\nHv0ObZL2Y1cG7eeayYF4AqD3kwQ7W89GN6YO9b/mkJgawk0/YLUyojTS9dbcTbdkfPzyUa\nvTMDjly+PIjfiWOEnUgPf1y3xONLkJU0ILyTmgTzSIMNdKngtdCGfytBCuNiPKU8hEdEVt\n82ebqgtLoSYn9cUcVVz6LewzUh8+YtoPb8Z/BIVEzU37HiE9MOYIBXjo1AEJSnOCkjwlVl\nPzLhcXKTPht0iwv/KnZNNg0LDmnU/z0n+nPq/EMflum8jRYbgp0C5hksPdc8e0eEKd9gak\nt7B0ta3Mjt5b8HPQdBGZI/QFufEnSOxfJmoK4Bvjy/oUwE0hGU6po5g+4T2j6Bqqm2I+yV\nEbkP/UiuD/kEiT0C3yCV547gIDjN2ME9tGJDkd023BFvqn3stFVVZ5WsisRKGc+lvTfqeA\nJyKFaVt5a23y68ztjEMVrMLksRuEF8gG5kV7EGyjAAAFiCzGBRksxgUZAAAAB3NzaC1yc2\nEAAAGBALyBlNKUaU4YSHGkj64Nvr+3F5fkHBtInG+OYXw+OMqwqF5KSae54R79Dm2S9mNX\nBu3nmsmBeAKg95MEO1vPRjemDvW/5pCYGsJNP2C1MqI00vXW3E23ZHz88lGr0zA45cvjyI\n34ljhJ1ID39ct8TjS5CVNCC8k5oE80iDDXSp4LXQhn8rQQrjYjylPIRHRFbfNnm6oLS6Em\nJ/XFHFVc+i3sM1IfPmLaD2/GfwSFRM1N+x4hPTDmCAV46NQBCUpzgpI8JVZT8y4XFykz4b\ndIsL/yp2TTYNCw5p1P89J/pz6vxDH5bpvI0WG4KdAuYZLD3XPHtHhCnfYGpLewdLWtzI7e\nW/Bz0HQRmSP0BbnxJ0jsXyZqCuAb48v6FMBNIRlOqaOYPuE9o+gaqptiPslRG5D/1Irg/5\nBIk9At8gleeO4CA4zdjBPbRiQ5HdNtwRb6p97LRVVWeVrIrEShnPpb036ngCcihWlbeWtt\n8uvM7YxDFazC5LEbhBfIBuZFexBsowAAAAMBAAEAAAGAflHjdb2oV4HkQetBsSRa18QM1m\ncxAoOE+SiTYRudGQ6KtSzY8MGZ/xca7QiXfXhbF1+llTTiQ/i0Dtu+H0blyfLIgZwIGIsl\nG2GCf/7MoG//kmhaFuY3O56Rj3MyQVVPgHLy+VhE6hFniske+C4jhicc/aL7nOu15n3Qad\nJLmV8KB9EIjevDoloXgk9ot/WyuXKLmMaa9rFIA+UDmJyGtfFbbsOrHbj8sS11/oSD14RT\nLBygEb2EUI52j2LmY/LEvUL+59oCuJ6Y/h+pMdFeuHJzGjrVb573KnGwejzY24HHzzebrC\nQ+9NyVCTyizPHNu9w52/GPEZQFQBi7o9cDMd3ITZEPIaIvDHsUwPXaHUBHy/XHQTs8pDqk\nzCMcAs5zdzao2I0LQ+ZFYyvl1rue82ITjDISX1WK6nFYLBVXugi0rLGEdH6P+Psfl3uCIf\naW7c12/BpZz2Pql5AuO1wsu4rmz2th68vaC/0IDqWekIbW9qihFbqnhfAxRsIURjpBAAAA\nwDhIQPsj9T9Vud3Z/TZjiAKCPbg3zi082u1GMMxXnNQtKO3J35wU7VUcAxAzosWr+emMqS\nU0qW+a5RXr3sqUOqH85b5+Xw0yv2sTr2pL0ALFW7Tq1mesCc3K0So3Yo30pWRIOxYM9ihm\nE4ci/3mN5kcKWwvLLomFPRU9u0XtIGKnF/cNByTuz9fceR6Pi6mQXZawv+OOMiBeu0gbyp\nF1uVe8PCshzCrWTE3UjRpQxy9gizvSbGZyGQi1Lm42JXKG3wAAAMEA4r4CLM1xsyxBBMld\nrxiTqy6bfrZjKkT5MPjBjp+57i5kW9NVqGCnIy/m98pLTuKjTCDmUuWQXS+oqhHw5vq/wj\nRvQYqkJDz1UGmC1lD2qyqERjOiWa8/iy4dXSLeHCT70+/xR2dBb0z8cT++yZEqLdEZSnHG\nyRaZMHot1OohVDqJS8nEbxOzgPGdopRMiX6ws/p5/k9YAGkHx0hszA8cn/Tk2/mdS5lugw\nY7mdXzfcKvxkgoFrG7XowqRVrozcvDAAAAwQDU1ITasquNLaQhKNqiHx/N7bvKVO33icAx\nNdShqJEWx/g9idvQ25sA1Ubc1a+Ot5Lgfrs2OBKe+LgSmPAZOjv4ShqBHtsSh3am8/K1xR\ngQKgojLL4FhtgxtwoZrVvovZHGV3g2A28BRGbKIGVGPsOszJALU7jlLlcTHlB7SCQBI8FQ\nvTi2UEsfTmA22NnuVPITeqbmAQQXkSZcZbpbvdc0vQzp/3iOb/OCrIMET3HqVEMyQVsVs6\nxa9026AMTGLaEAAAATcm9vdEBvcGVuc3NoLXNlcnZlcg==\n-----END OPENSSH PRIVATE KEY-----",  # noqa: E501
+        },
+    }
+
+
+def create_engine(target_postgres: TargetPostgres) -> sqlalchemy.engine.Engine:
+    return TargetPostgres.default_sink_class.connector_class(
+        config=target_postgres.config
+    )._engine
diff --git a/target_postgres/tests/test_sdk.py b/target_postgres/tests/test_sdk.py
@@ -0,0 +1,77 @@
+""" SDK tests for target postgres """
+# flake8: noqa
+import pytest
+from singer_sdk.testing import get_target_test_class
+from singer_sdk.testing.suites import TestSuite
+from singer_sdk.testing.target_tests import (
+    TargetArrayData,
+    TargetCamelcaseComplexSchema,
+    TargetCamelcaseTest,
+    TargetCliPrintsTest,
+    TargetDuplicateRecords,
+    TargetEncodedStringData,
+    TargetInvalidSchemaTest,
+    TargetMultipleStateMessages,
+    TargetNoPrimaryKeys,
+    TargetOptionalAttributes,
+    TargetRecordBeforeSchemaTest,
+    TargetRecordMissingKeyProperty,
+    TargetRecordMissingOptionalFields,
+    TargetRecordMissingRequiredProperty,
+    TargetSchemaNoProperties,
+    TargetSchemaUpdates,
+    TargetSpecialCharsInAttributes,
+)
+
+from target_postgres.target import TargetPostgres
+
+from .core import create_engine, postgres_config
+
+target_tests = TestSuite(
+    kind="target",
+    tests=[
+        TargetArrayData,
+        TargetCamelcaseComplexSchema,
+        TargetCamelcaseTest,
+        TargetCliPrintsTest,
+        TargetDuplicateRecords,
+        # Postgres doesn't support NULL characters in strings
+        # TargetEncodedStringData,
+        TargetInvalidSchemaTest,
+        # This tap only outputs one state message at the end of execution, fails assertion.
+        # Separate custom test in test_target_postgres.py
+        # TargetMultipleStateMessages,
+        TargetNoPrimaryKeys,
+        TargetOptionalAttributes,
+        TargetRecordBeforeSchemaTest,
+        TargetRecordMissingKeyProperty,
+        # Fails, but gives appropriate error message
+        # TargetRecordMissingRequiredProperty,
+        TargetSchemaNoProperties,
+        TargetSchemaUpdates,
+        TargetSpecialCharsInAttributes,
+        TargetRecordMissingOptionalFields,
+    ],
+)
+
+
+class BasePostgresSDKTests:
+    """Base class for Postgres SDK tests."""
+
+    @pytest.fixture()
+    def connection(self, runner):
+        engine = create_engine(runner)
+        return engine.connect()
+
+
+SDKTests = get_target_test_class(
+    target_class=TargetPostgres,
+    config=postgres_config(),
+    custom_suites=[target_tests],
+    suite_config=None,
+    include_target_tests=False,
+)
+
+
+class TestTargetPostgres(BasePostgresSDKTests, SDKTests):
+    """SDK tests"""
diff --git a/...et_postgres/tests/test_standard_target.py → ...et_postgres/tests/test_target_postgres.py b/...et_postgres/tests/test_standard_target.py → ...et_postgres/tests/test_target_postgres.py
@@ -1,16 +1,15 @@
-""" Attempt at making some standard Target Tests. """
+""" Postgres target tests """
 # flake8: noqa
 import copy
 import io
-import uuid
 from contextlib import redirect_stdout
 from pathlib import Path
 
 import jsonschema
 import pytest
 import sqlalchemy
 from singer_sdk.exceptions import MissingKeyPropertiesError
-from singer_sdk.testing import sync_end_to_end
+from singer_sdk.testing import get_target_test_class, sync_end_to_end
 from sqlalchemy.dialects.postgresql import ARRAY
 from sqlalchemy.types import TEXT, TIMESTAMP
 
@@ -21,68 +20,34 @@
     SampleTapCountries,
 )
 
+from .core import (
+    create_engine,
+    postgres_config,
+    postgres_config_no_ssl,
+    postgres_config_ssh_tunnel,
+)
 
-@pytest.fixture(scope="session")
-def postgres_config():
-    return {
-        "dialect+driver": "postgresql+psycopg2",
-        "host": "localhost",
-        "user": "postgres",
-        "password": "postgres",
-        "database": "postgres",
-        "port": 5432,
-        "ssl_enable": True,
-        "ssl_client_certificate_enable": True,
-        "ssl_mode": "verify-full",
-        "ssl_certificate_authority": "./ssl/root.crt",
-        "ssl_client_certificate": "./ssl/cert.crt",
-        "ssl_client_private_key": "./ssl/pkey.key",
-        "add_record_metadata": True,
-        "hard_delete": False,
-        "default_target_schema": "melty",
-    }
 
+@pytest.fixture(scope="session", name="postgres_config")
+def postgres_config_fixture():
+    return postgres_config()
 
-@pytest.fixture(scope="session")
-def postgres_config_no_ssl():
-    return {
-        "dialect+driver": "postgresql+psycopg2",
-        "host": "localhost",
-        "user": "postgres",
-        "password": "postgres",
-        "database": "postgres",
-        "port": 5433,
-        "add_record_metadata": True,
-        "hard_delete": False,
-        "default_target_schema": "melty",
-    }
 
+@pytest.fixture(scope="session", name="postgres_config_no_ssl")
+def postgres_config_no_ssl_fixture():
+    return postgres_config_no_ssl()
 
-@pytest.fixture(scope="session")
-def postgres_config_ssh_tunnel():
-    return {
-        "sqlalchemy_url": "postgresql://postgres:[email protected]:5432/main",
-        "ssh_tunnel": {
-            "enable": True,
-            "host": "127.0.0.1",
-            "port": 2223,
-            "username": "melty",
-            "private_key": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn\nNhAAAAAwEAAQAAAYEAvIGU0pRpThhIcaSPrg2+v7cXl+QcG0icb45hfD44yrCoXkpJp7nh\nHv0ObZL2Y1cG7eeayYF4AqD3kwQ7W89GN6YO9b/mkJgawk0/YLUyojTS9dbcTbdkfPzyUa\nvTMDjly+PIjfiWOEnUgPf1y3xONLkJU0ILyTmgTzSIMNdKngtdCGfytBCuNiPKU8hEdEVt\n82ebqgtLoSYn9cUcVVz6LewzUh8+YtoPb8Z/BIVEzU37HiE9MOYIBXjo1AEJSnOCkjwlVl\nPzLhcXKTPht0iwv/KnZNNg0LDmnU/z0n+nPq/EMflum8jRYbgp0C5hksPdc8e0eEKd9gak\nt7B0ta3Mjt5b8HPQdBGZI/QFufEnSOxfJmoK4Bvjy/oUwE0hGU6po5g+4T2j6Bqqm2I+yV\nEbkP/UiuD/kEiT0C3yCV547gIDjN2ME9tGJDkd023BFvqn3stFVVZ5WsisRKGc+lvTfqeA\nJyKFaVt5a23y68ztjEMVrMLksRuEF8gG5kV7EGyjAAAFiCzGBRksxgUZAAAAB3NzaC1yc2\nEAAAGBALyBlNKUaU4YSHGkj64Nvr+3F5fkHBtInG+OYXw+OMqwqF5KSae54R79Dm2S9mNX\nBu3nmsmBeAKg95MEO1vPRjemDvW/5pCYGsJNP2C1MqI00vXW3E23ZHz88lGr0zA45cvjyI\n34ljhJ1ID39ct8TjS5CVNCC8k5oE80iDDXSp4LXQhn8rQQrjYjylPIRHRFbfNnm6oLS6Em\nJ/XFHFVc+i3sM1IfPmLaD2/GfwSFRM1N+x4hPTDmCAV46NQBCUpzgpI8JVZT8y4XFykz4b\ndIsL/yp2TTYNCw5p1P89J/pz6vxDH5bpvI0WG4KdAuYZLD3XPHtHhCnfYGpLewdLWtzI7e\nW/Bz0HQRmSP0BbnxJ0jsXyZqCuAb48v6FMBNIRlOqaOYPuE9o+gaqptiPslRG5D/1Irg/5\nBIk9At8gleeO4CA4zdjBPbRiQ5HdNtwRb6p97LRVVWeVrIrEShnPpb036ngCcihWlbeWtt\n8uvM7YxDFazC5LEbhBfIBuZFexBsowAAAAMBAAEAAAGAflHjdb2oV4HkQetBsSRa18QM1m\ncxAoOE+SiTYRudGQ6KtSzY8MGZ/xca7QiXfXhbF1+llTTiQ/i0Dtu+H0blyfLIgZwIGIsl\nG2GCf/7MoG//kmhaFuY3O56Rj3MyQVVPgHLy+VhE6hFniske+C4jhicc/aL7nOu15n3Qad\nJLmV8KB9EIjevDoloXgk9ot/WyuXKLmMaa9rFIA+UDmJyGtfFbbsOrHbj8sS11/oSD14RT\nLBygEb2EUI52j2LmY/LEvUL+59oCuJ6Y/h+pMdFeuHJzGjrVb573KnGwejzY24HHzzebrC\nQ+9NyVCTyizPHNu9w52/GPEZQFQBi7o9cDMd3ITZEPIaIvDHsUwPXaHUBHy/XHQTs8pDqk\nzCMcAs5zdzao2I0LQ+ZFYyvl1rue82ITjDISX1WK6nFYLBVXugi0rLGEdH6P+Psfl3uCIf\naW7c12/BpZz2Pql5AuO1wsu4rmz2th68vaC/0IDqWekIbW9qihFbqnhfAxRsIURjpBAAAA\nwDhIQPsj9T9Vud3Z/TZjiAKCPbg3zi082u1GMMxXnNQtKO3J35wU7VUcAxAzosWr+emMqS\nU0qW+a5RXr3sqUOqH85b5+Xw0yv2sTr2pL0ALFW7Tq1mesCc3K0So3Yo30pWRIOxYM9ihm\nE4ci/3mN5kcKWwvLLomFPRU9u0XtIGKnF/cNByTuz9fceR6Pi6mQXZawv+OOMiBeu0gbyp\nF1uVe8PCshzCrWTE3UjRpQxy9gizvSbGZyGQi1Lm42JXKG3wAAAMEA4r4CLM1xsyxBBMld\nrxiTqy6bfrZjKkT5MPjBjp+57i5kW9NVqGCnIy/m98pLTuKjTCDmUuWQXS+oqhHw5vq/wj\nRvQYqkJDz1UGmC1lD2qyqERjOiWa8/iy4dXSLeHCT70+/xR2dBb0z8cT++yZEqLdEZSnHG\nyRaZMHot1OohVDqJS8nEbxOzgPGdopRMiX6ws/p5/k9YAGkHx0hszA8cn/Tk2/mdS5lugw\nY7mdXzfcKvxkgoFrG7XowqRVrozcvDAAAAwQDU1ITasquNLaQhKNqiHx/N7bvKVO33icAx\nNdShqJEWx/g9idvQ25sA1Ubc1a+Ot5Lgfrs2OBKe+LgSmPAZOjv4ShqBHtsSh3am8/K1xR\ngQKgojLL4FhtgxtwoZrVvovZHGV3g2A28BRGbKIGVGPsOszJALU7jlLlcTHlB7SCQBI8FQ\nvTi2UEsfTmA22NnuVPITeqbmAQQXkSZcZbpbvdc0vQzp/3iOb/OCrIMET3HqVEMyQVsVs6\nxa9026AMTGLaEAAAATcm9vdEBvcGVuc3NoLXNlcnZlcg==\n-----END OPENSSH PRIVATE KEY-----",  # noqa: E501
-        },
-    }
+
+@pytest.fixture(scope="session", name="postgres_config_ssh_tunnel")
+def postgres_config_ssh_tunnel_fixture():
+    return postgres_config_ssh_tunnel()
 
 
 @pytest.fixture
 def postgres_target(postgres_config) -> TargetPostgres:
     return TargetPostgres(config=postgres_config)
 
 
-def create_engine(target_postgres: TargetPostgres) -> sqlalchemy.engine.Engine:
-    return TargetPostgres.default_sink_class.connector_class(
-        config=target_postgres.config
-    )._engine
-
-
 def singer_file_to_target(file_name, target) -> None:
     """Singer file to Target, emulates a tap run