diff --git a/target_postgres/tests/core.py b/target_postgres/tests/core.py new file mode 100644 index 00000000..ba5662be --- /dev/null +++ b/target_postgres/tests/core.py @@ -0,0 +1,59 @@ +""" Config and base values for target-postgres testing """ +# flake8: noqa +import sqlalchemy + +from target_postgres.target import TargetPostgres + + +def postgres_config(): + return { + "dialect+driver": "postgresql+psycopg2", + "host": "localhost", + "user": "postgres", + "password": "postgres", + "database": "postgres", + "port": 5432, + "ssl_enable": True, + "ssl_client_certificate_enable": True, + "ssl_mode": "verify-full", + "ssl_certificate_authority": "./ssl/root.crt", + "ssl_client_certificate": "./ssl/cert.crt", + "ssl_client_private_key": "./ssl/pkey.key", + "ssl_storage_directory": ".secrets", + "add_record_metadata": True, + "hard_delete": False, + "default_target_schema": "melty", + } + + +def postgres_config_no_ssl(): + return { + "dialect+driver": "postgresql+psycopg2", + "host": "localhost", + "user": "postgres", + "password": "postgres", + "database": "postgres", + "port": 5433, + "add_record_metadata": True, + "hard_delete": False, + "default_target_schema": "melty", + } + + +def postgres_config_ssh_tunnel(): + return { + "sqlalchemy_url": "postgresql://postgres:postgres@10.5.0.5:5432/main", + "ssh_tunnel": { + "enable": True, + "host": "127.0.0.1", + "port": 2223, + "username": "melty", + "private_key": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn\nNhAAAAAwEAAQAAAYEAvIGU0pRpThhIcaSPrg2+v7cXl+QcG0icb45hfD44yrCoXkpJp7nh\nHv0ObZL2Y1cG7eeayYF4AqD3kwQ7W89GN6YO9b/mkJgawk0/YLUyojTS9dbcTbdkfPzyUa\nvTMDjly+PIjfiWOEnUgPf1y3xONLkJU0ILyTmgTzSIMNdKngtdCGfytBCuNiPKU8hEdEVt\n82ebqgtLoSYn9cUcVVz6LewzUh8+YtoPb8Z/BIVEzU37HiE9MOYIBXjo1AEJSnOCkjwlVl\nPzLhcXKTPht0iwv/KnZNNg0LDmnU/z0n+nPq/EMflum8jRYbgp0C5hksPdc8e0eEKd9gak\nt7B0ta3Mjt5b8HPQdBGZI/QFufEnSOxfJmoK4Bvjy/oUwE0hGU6po5g+4T2j6Bqqm2I+yV\nEbkP/UiuD/kEiT0C3yCV547gIDjN2ME9tGJDkd023BFvqn3stFVVZ5WsisRKGc+lvTfqeA\nJyKFaVt5a23y68ztjEMVrMLksRuEF8gG5kV7EGyjAAAFiCzGBRksxgUZAAAAB3NzaC1yc2\nEAAAGBALyBlNKUaU4YSHGkj64Nvr+3F5fkHBtInG+OYXw+OMqwqF5KSae54R79Dm2S9mNX\nBu3nmsmBeAKg95MEO1vPRjemDvW/5pCYGsJNP2C1MqI00vXW3E23ZHz88lGr0zA45cvjyI\n34ljhJ1ID39ct8TjS5CVNCC8k5oE80iDDXSp4LXQhn8rQQrjYjylPIRHRFbfNnm6oLS6Em\nJ/XFHFVc+i3sM1IfPmLaD2/GfwSFRM1N+x4hPTDmCAV46NQBCUpzgpI8JVZT8y4XFykz4b\ndIsL/yp2TTYNCw5p1P89J/pz6vxDH5bpvI0WG4KdAuYZLD3XPHtHhCnfYGpLewdLWtzI7e\nW/Bz0HQRmSP0BbnxJ0jsXyZqCuAb48v6FMBNIRlOqaOYPuE9o+gaqptiPslRG5D/1Irg/5\nBIk9At8gleeO4CA4zdjBPbRiQ5HdNtwRb6p97LRVVWeVrIrEShnPpb036ngCcihWlbeWtt\n8uvM7YxDFazC5LEbhBfIBuZFexBsowAAAAMBAAEAAAGAflHjdb2oV4HkQetBsSRa18QM1m\ncxAoOE+SiTYRudGQ6KtSzY8MGZ/xca7QiXfXhbF1+llTTiQ/i0Dtu+H0blyfLIgZwIGIsl\nG2GCf/7MoG//kmhaFuY3O56Rj3MyQVVPgHLy+VhE6hFniske+C4jhicc/aL7nOu15n3Qad\nJLmV8KB9EIjevDoloXgk9ot/WyuXKLmMaa9rFIA+UDmJyGtfFbbsOrHbj8sS11/oSD14RT\nLBygEb2EUI52j2LmY/LEvUL+59oCuJ6Y/h+pMdFeuHJzGjrVb573KnGwejzY24HHzzebrC\nQ+9NyVCTyizPHNu9w52/GPEZQFQBi7o9cDMd3ITZEPIaIvDHsUwPXaHUBHy/XHQTs8pDqk\nzCMcAs5zdzao2I0LQ+ZFYyvl1rue82ITjDISX1WK6nFYLBVXugi0rLGEdH6P+Psfl3uCIf\naW7c12/BpZz2Pql5AuO1wsu4rmz2th68vaC/0IDqWekIbW9qihFbqnhfAxRsIURjpBAAAA\nwDhIQPsj9T9Vud3Z/TZjiAKCPbg3zi082u1GMMxXnNQtKO3J35wU7VUcAxAzosWr+emMqS\nU0qW+a5RXr3sqUOqH85b5+Xw0yv2sTr2pL0ALFW7Tq1mesCc3K0So3Yo30pWRIOxYM9ihm\nE4ci/3mN5kcKWwvLLomFPRU9u0XtIGKnF/cNByTuz9fceR6Pi6mQXZawv+OOMiBeu0gbyp\nF1uVe8PCshzCrWTE3UjRpQxy9gizvSbGZyGQi1Lm42JXKG3wAAAMEA4r4CLM1xsyxBBMld\nrxiTqy6bfrZjKkT5MPjBjp+57i5kW9NVqGCnIy/m98pLTuKjTCDmUuWQXS+oqhHw5vq/wj\nRvQYqkJDz1UGmC1lD2qyqERjOiWa8/iy4dXSLeHCT70+/xR2dBb0z8cT++yZEqLdEZSnHG\nyRaZMHot1OohVDqJS8nEbxOzgPGdopRMiX6ws/p5/k9YAGkHx0hszA8cn/Tk2/mdS5lugw\nY7mdXzfcKvxkgoFrG7XowqRVrozcvDAAAAwQDU1ITasquNLaQhKNqiHx/N7bvKVO33icAx\nNdShqJEWx/g9idvQ25sA1Ubc1a+Ot5Lgfrs2OBKe+LgSmPAZOjv4ShqBHtsSh3am8/K1xR\ngQKgojLL4FhtgxtwoZrVvovZHGV3g2A28BRGbKIGVGPsOszJALU7jlLlcTHlB7SCQBI8FQ\nvTi2UEsfTmA22NnuVPITeqbmAQQXkSZcZbpbvdc0vQzp/3iOb/OCrIMET3HqVEMyQVsVs6\nxa9026AMTGLaEAAAATcm9vdEBvcGVuc3NoLXNlcnZlcg==\n-----END OPENSSH PRIVATE KEY-----", # noqa: E501 + }, + } + + +def create_engine(target_postgres: TargetPostgres) -> sqlalchemy.engine.Engine: + return TargetPostgres.default_sink_class.connector_class( + config=target_postgres.config + )._engine diff --git a/target_postgres/tests/test_sdk.py b/target_postgres/tests/test_sdk.py new file mode 100644 index 00000000..3f95c393 --- /dev/null +++ b/target_postgres/tests/test_sdk.py @@ -0,0 +1,77 @@ +""" SDK tests for target postgres """ +# flake8: noqa +import pytest +from singer_sdk.testing import get_target_test_class +from singer_sdk.testing.suites import TestSuite +from singer_sdk.testing.target_tests import ( + TargetArrayData, + TargetCamelcaseComplexSchema, + TargetCamelcaseTest, + TargetCliPrintsTest, + TargetDuplicateRecords, + TargetEncodedStringData, + TargetInvalidSchemaTest, + TargetMultipleStateMessages, + TargetNoPrimaryKeys, + TargetOptionalAttributes, + TargetRecordBeforeSchemaTest, + TargetRecordMissingKeyProperty, + TargetRecordMissingOptionalFields, + TargetRecordMissingRequiredProperty, + TargetSchemaNoProperties, + TargetSchemaUpdates, + TargetSpecialCharsInAttributes, +) + +from target_postgres.target import TargetPostgres + +from .core import create_engine, postgres_config + +target_tests = TestSuite( + kind="target", + tests=[ + TargetArrayData, + TargetCamelcaseComplexSchema, + TargetCamelcaseTest, + TargetCliPrintsTest, + TargetDuplicateRecords, + # Postgres doesn't support NULL characters in strings + # TargetEncodedStringData, + TargetInvalidSchemaTest, + # This tap only outputs one state message at the end of execution, fails assertion. + # Separate custom test in test_target_postgres.py + # TargetMultipleStateMessages, + TargetNoPrimaryKeys, + TargetOptionalAttributes, + TargetRecordBeforeSchemaTest, + TargetRecordMissingKeyProperty, + # Fails, but gives appropriate error message + # TargetRecordMissingRequiredProperty, + TargetSchemaNoProperties, + TargetSchemaUpdates, + TargetSpecialCharsInAttributes, + TargetRecordMissingOptionalFields, + ], +) + + +class BasePostgresSDKTests: + """Base class for Postgres SDK tests.""" + + @pytest.fixture() + def connection(self, runner): + engine = create_engine(runner) + return engine.connect() + + +SDKTests = get_target_test_class( + target_class=TargetPostgres, + config=postgres_config(), + custom_suites=[target_tests], + suite_config=None, + include_target_tests=False, +) + + +class TestTargetPostgres(BasePostgresSDKTests, SDKTests): + """SDK tests""" diff --git a/target_postgres/tests/test_standard_target.py b/target_postgres/tests/test_target_postgres.py similarity index 85% rename from target_postgres/tests/test_standard_target.py rename to target_postgres/tests/test_target_postgres.py index 4a5068b7..6974f703 100644 --- a/target_postgres/tests/test_standard_target.py +++ b/target_postgres/tests/test_target_postgres.py @@ -1,8 +1,7 @@ -""" Attempt at making some standard Target Tests. """ +""" Postgres target tests """ # flake8: noqa import copy import io -import uuid from contextlib import redirect_stdout from pathlib import Path @@ -10,7 +9,7 @@ import pytest import sqlalchemy from singer_sdk.exceptions import MissingKeyPropertiesError -from singer_sdk.testing import sync_end_to_end +from singer_sdk.testing import get_target_test_class, sync_end_to_end from sqlalchemy.dialects.postgresql import ARRAY from sqlalchemy.types import TEXT, TIMESTAMP @@ -21,55 +20,27 @@ SampleTapCountries, ) +from .core import ( + create_engine, + postgres_config, + postgres_config_no_ssl, + postgres_config_ssh_tunnel, +) -@pytest.fixture(scope="session") -def postgres_config(): - return { - "dialect+driver": "postgresql+psycopg2", - "host": "localhost", - "user": "postgres", - "password": "postgres", - "database": "postgres", - "port": 5432, - "ssl_enable": True, - "ssl_client_certificate_enable": True, - "ssl_mode": "verify-full", - "ssl_certificate_authority": "./ssl/root.crt", - "ssl_client_certificate": "./ssl/cert.crt", - "ssl_client_private_key": "./ssl/pkey.key", - "add_record_metadata": True, - "hard_delete": False, - "default_target_schema": "melty", - } +@pytest.fixture(scope="session", name="postgres_config") +def postgres_config_fixture(): + return postgres_config() -@pytest.fixture(scope="session") -def postgres_config_no_ssl(): - return { - "dialect+driver": "postgresql+psycopg2", - "host": "localhost", - "user": "postgres", - "password": "postgres", - "database": "postgres", - "port": 5433, - "add_record_metadata": True, - "hard_delete": False, - "default_target_schema": "melty", - } +@pytest.fixture(scope="session", name="postgres_config_no_ssl") +def postgres_config_no_ssl_fixture(): + return postgres_config_no_ssl() -@pytest.fixture(scope="session") -def postgres_config_ssh_tunnel(): - return { - "sqlalchemy_url": "postgresql://postgres:postgres@10.5.0.5:5432/main", - "ssh_tunnel": { - "enable": True, - "host": "127.0.0.1", - "port": 2223, - "username": "melty", - "private_key": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn\nNhAAAAAwEAAQAAAYEAvIGU0pRpThhIcaSPrg2+v7cXl+QcG0icb45hfD44yrCoXkpJp7nh\nHv0ObZL2Y1cG7eeayYF4AqD3kwQ7W89GN6YO9b/mkJgawk0/YLUyojTS9dbcTbdkfPzyUa\nvTMDjly+PIjfiWOEnUgPf1y3xONLkJU0ILyTmgTzSIMNdKngtdCGfytBCuNiPKU8hEdEVt\n82ebqgtLoSYn9cUcVVz6LewzUh8+YtoPb8Z/BIVEzU37HiE9MOYIBXjo1AEJSnOCkjwlVl\nPzLhcXKTPht0iwv/KnZNNg0LDmnU/z0n+nPq/EMflum8jRYbgp0C5hksPdc8e0eEKd9gak\nt7B0ta3Mjt5b8HPQdBGZI/QFufEnSOxfJmoK4Bvjy/oUwE0hGU6po5g+4T2j6Bqqm2I+yV\nEbkP/UiuD/kEiT0C3yCV547gIDjN2ME9tGJDkd023BFvqn3stFVVZ5WsisRKGc+lvTfqeA\nJyKFaVt5a23y68ztjEMVrMLksRuEF8gG5kV7EGyjAAAFiCzGBRksxgUZAAAAB3NzaC1yc2\nEAAAGBALyBlNKUaU4YSHGkj64Nvr+3F5fkHBtInG+OYXw+OMqwqF5KSae54R79Dm2S9mNX\nBu3nmsmBeAKg95MEO1vPRjemDvW/5pCYGsJNP2C1MqI00vXW3E23ZHz88lGr0zA45cvjyI\n34ljhJ1ID39ct8TjS5CVNCC8k5oE80iDDXSp4LXQhn8rQQrjYjylPIRHRFbfNnm6oLS6Em\nJ/XFHFVc+i3sM1IfPmLaD2/GfwSFRM1N+x4hPTDmCAV46NQBCUpzgpI8JVZT8y4XFykz4b\ndIsL/yp2TTYNCw5p1P89J/pz6vxDH5bpvI0WG4KdAuYZLD3XPHtHhCnfYGpLewdLWtzI7e\nW/Bz0HQRmSP0BbnxJ0jsXyZqCuAb48v6FMBNIRlOqaOYPuE9o+gaqptiPslRG5D/1Irg/5\nBIk9At8gleeO4CA4zdjBPbRiQ5HdNtwRb6p97LRVVWeVrIrEShnPpb036ngCcihWlbeWtt\n8uvM7YxDFazC5LEbhBfIBuZFexBsowAAAAMBAAEAAAGAflHjdb2oV4HkQetBsSRa18QM1m\ncxAoOE+SiTYRudGQ6KtSzY8MGZ/xca7QiXfXhbF1+llTTiQ/i0Dtu+H0blyfLIgZwIGIsl\nG2GCf/7MoG//kmhaFuY3O56Rj3MyQVVPgHLy+VhE6hFniske+C4jhicc/aL7nOu15n3Qad\nJLmV8KB9EIjevDoloXgk9ot/WyuXKLmMaa9rFIA+UDmJyGtfFbbsOrHbj8sS11/oSD14RT\nLBygEb2EUI52j2LmY/LEvUL+59oCuJ6Y/h+pMdFeuHJzGjrVb573KnGwejzY24HHzzebrC\nQ+9NyVCTyizPHNu9w52/GPEZQFQBi7o9cDMd3ITZEPIaIvDHsUwPXaHUBHy/XHQTs8pDqk\nzCMcAs5zdzao2I0LQ+ZFYyvl1rue82ITjDISX1WK6nFYLBVXugi0rLGEdH6P+Psfl3uCIf\naW7c12/BpZz2Pql5AuO1wsu4rmz2th68vaC/0IDqWekIbW9qihFbqnhfAxRsIURjpBAAAA\nwDhIQPsj9T9Vud3Z/TZjiAKCPbg3zi082u1GMMxXnNQtKO3J35wU7VUcAxAzosWr+emMqS\nU0qW+a5RXr3sqUOqH85b5+Xw0yv2sTr2pL0ALFW7Tq1mesCc3K0So3Yo30pWRIOxYM9ihm\nE4ci/3mN5kcKWwvLLomFPRU9u0XtIGKnF/cNByTuz9fceR6Pi6mQXZawv+OOMiBeu0gbyp\nF1uVe8PCshzCrWTE3UjRpQxy9gizvSbGZyGQi1Lm42JXKG3wAAAMEA4r4CLM1xsyxBBMld\nrxiTqy6bfrZjKkT5MPjBjp+57i5kW9NVqGCnIy/m98pLTuKjTCDmUuWQXS+oqhHw5vq/wj\nRvQYqkJDz1UGmC1lD2qyqERjOiWa8/iy4dXSLeHCT70+/xR2dBb0z8cT++yZEqLdEZSnHG\nyRaZMHot1OohVDqJS8nEbxOzgPGdopRMiX6ws/p5/k9YAGkHx0hszA8cn/Tk2/mdS5lugw\nY7mdXzfcKvxkgoFrG7XowqRVrozcvDAAAAwQDU1ITasquNLaQhKNqiHx/N7bvKVO33icAx\nNdShqJEWx/g9idvQ25sA1Ubc1a+Ot5Lgfrs2OBKe+LgSmPAZOjv4ShqBHtsSh3am8/K1xR\ngQKgojLL4FhtgxtwoZrVvovZHGV3g2A28BRGbKIGVGPsOszJALU7jlLlcTHlB7SCQBI8FQ\nvTi2UEsfTmA22NnuVPITeqbmAQQXkSZcZbpbvdc0vQzp/3iOb/OCrIMET3HqVEMyQVsVs6\nxa9026AMTGLaEAAAATcm9vdEBvcGVuc3NoLXNlcnZlcg==\n-----END OPENSSH PRIVATE KEY-----", # noqa: E501 - }, - } + +@pytest.fixture(scope="session", name="postgres_config_ssh_tunnel") +def postgres_config_ssh_tunnel_fixture(): + return postgres_config_ssh_tunnel() @pytest.fixture @@ -77,12 +48,6 @@ def postgres_target(postgres_config) -> TargetPostgres: return TargetPostgres(config=postgres_config) -def create_engine(target_postgres: TargetPostgres) -> sqlalchemy.engine.Engine: - return TargetPostgres.default_sink_class.connector_class( - config=target_postgres.config - )._engine - - def singer_file_to_target(file_name, target) -> None: """Singer file to Target, emulates a tap run