From ab1cbae1d51c19efe8a30bff651505542e35ce55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Wed, 3 Jul 2024 14:14:28 -0600 Subject: [PATCH] fix: Map `real`/`float4` types to JSON schema `number` (#450) https://www.postgresql.org/docs/current/datatype.html --- .github/workflows/test.yml | 8 +++++++- tap_postgres/client.py | 2 ++ tests/test_core.py | 26 ++++++++++++++++++++------ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e191fc4d..85c853b9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,8 +18,14 @@ jobs: env: PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt strategy: + fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" steps: - name: Checkout code diff --git a/tap_postgres/client.py b/tap_postgres/client.py index f6d8b888..75c88954 100644 --- a/tap_postgres/client.py +++ b/tap_postgres/client.py @@ -212,6 +212,8 @@ def sdk_typing_object( "decimal": th.NumberType(), "double": th.NumberType(), "float": th.NumberType(), + "real": th.NumberType(), + "float4": th.NumberType(), "string": th.StringType(), "text": th.StringType(), "char": th.StringType(), diff --git a/tests/test_core.py b/tests/test_core.py index b59b2ee0..b46da656 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -339,7 +339,7 @@ def test_jsonb_array(): assert test_runner.records[altered_table_name][i] == rows[i] -def test_decimal(): +def test_numeric_types(): """Schema was wrong for Decimal objects. Check they are correctly selected.""" table_name = "test_decimal" engine = sa.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) @@ -348,20 +348,31 @@ def test_decimal(): table = sa.Table( table_name, metadata_obj, - sa.Column("column", sa.Numeric()), + sa.Column("my_numeric", sa.Numeric()), + sa.Column("my_real", sa.REAL()), ) with engine.begin() as conn: table.drop(conn, checkfirst=True) metadata_obj.create_all(conn) - insert = table.insert().values(column=decimal.Decimal("3.14")) + insert = table.insert().values( + my_numeric=decimal.Decimal("3.14"), + my_real=3.14, + ) conn.execute(insert) - insert = table.insert().values(column=decimal.Decimal("12")) + insert = table.insert().values( + my_numeric=decimal.Decimal("12"), + my_real=12, + ) conn.execute(insert) - insert = table.insert().values(column=decimal.Decimal("10000.00001")) + insert = table.insert().values( + my_numeric=decimal.Decimal("10000.00001"), + my_real=10000.00001, + ) conn.execute(insert) tap = TapPostgres(config=SAMPLE_CONFIG) tap_catalog = json.loads(tap.catalog_json_text) altered_table_name = f"{DB_SCHEMA_NAME}-{table_name}" + for stream in tap_catalog["streams"]: if stream.get("stream") and altered_table_name not in stream["stream"]: for metadata in stream["metadata"]: @@ -376,12 +387,15 @@ def test_decimal(): tap_class=TapPostgres, config=SAMPLE_CONFIG, catalog=tap_catalog ) test_runner.sync_all() + for schema_message in test_runner.schema_messages: if ( "stream" in schema_message and schema_message["stream"] == altered_table_name ): - assert "number" in schema_message["schema"]["properties"]["column"]["type"] + props = schema_message["schema"]["properties"] + assert "number" in props["my_numeric"]["type"] + assert "number" in props["my_real"]["type"] def test_filter_schemas():