From 0ca861765e5714ba0d771ae4adb42d935759f4bf Mon Sep 17 00:00:00 2001 From: hholgersen Date: Tue, 10 Jan 2023 16:22:43 +0100 Subject: [PATCH 01/12] added settings in target spec --- meltano.yml | 1 - target_mssql/connector.py | 2 +- target_mssql/target.py | 25 +++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/meltano.yml b/meltano.yml index 91235b8..81a530b 100644 --- a/meltano.yml +++ b/meltano.yml @@ -19,7 +19,6 @@ plugins: config: start_date: '2010-01-01T00:00:00Z' settings: - # TODO: To configure using Meltano, declare settings and their types here: - name: username - name: password kind: password diff --git a/target_mssql/connector.py b/target_mssql/connector.py index 662cd2d..7c9f018 100644 --- a/target_mssql/connector.py +++ b/target_mssql/connector.py @@ -62,7 +62,7 @@ def get_sqlalchemy_url(self, config: dict) -> str: connection_url = sqlalchemy.engine.url.URL.create( drivername="mssql+pymssql", - username=config["user"], + username=config["username"], password=config["password"], host=config["host"], port=config["port"], diff --git a/target_mssql/target.py b/target_mssql/target.py index a13cefe..df591ee 100644 --- a/target_mssql/target.py +++ b/target_mssql/target.py @@ -18,6 +18,31 @@ class Targetmssql(SQLTarget): th.StringType, description="SQLAlchemy connection string", ), + th.Property( + "username", + th.StringType, + description="SQL Server username", + ), + th.Property( + "password", + th.StringType, + description="SQL Server password", + ), + th.Property( + "host", + th.StringType, + description="SQL Server host", + ), + th.Property( + "port", + th.StringType, + description="SQL Server port", + ), + th.Property( + "database", + th.StringType, + description="SQL Server database", + ) ).to_dict() default_sink_class = mssqlSink From 17b3f1c67b2a3fae5c2860d68f0ad96e25416e06 Mon Sep 17 00:00:00 2001 From: hholgersen Date: Tue, 10 Jan 2023 16:24:59 +0100 Subject: [PATCH 02/12] added settings in target spec --- target_mssql/target.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/target_mssql/target.py b/target_mssql/target.py index df591ee..004373c 100644 --- a/target_mssql/target.py +++ b/target_mssql/target.py @@ -36,12 +36,18 @@ class Targetmssql(SQLTarget): th.Property( "port", th.StringType, + default='1433', description="SQL Server port", ), th.Property( "database", th.StringType, description="SQL Server database", + ), + th.Property( + "default_target_schema", + th.StringType, + description="Default target schema to write to", ) ).to_dict() From 27720ece724e6e3f3ea6b8de6086621e10314f7b Mon Sep 17 00:00:00 2001 From: hholgersen Date: Wed, 11 Jan 2023 15:57:49 +0100 Subject: [PATCH 03/12] replace temp table --- target_mssql/connector.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/target_mssql/connector.py b/target_mssql/connector.py index 7c9f018..d46fd5d 100644 --- a/target_mssql/connector.py +++ b/target_mssql/connector.py @@ -374,10 +374,13 @@ def create_temp_table_from_table(self, from_table_name): f"{schema_name}.#{table_name}" if schema_name else f"#{table_name}" ) + droptable = f"DROP TABLE IF EXISTS {full_table_name}" + self.connection.execute(droptable) + ddl = f""" SELECT TOP 0 * into {tmp_full_table_name} FROM {full_table_name} """ - + self.connection.execute(ddl) From 5a5ace45b5bdad31da7d96cf1f783a011ffe44d1 Mon Sep 17 00:00:00 2001 From: hholgersen Date: Wed, 11 Jan 2023 16:53:09 +0100 Subject: [PATCH 04/12] replace temp table --- target_mssql/connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_mssql/connector.py b/target_mssql/connector.py index d46fd5d..c2a79f4 100644 --- a/target_mssql/connector.py +++ b/target_mssql/connector.py @@ -374,7 +374,7 @@ def create_temp_table_from_table(self, from_table_name): f"{schema_name}.#{table_name}" if schema_name else f"#{table_name}" ) - droptable = f"DROP TABLE IF EXISTS {full_table_name}" + droptable = f"DROP TABLE IF EXISTS {tmp_full_table_name}" self.connection.execute(droptable) ddl = f""" From 34c9ffb64baa323f969277b8b02421345f76fd6c Mon Sep 17 00:00:00 2001 From: hholgersen Date: Wed, 11 Jan 2023 17:02:09 +0100 Subject: [PATCH 05/12] replace temp table --- target_mssql/sinks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_mssql/sinks.py b/target_mssql/sinks.py index 5af916e..1e93570 100644 --- a/target_mssql/sinks.py +++ b/target_mssql/sinks.py @@ -237,7 +237,7 @@ def merge_upsert_from_table( self.connection.execute(merge_sql) - self.connection.execute("COMMIT") +# self.connection.execute("COMMIT") def parse_full_table_name( self, full_table_name: str From 2227aaa860a963bc4534fda73d4535aa0a007d0b Mon Sep 17 00:00:00 2001 From: hholgersen Date: Mon, 16 Jan 2023 16:36:27 +0100 Subject: [PATCH 06/12] black reformatting and max varchar length --- target_mssql/connector.py | 4 ++-- target_mssql/sinks.py | 2 +- target_mssql/target.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/target_mssql/connector.py b/target_mssql/connector.py index c2a79f4..e783893 100644 --- a/target_mssql/connector.py +++ b/target_mssql/connector.py @@ -343,7 +343,7 @@ def to_sql_type(self, jsonschema_type: dict) -> sqlalchemy.types.TypeEngine: # if datelike_type == "date": return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.DATE()) - maxlength = jsonschema_type.get("maxLength") + maxlength = min(jsonschema_type.get("maxLength"), 8000) return cast( sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(maxlength) ) @@ -382,5 +382,5 @@ def create_temp_table_from_table(self, from_table_name): into {tmp_full_table_name} FROM {full_table_name} """ - + self.connection.execute(ddl) diff --git a/target_mssql/sinks.py b/target_mssql/sinks.py index 1e93570..ee16eeb 100644 --- a/target_mssql/sinks.py +++ b/target_mssql/sinks.py @@ -237,7 +237,7 @@ def merge_upsert_from_table( self.connection.execute(merge_sql) -# self.connection.execute("COMMIT") + # self.connection.execute("COMMIT") def parse_full_table_name( self, full_table_name: str diff --git a/target_mssql/target.py b/target_mssql/target.py index 004373c..7aca73e 100644 --- a/target_mssql/target.py +++ b/target_mssql/target.py @@ -36,7 +36,7 @@ class Targetmssql(SQLTarget): th.Property( "port", th.StringType, - default='1433', + default="1433", description="SQL Server port", ), th.Property( @@ -48,7 +48,7 @@ class Targetmssql(SQLTarget): "default_target_schema", th.StringType, description="Default target schema to write to", - ) + ), ).to_dict() default_sink_class = mssqlSink From c7fa58ceb6f3571b46407f94f36aa94325ef096b Mon Sep 17 00:00:00 2001 From: hholgersen Date: Mon, 16 Jan 2023 17:05:39 +0100 Subject: [PATCH 07/12] connectionstring bugfix --- target_mssql/tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_mssql/tests/test_core.py b/target_mssql/tests/test_core.py index 1aaaeb5..4b261cc 100644 --- a/target_mssql/tests/test_core.py +++ b/target_mssql/tests/test_core.py @@ -17,7 +17,7 @@ @pytest.fixture() def mssql_config(): return { - "sqlalchemy_url": "mssql+pymssql://sa:p@55w0rd@localhost:1433/master", + #"sqlalchemy_url": "mssql+pymssql://sa:p@55w0rd@localhost:1433/master", "schema": "dbo", "user": "sa", "password": "P@55w0rd", From 30e78692ee16c42f11cb4eb42383198b326d008b Mon Sep 17 00:00:00 2001 From: hholgersen Date: Mon, 16 Jan 2023 17:32:32 +0100 Subject: [PATCH 08/12] connectionstring bugfix --- target_mssql/tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_mssql/tests/test_core.py b/target_mssql/tests/test_core.py index 4b261cc..e05b931 100644 --- a/target_mssql/tests/test_core.py +++ b/target_mssql/tests/test_core.py @@ -19,7 +19,7 @@ def mssql_config(): return { #"sqlalchemy_url": "mssql+pymssql://sa:p@55w0rd@localhost:1433/master", "schema": "dbo", - "user": "sa", + "username": "sa", "password": "P@55w0rd", "host": "localhost", "port": "1433", From 092b63dc7cdbf832e0598f086b5dc8da4897223b Mon Sep 17 00:00:00 2001 From: hholgersen Date: Mon, 16 Jan 2023 18:05:49 +0100 Subject: [PATCH 09/12] more bugfix because min of nontype no fun --- target_mssql/connector.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/target_mssql/connector.py b/target_mssql/connector.py index e783893..99cf54b 100644 --- a/target_mssql/connector.py +++ b/target_mssql/connector.py @@ -343,7 +343,10 @@ def to_sql_type(self, jsonschema_type: dict) -> sqlalchemy.types.TypeEngine: # if datelike_type == "date": return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.DATE()) - maxlength = min(jsonschema_type.get("maxLength"), 8000) + maxlength = jsonschema_type.get("maxLength") + if maxlength > 8000: + maxlength = 8000 + return cast( sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(maxlength) ) From 1fe90c588e73f7fb730e25542f0d957cb9ba29bd Mon Sep 17 00:00:00 2001 From: hholgersen Date: Mon, 16 Jan 2023 18:19:06 +0100 Subject: [PATCH 10/12] more bugfix because min of nontype no fun --- target_mssql/connector.py | 7 +++---- target_mssql/tests/test_core.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/target_mssql/connector.py b/target_mssql/connector.py index 99cf54b..6bbf960 100644 --- a/target_mssql/connector.py +++ b/target_mssql/connector.py @@ -343,12 +343,11 @@ def to_sql_type(self, jsonschema_type: dict) -> sqlalchemy.types.TypeEngine: # if datelike_type == "date": return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.DATE()) - maxlength = jsonschema_type.get("maxLength") - if maxlength > 8000: - maxlength = 8000 + maxlength = jsonschema_type.get("maxLength") or 8000 + target_length = min(maxlength, 8000) return cast( - sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(maxlength) + sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(target_length) ) if self._jsonschema_type_check(jsonschema_type, ("integer",)): diff --git a/target_mssql/tests/test_core.py b/target_mssql/tests/test_core.py index e05b931..9eb82c4 100644 --- a/target_mssql/tests/test_core.py +++ b/target_mssql/tests/test_core.py @@ -17,7 +17,7 @@ @pytest.fixture() def mssql_config(): return { - #"sqlalchemy_url": "mssql+pymssql://sa:p@55w0rd@localhost:1433/master", + # "sqlalchemy_url": "mssql+pymssql://sa:p@55w0rd@localhost:1433/master", "schema": "dbo", "username": "sa", "password": "P@55w0rd", From 25d1359f268d0cf5524b54dbb53e52b6785e8a00 Mon Sep 17 00:00:00 2001 From: Henning Date: Thu, 19 Jan 2023 11:46:17 +0100 Subject: [PATCH 11/12] handling fields >8000 char --- target_mssql/connector.py | 10 +++++++--- target_mssql/sinks.py | 2 +- target_mssql/tests/data_files/long_string.singer | 3 +++ 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 target_mssql/tests/data_files/long_string.singer diff --git a/target_mssql/connector.py b/target_mssql/connector.py index 6bbf960..8328c45 100644 --- a/target_mssql/connector.py +++ b/target_mssql/connector.py @@ -343,11 +343,15 @@ def to_sql_type(self, jsonschema_type: dict) -> sqlalchemy.types.TypeEngine: # if datelike_type == "date": return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.DATE()) - maxlength = jsonschema_type.get("maxLength") or 8000 - target_length = min(maxlength, 8000) + maxlength = jsonschema_type.get("maxLength") + if maxlength is not None: + if maxlength > 8000: + return cast( + sqlalchemy.types.TypeEngine, sqlalchemy.types.TEXT() + ) return cast( - sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(target_length) + sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(maxlength) ) if self._jsonschema_type_check(jsonschema_type, ("integer",)): diff --git a/target_mssql/sinks.py b/target_mssql/sinks.py index ee16eeb..5af916e 100644 --- a/target_mssql/sinks.py +++ b/target_mssql/sinks.py @@ -237,7 +237,7 @@ def merge_upsert_from_table( self.connection.execute(merge_sql) - # self.connection.execute("COMMIT") + self.connection.execute("COMMIT") def parse_full_table_name( self, full_table_name: str diff --git a/target_mssql/tests/data_files/long_string.singer b/target_mssql/tests/data_files/long_string.singer new file mode 100644 index 0000000..ccfa532 --- /dev/null +++ b/target_mssql/tests/data_files/long_string.singer @@ -0,0 +1,3 @@ +{"type": "SCHEMA", "stream": "long_string", "schema": {"type": "object", "properties": { "CustomerID": {"type": "string", "maxlength": 12000}, "clientName": {"type": "string"} }}, "key_properties": ["CustomerID"]} +{"type": "RECORD", "stream": "long_string", "record": {"CustomerID": "1", "clientName": "Gitter Windows Desktop App"}} +{"type": "RECORD", "stream": "long_string", "record": {"CustomerID": "2", "clientName": "Gitter iOS App"}} From ac65ff8a9ac61d398c993ddf1320ae12cd19283e Mon Sep 17 00:00:00 2001 From: Henning Date: Thu, 19 Jan 2023 12:01:55 +0100 Subject: [PATCH 12/12] back reformatting --- target_mssql/connector.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/target_mssql/connector.py b/target_mssql/connector.py index 8328c45..578e2cf 100644 --- a/target_mssql/connector.py +++ b/target_mssql/connector.py @@ -346,9 +346,7 @@ def to_sql_type(self, jsonschema_type: dict) -> sqlalchemy.types.TypeEngine: # maxlength = jsonschema_type.get("maxLength") if maxlength is not None: if maxlength > 8000: - return cast( - sqlalchemy.types.TypeEngine, sqlalchemy.types.TEXT() - ) + return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.TEXT()) return cast( sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(maxlength)