From 3de7325924be659f0f03800f589fc05c97beaf83 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Mon, 8 Jan 2024 17:12:31 -0700 Subject: [PATCH 01/24] Revert "Upgraded to dbt-core 1.4. (#146)" This reverts commit 3db05eb15d351d050caa52e50a589b1b82bcd6e2. --- CHANGELOG.md | 2 -- dbt/adapters/mariadb/__version__.py | 2 +- dbt/adapters/mariadb/connections.py | 21 +++++++++------------ dbt/adapters/mariadb/impl.py | 24 ++++++++++-------------- dbt/adapters/mariadb/relation.py | 14 ++++++-------- dbt/adapters/mysql/__version__.py | 2 +- dbt/adapters/mysql/connections.py | 17 +++++++---------- dbt/adapters/mysql/impl.py | 24 ++++++++++-------------- dbt/adapters/mysql/relation.py | 14 ++++++-------- dbt/adapters/mysql5/__version__.py | 2 +- dbt/adapters/mysql5/connections.py | 17 +++++++---------- dbt/adapters/mysql5/impl.py | 24 ++++++++++-------------- dbt/adapters/mysql5/relation.py | 14 ++++++-------- 13 files changed, 74 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 824421a..85d01e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,12 @@ ## Unreleased (TBD) ### Features -- Support dbt v1.4 ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) - Migrate CircleCI to GitHub Actions ([#120](https://github.com/dbeatty10/dbt-mysql/issues/120)) ### Fixes - Fix incremental composite keys ([#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) ### Contributors -- [@lpezet](https://github.com/lpezet) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146), [#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) - [@moszutij](https://github.com/moszutij) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146), [#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) - [@wesen](https://github.com/wesen) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) - [@mwallace582](https://github.com/mwallace582) ([#162](https://github.com/dbeatty10/dbt-mysql/pull/162)) diff --git a/dbt/adapters/mariadb/__version__.py b/dbt/adapters/mariadb/__version__.py index 70ba273..a6b9772 100644 --- a/dbt/adapters/mariadb/__version__.py +++ b/dbt/adapters/mariadb/__version__.py @@ -1 +1 @@ -version = "1.4.0a1" +version = "1.2.0a1" diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index cd50ea9..d85ec29 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -40,11 +40,11 @@ def __init__(self, **kwargs): def __post_init__(self): # Database and schema are treated as the same thing if self.database is not None and self.database != self.schema: - raise dbt.exceptions.DbtRuntimeError( + raise dbt.exceptions.RuntimeException( f" schema: {self.schema} \n" f" database: {self.database} \n" - f"On MariaDB, database must be omitted" - f" or have the same value as schema." + f"On MariaDB, database must be omitted or have the same value as" + f" schema." ) @property @@ -117,7 +117,7 @@ def open(cls, connection): connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectError(str(e)) + raise dbt.exceptions.FailedToConnectException(str(e)) return connection @@ -142,19 +142,19 @@ def exception_handler(self, sql): logger.debug("Failed to release connection!") pass - raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e + raise dbt.exceptions.DatabaseException(str(e).strip()) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() - if isinstance(e, dbt.exceptions.DbtRuntimeError): + if isinstance(e, dbt.exceptions.RuntimeException): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise - raise dbt.exceptions.DbtRuntimeError(e) from e + raise dbt.exceptions.RuntimeException(e) from e @classmethod def get_response(cls, cursor) -> AdapterResponse: @@ -164,11 +164,8 @@ def get_response(cls, cursor) -> AdapterResponse: if cursor is not None and cursor.rowcount is not None: num_rows = cursor.rowcount - # There's no real way to get the status from - # the mysql-connector-python driver. + # There's no real way to get the status from the mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), - rows_affected=num_rows, - code=code + _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code ) diff --git a/dbt/adapters/mariadb/impl.py b/dbt/adapters/mariadb/impl.py index 2557f36..6a16bcb 100644 --- a/dbt/adapters/mariadb/impl.py +++ b/dbt/adapters/mariadb/impl.py @@ -32,8 +32,7 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, - col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -44,9 +43,8 @@ def list_relations_without_caching( ) -> List[MariaDBRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, - kwargs=kwargs) - except dbt.exceptions.DbtRuntimeError as e: + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) + except dbt.exceptions.RuntimeException as e: errmsg = getattr(e, "msg", "") if f"MariaDB database '{schema_relation}' not found" in errmsg: return [] @@ -58,7 +56,7 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.DbtRuntimeError( + raise dbt.exceptions.RuntimeException( "Invalid value from " f'"mariadb__list_relations_without_caching({kwargs})", ' f"got {len(row)} values, expected 4" @@ -71,8 +69,7 @@ def list_relations_without_caching( return relations - def get_columns_in_relation(self, - relation: Relation) -> List[MariaDBColumn]: + def get_columns_in_relation(self, relation: Relation) -> List[MariaDBColumn]: rows: List[agate.Row] = super().get_columns_in_relation(relation) return self.parse_show_columns(relation, rows) @@ -92,7 +89,7 @@ def _get_columns_for_catalog( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: - if not self.Relation.get_default_include_policy().database: + if not self.Relation.include_policy.database: database = None return super().get_relation(database, schema, identifier) @@ -118,7 +115,7 @@ def parse_show_columns( def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: - raise dbt.exceptions.CompilationError( + dbt.exceptions.raise_compiler_error( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) @@ -147,7 +144,7 @@ def _get_one_catalog( manifest, ) -> agate.Table: if len(schemas) != 1: - raise dbt.exceptions.CompilationError( + dbt.exceptions.raise_compiler_error( f"Expected only one schema in mariadb _get_one_catalog, found " f"{schemas}" ) @@ -159,8 +156,7 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) - return agate.Table.from_object(columns, - column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro( @@ -203,7 +199,7 @@ def string_add_sql( elif location == "prepend": return f"concat({value}, '{add_to}')" else: - raise dbt.exceptions.DbtRuntimeError( + raise dbt.exceptions.RuntimeException( f'Got an unexpected location value of "{location}"' ) diff --git a/dbt/adapters/mariadb/relation.py b/dbt/adapters/mariadb/relation.py index fd1e883..0b21aa0 100644 --- a/dbt/adapters/mariadb/relation.py +++ b/dbt/adapters/mariadb/relation.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import DbtRuntimeError +from dbt.exceptions import RuntimeException @dataclass @@ -20,15 +20,13 @@ class MariaDBIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MariaDBRelation(BaseRelation): - quote_policy: MariaDBQuotePolicy = field( - default_factory=lambda: MariaDBQuotePolicy()) - include_policy: MariaDBIncludePolicy = field( - default_factory=lambda: MariaDBIncludePolicy()) + quote_policy: MariaDBQuotePolicy = MariaDBQuotePolicy() + include_policy: MariaDBIncludePolicy = MariaDBIncludePolicy() quote_character: str = "`" def __post_init__(self): if self.database != self.schema and self.database: - raise DbtRuntimeError( + raise RuntimeException( f"Cannot set `database` to '{self.database}' in MariaDB!" "You can either unset `database`, or make it match `schema`, " f"currently set to '{self.schema}'" @@ -36,7 +34,7 @@ def __post_init__(self): def render(self): if self.include_policy.database and self.include_policy.schema: - raise DbtRuntimeError( + raise RuntimeException( "Got a MariaDB relation with schema and database set to " "include, but only one can be set" ) diff --git a/dbt/adapters/mysql/__version__.py b/dbt/adapters/mysql/__version__.py index 70ba273..a6b9772 100644 --- a/dbt/adapters/mysql/__version__.py +++ b/dbt/adapters/mysql/__version__.py @@ -1 +1 @@ -version = "1.4.0a1" +version = "1.2.0a1" diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 42880f6..6a4e285 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -39,7 +39,7 @@ def __init__(self, **kwargs): def __post_init__(self): # mysql classifies database and schema as the same thing if self.database is not None and self.database != self.schema: - raise dbt.exceptions.DbtRuntimeError( + raise dbt.exceptions.RuntimeException( f" schema: {self.schema} \n" f" database: {self.database} \n" f"On MySQL, database must be omitted or have the same value as" @@ -113,7 +113,7 @@ def open(cls, connection): connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectError(str(e)) + raise dbt.exceptions.FailedToConnectException(str(e)) return connection @@ -138,19 +138,19 @@ def exception_handler(self, sql): logger.debug("Failed to release connection!") pass - raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e + raise dbt.exceptions.DatabaseException(str(e).strip()) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() - if isinstance(e, dbt.exceptions.DbtRuntimeError): + if isinstance(e, dbt.exceptions.RuntimeException): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise - raise dbt.exceptions.DbtRuntimeError(e) from e + raise dbt.exceptions.RuntimeException(e) from e @classmethod def get_response(cls, cursor) -> AdapterResponse: @@ -160,11 +160,8 @@ def get_response(cls, cursor) -> AdapterResponse: if cursor is not None and cursor.rowcount is not None: num_rows = cursor.rowcount - # There's no real way to get the status from the - # mysql-connector-python driver. + # There's no real way to get the status from the mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), - rows_affected=num_rows, - code=code + _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code ) diff --git a/dbt/adapters/mysql/impl.py b/dbt/adapters/mysql/impl.py index 7e449ef..f1e11d1 100644 --- a/dbt/adapters/mysql/impl.py +++ b/dbt/adapters/mysql/impl.py @@ -32,8 +32,7 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, - col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -44,9 +43,8 @@ def list_relations_without_caching( ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, - kwargs=kwargs) - except dbt.exceptions.DbtRuntimeError as e: + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) + except dbt.exceptions.RuntimeException as e: errmsg = getattr(e, "msg", "") if f"MySQL database '{schema_relation}' not found" in errmsg: return [] @@ -58,7 +56,7 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.DbtRuntimeError( + raise dbt.exceptions.RuntimeException( "Invalid value from " f'"mysql__list_relations_without_caching({kwargs})", ' f"got {len(row)} values, expected 4" @@ -91,7 +89,7 @@ def _get_columns_for_catalog( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: - if not self.Relation.get_default_include_policy().database: + if not self.Relation.include_policy.database: database = None return super().get_relation(database, schema, identifier) @@ -118,7 +116,7 @@ def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: - raise dbt.exceptions.CompilationError( + dbt.exceptions.raise_compiler_error( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) @@ -147,7 +145,7 @@ def _get_one_catalog( manifest, ) -> agate.Table: if len(schemas) != 1: - raise dbt.exceptions.CompilationError( + dbt.exceptions.raise_compiler_error( f"Expected only one schema in mysql _get_one_catalog, found " f"{schemas}" ) @@ -159,8 +157,7 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) - return agate.Table.from_object(columns, - column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro( @@ -203,7 +200,7 @@ def string_add_sql( elif location == "prepend": return f"concat({value}, '{add_to}')" else: - raise dbt.exceptions.DbtRuntimeError( + raise dbt.exceptions.RuntimeException( f'Got an unexpected location value of "{location}"' ) @@ -230,8 +227,7 @@ def get_rows_different_sql( ) first_column = names[0] - # MySQL doesn't have an EXCEPT or MINUS operator, - # so we need to simulate it + # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it COLUMNS_EQUAL_SQL = """ WITH a_except_b as ( diff --git a/dbt/adapters/mysql/relation.py b/dbt/adapters/mysql/relation.py index 5170dbd..859afc1 100644 --- a/dbt/adapters/mysql/relation.py +++ b/dbt/adapters/mysql/relation.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import DbtRuntimeError +from dbt.exceptions import RuntimeException @dataclass @@ -20,15 +20,13 @@ class MySQLIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MySQLRelation(BaseRelation): - quote_policy: MySQLQuotePolicy = field( - default_factory=lambda: MySQLQuotePolicy()) - include_policy: MySQLIncludePolicy = field( - default_factory=lambda: MySQLIncludePolicy()) + quote_policy: MySQLQuotePolicy = MySQLQuotePolicy() + include_policy: MySQLIncludePolicy = MySQLIncludePolicy() quote_character: str = "`" def __post_init__(self): if self.database != self.schema and self.database: - raise DbtRuntimeError( + raise RuntimeException( f"Cannot set `database` to '{self.database}' in mysql!" "You can either unset `database`, or make it match `schema`, " f"currently set to '{self.schema}'" @@ -36,7 +34,7 @@ def __post_init__(self): def render(self): if self.include_policy.database and self.include_policy.schema: - raise DbtRuntimeError( + raise RuntimeException( "Got a mysql relation with schema and database set to " "include, but only one can be set" ) diff --git a/dbt/adapters/mysql5/__version__.py b/dbt/adapters/mysql5/__version__.py index 70ba273..a6b9772 100644 --- a/dbt/adapters/mysql5/__version__.py +++ b/dbt/adapters/mysql5/__version__.py @@ -1 +1 @@ -version = "1.4.0a1" +version = "1.2.0a1" diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index 6199ff5..c8c1d20 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -40,7 +40,7 @@ def __init__(self, **kwargs): def __post_init__(self): # mysql classifies database and schema as the same thing if self.database is not None and self.database != self.schema: - raise dbt.exceptions.DbtRuntimeError( + raise dbt.exceptions.RuntimeException( f" schema: {self.schema} \n" f" database: {self.database} \n" f"On MySQL, database must be omitted or have the same value as" @@ -117,7 +117,7 @@ def open(cls, connection): connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectError(str(e)) + raise dbt.exceptions.FailedToConnectException(str(e)) return connection @@ -142,19 +142,19 @@ def exception_handler(self, sql): logger.debug("Failed to release connection!") pass - raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e + raise dbt.exceptions.DatabaseException(str(e).strip()) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() - if isinstance(e, dbt.exceptions.DbtRuntimeError): + if isinstance(e, dbt.exceptions.RuntimeException): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise - raise dbt.exceptions.DbtRuntimeError(e) from e + raise dbt.exceptions.RuntimeException(e) from e @classmethod def get_response(cls, cursor) -> AdapterResponse: @@ -164,11 +164,8 @@ def get_response(cls, cursor) -> AdapterResponse: if cursor is not None and cursor.rowcount is not None: num_rows = cursor.rowcount - # There's no real way to get the status from - # the mysql-connector-python driver. + # There's no real way to get the status from the mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), - rows_affected=num_rows, - code=code + _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code ) diff --git a/dbt/adapters/mysql5/impl.py b/dbt/adapters/mysql5/impl.py index e0d61a3..2582c83 100644 --- a/dbt/adapters/mysql5/impl.py +++ b/dbt/adapters/mysql5/impl.py @@ -32,8 +32,7 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, - col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -44,9 +43,8 @@ def list_relations_without_caching( ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, - kwargs=kwargs) - except dbt.exceptions.DbtRuntimeError as e: + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) + except dbt.exceptions.RuntimeException as e: errmsg = getattr(e, "msg", "") if f"MySQL database '{schema_relation}' not found" in errmsg: return [] @@ -58,7 +56,7 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.DbtRuntimeError( + raise dbt.exceptions.RuntimeException( "Invalid value from " f'"mysql5__list_relations_without_caching({kwargs})", ' f"got {len(row)} values, expected 4" @@ -91,7 +89,7 @@ def _get_columns_for_catalog( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: - if not self.Relation.get_default_include_policy().database: + if not self.Relation.include_policy.database: database = None return super().get_relation(database, schema, identifier) @@ -117,7 +115,7 @@ def parse_show_columns( def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: - raise dbt.exceptions.CompilationError( + dbt.exceptions.raise_compiler_error( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) @@ -146,7 +144,7 @@ def _get_one_catalog( manifest, ) -> agate.Table: if len(schemas) != 1: - raise dbt.exceptions.CompilationError( + dbt.exceptions.raise_compiler_error( f"Expected only one schema in mysql5 _get_one_catalog, found " f"{schemas}" ) @@ -158,8 +156,7 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) - return agate.Table.from_object(columns, - column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro( @@ -202,7 +199,7 @@ def string_add_sql( elif location == "prepend": return f"concat({value}, '{add_to}')" else: - raise dbt.exceptions.DbtRuntimeError( + raise dbt.exceptions.RuntimeException( f'Got an unexpected location value of "{location}"' ) @@ -229,8 +226,7 @@ def get_rows_different_sql( ) first_column = names[0] - # MySQL doesn't have an EXCEPT or MINUS operator, - # so we need to simulate it + # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it COLUMNS_EQUAL_SQL = """ SELECT row_count_diff.difference as row_count_difference, diff --git a/dbt/adapters/mysql5/relation.py b/dbt/adapters/mysql5/relation.py index c7a150e..1b03317 100644 --- a/dbt/adapters/mysql5/relation.py +++ b/dbt/adapters/mysql5/relation.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import DbtRuntimeError +from dbt.exceptions import RuntimeException @dataclass @@ -20,15 +20,13 @@ class MySQLIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MySQLRelation(BaseRelation): - quote_policy: MySQLQuotePolicy = field( - default_factory=lambda: MySQLQuotePolicy()) - include_policy: MySQLIncludePolicy = field( - default_factory=lambda: MySQLIncludePolicy()) + quote_policy: MySQLQuotePolicy = MySQLQuotePolicy() + include_policy: MySQLIncludePolicy = MySQLIncludePolicy() quote_character: str = "`" def __post_init__(self): if self.database != self.schema and self.database: - raise DbtRuntimeError( + raise RuntimeException( f"Cannot set `database` to '{self.database}' in mysql5!" "You can either unset `database`, or make it match `schema`, " f"currently set to '{self.schema}'" @@ -36,7 +34,7 @@ def __post_init__(self): def render(self): if self.include_policy.database and self.include_policy.schema: - raise DbtRuntimeError( + raise RuntimeException( "Got a mysql5 relation with schema and database set to " "include, but only one can be set" ) From 6a93e441977fdf05e813f69ddf446077208855de Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Mon, 8 Jan 2024 17:48:23 -0700 Subject: [PATCH 02/24] Fix version bumping * Migration bump2version (unmaintained) to bump-my-version * Import `bumpversion.cfg` from `dbt-snowflake` * Convert `bumpversion.cfg` to `bumpversion.toml` * Fix a few incorrect version labels --- .bumpversion.cfg | 35 --------------------------- .bumpversion.toml | 56 ++++++++++++++++++++++++++++++++++++++++++++ RELEASE.md | 10 ++++---- dev-requirements.txt | 4 ++-- setup.py | 2 +- 5 files changed, 64 insertions(+), 43 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .bumpversion.toml diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 3798f0a..0000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[bumpversion] -current_version = 1.2.0a1 -commit = False -tag = False -parse = (?P\d+) - \.(?P\d+) - \.(?P\d+) - ((?P[a-z]+) - ?(\.)? - (?P\d+))? -serialize = - {major}.{minor}.{patch}{prerelease}{num} - {major}.{minor}.{patch} - -[bumpversion:part:prerelease] -first_value = a -values = - a - b - rc - -[bumpversion:part:num] -first_value = 1 - -[bumpversion:file:dbt/adapters/mysql/__version__.py] -search = version = "{current_version}" -replace = version = "{new_version}" - -[bumpversion:file:dbt/adapters/mysql5/__version__.py] -search = version = "{current_version}" -replace = version = "{new_version}" - -[bumpversion:file:dbt/adapters/mariadb/__version__.py] -search = version = "{current_version}" -replace = version = "{new_version}" diff --git a/.bumpversion.toml b/.bumpversion.toml new file mode 100644 index 0000000..e09d8c3 --- /dev/null +++ b/.bumpversion.toml @@ -0,0 +1,56 @@ +[tool.bumpversion] +current_version = "1.2.0a1" +parse = ''' + (?P[\d]+) # major version number + \.(?P[\d]+) # minor version number + \.(?P[\d]+) # patch version number + (?P # optional pre-release - ex: a1, b2, rc25 + (?Pa|b|rc) # pre-release type + (?P[\d]+) # pre-release version number + )? + ( # optional nightly release indicator + \.(?Pdev[0-9]+) # ex: .dev02142023 + )? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0` +''' +serialize = [ + "{major}.{minor}.{patch}{prekind}{num}.{nightly}", + "{major}.{minor}.{patch}.{nightly}", + "{major}.{minor}.{patch}{prekind}{num}", + "{major}.{minor}.{patch}", +] +commit = false +tag = false + +[tool.bumpversion.parts.prekind] +first_value = "a" +optional_value = "final" +values = [ + "a", + "b", + "rc", + "final", +] + +[tool.bumpversion.parts.num] +first_value = "1" + +[[tool.bumpversion.files]] +filename = "setup.py" + +[[tool.bumpversion.files]] +filename = "dbt/adapters/mysql/__version__.py" + +[[tool.bumpversion.files]] +filename = "dbt/adapters/mysql5/__version__.py" + +[[tool.bumpversion.files]] +filename = "dbt/adapters/mariadb/__version__.py" + +[[tool.bumpversion.files]] +filename = "dev-requirements.txt" +parse = ''' + (?P[\d]+) # major version number + \.(?P[\d]+) # minor version number + .latest +''' +serialize = ["{major}.{minor}.latest"] diff --git a/RELEASE.md b/RELEASE.md index c423720..7500413 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,15 +10,15 @@ 1. Open a branch for the release - `git checkout -b releases/1.5.0` 1. Update [`CHANGELOG.md`](CHANGELOG.md) with the most recent changes -1. Bump the version using [`bump2version`](https://github.com/c4urself/bump2version/#bump2version): - 1. Dry run first by running `bumpversion --dry-run --verbose --new-version `. Some examples: +1. Bump the version using [`bump-my-version`](https://github.com/callowayproject/bump-my-version): + 1. Dry run first by running `bump-my-version bump --dry-run --verbose --new-version `. Some examples: - Release candidates: `--new-version 1.5.0rc1 num` - Alpha releases: `--new-version 1.5.0a1 num` - Patch releases: `--new-version 1.5.1 patch` - Minor releases: `--new-version 1.5.0 minor` - Major releases: `--new-version 2.0.0 major` - 1. Actually modify the files: `bumpversion --no-tag --new-version `. An example: - - Minor releases: `bumpversion --no-tag --new-version 1.5.0 minor` + 1. Actually modify the files: `bump-my-version bump --no-tag --new-version `. An example: + - Minor releases: `bump-my-version bump --no-tag --new-version 1.5.0 minor` 1. Check the diff with `git diff` 1. Add the files that were changed with `git add --update` 1. Commit with message `Release dbt-mysql v` @@ -60,7 +60,7 @@ PyPI recognizes [pre-release versioning conventions](https://packaging.python.or - `git pull` - `git checkout -b bump-1.6.0a1` - Minor releases: - `bumpversion --no-tag --new-version 1.6.0a1 num` + `bump-my-version bump --no-tag --new-version 1.6.0a1 num` - Update the branch names in `dev-requirements.txt` from `@{previous-version}.latest` to `@{minor-version}.latest` (or `@main`) - Commit with message `Bump dbt-mysql 1.6.0a1` - `git push` diff --git a/dev-requirements.txt b/dev-requirements.txt index bb4f4d8..75723b7 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -# git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@1.2.latest#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@1.2.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor diff --git a/setup.py b/setup.py index 7c67449..c176ec4 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ def _get_dbt_core_version(): package_name = "dbt-mysql" -package_version = "1.4.0a1" +package_version = "1.2.0a1" dbt_core_version = _get_dbt_core_version() description = """The MySQL adapter plugin for dbt""" From 43fd634729dea431022f208d05b4ec8bd70e9016 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 9 Jan 2024 08:59:04 -0700 Subject: [PATCH 03/24] Remove macos and windows as integration targets --- .github/scripts/integration-test-matrix.js | 31 ---------------------- 1 file changed, 31 deletions(-) diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 5f6d93f..2a0e3ca 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -39,24 +39,6 @@ module.exports = ({ context }) => { "database-image": adapterImages[adapter], "python-version": pythonVersion, }); - - if (labels.includes("test windows") || testAllLabel) { - include.push({ - os: "windows-latest", - adapter, - "database-image": adapterImages[adapter], - "python-version": pythonVersion, - }); - } - - if (labels.includes("test macos") || testAllLabel) { - include.push({ - os: "macos-latest", - adapter, - "database-image": adapterImages[adapter], - "python-version": pythonVersion, - }); - } } } } @@ -84,19 +66,6 @@ module.exports = ({ context }) => { } } - // additionally include runs for all adapters, on macos and windows, - // but only for the default python version - for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-latest"]) { - include.push({ - os: operatingSystem, - adapter: adapter, - "database-image": adapterImages[adapter], - "python-version": defaultPythonVersion, - }); - } - } - console.log("matrix", { include }); return { From d43b233c9c79fd03ebff4c80051d9e637e35820c Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 9 Jan 2024 08:59:20 -0700 Subject: [PATCH 04/24] Fix requirements conflict between dbt 1.2 and tox --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 75723b7..b52651f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -19,6 +19,6 @@ pytest-dotenv~=0.5.2 pytest-logbook~=1.2 pytest-xdist~=3.5 pytz~=2023.3 -tox~=4.11 +tox~=3.28 twine~=4.0 wheel~=0.42 From 1be9405cb4117aa63ce9ab24f98820fa0b3f902e Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 9 Jan 2024 09:07:56 -0700 Subject: [PATCH 05/24] Bump version to 1.3 --- .bumpversion.toml | 2 +- dbt/adapters/mariadb/__version__.py | 2 +- dbt/adapters/mysql/__version__.py | 2 +- dbt/adapters/mysql5/__version__.py | 2 +- dev-requirements.txt | 4 ++-- setup.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index e09d8c3..69fa56f 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "1.2.0a1" +current_version = "1.3.0a1" parse = ''' (?P[\d]+) # major version number \.(?P[\d]+) # minor version number diff --git a/dbt/adapters/mariadb/__version__.py b/dbt/adapters/mariadb/__version__.py index a6b9772..a9fe3c3 100644 --- a/dbt/adapters/mariadb/__version__.py +++ b/dbt/adapters/mariadb/__version__.py @@ -1 +1 @@ -version = "1.2.0a1" +version = "1.3.0a1" diff --git a/dbt/adapters/mysql/__version__.py b/dbt/adapters/mysql/__version__.py index a6b9772..a9fe3c3 100644 --- a/dbt/adapters/mysql/__version__.py +++ b/dbt/adapters/mysql/__version__.py @@ -1 +1 @@ -version = "1.2.0a1" +version = "1.3.0a1" diff --git a/dbt/adapters/mysql5/__version__.py b/dbt/adapters/mysql5/__version__.py index a6b9772..a9fe3c3 100644 --- a/dbt/adapters/mysql5/__version__.py +++ b/dbt/adapters/mysql5/__version__.py @@ -1 +1 @@ -version = "1.2.0a1" +version = "1.3.0a1" diff --git a/dev-requirements.txt b/dev-requirements.txt index b52651f..75fdb4f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@1.2.latest#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@1.2.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@1.3.latest#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@1.3.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor diff --git a/setup.py b/setup.py index c176ec4..213ac9c 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ def _get_dbt_core_version(): package_name = "dbt-mysql" -package_version = "1.2.0a1" +package_version = "1.3.0a1" dbt_core_version = _get_dbt_core_version() description = """The MySQL adapter plugin for dbt""" From d4332d9b9679a46d51515eca8b8e0e7ce723e778 Mon Sep 17 00:00:00 2001 From: Luke Date: Sun, 11 Jun 2023 01:02:23 +0100 Subject: [PATCH 06/24] Upgraded to dbt-core 1.4. (#146) * Upgraded to dbt-core 1.4. * Updated CHANGELOG. * Fixed policy fields definitions for mariadb and mysql5. * Replaced deprecated dbt.exceptions.raise_compiler_error() with dbt.exceptions.CompilationError. * Now using dbt.exceptions.DbtDatabaseError insead of dbt.exceptions.DatabaseException. * Update version * Update changelog --------- Co-authored-by: Doug Beatty --- .bumpversion.toml | 2 +- CHANGELOG.md | 2 ++ dbt/adapters/mariadb/__version__.py | 2 +- dbt/adapters/mariadb/connections.py | 21 ++++++++++++--------- dbt/adapters/mariadb/impl.py | 24 ++++++++++++++---------- dbt/adapters/mariadb/relation.py | 14 ++++++++------ dbt/adapters/mysql/__version__.py | 2 +- dbt/adapters/mysql/connections.py | 17 ++++++++++------- dbt/adapters/mysql/impl.py | 24 ++++++++++++++---------- dbt/adapters/mysql/relation.py | 14 ++++++++------ dbt/adapters/mysql5/__version__.py | 2 +- dbt/adapters/mysql5/connections.py | 17 ++++++++++------- dbt/adapters/mysql5/impl.py | 24 ++++++++++++++---------- dbt/adapters/mysql5/relation.py | 14 ++++++++------ dev-requirements.txt | 6 +++--- setup.py | 2 +- 16 files changed, 108 insertions(+), 79 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 69fa56f..2a23ab6 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "1.3.0a1" +current_version = "1.4.0a1" parse = ''' (?P[\d]+) # major version number \.(?P[\d]+) # minor version number diff --git a/CHANGELOG.md b/CHANGELOG.md index 85d01e2..b462295 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,13 @@ ### Features - Migrate CircleCI to GitHub Actions ([#120](https://github.com/dbeatty10/dbt-mysql/issues/120)) +- Support dbt v1.4 ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) ### Fixes - Fix incremental composite keys ([#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) ### Contributors +- [@lpezet](https://github.com/lpezet) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) - [@moszutij](https://github.com/moszutij) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146), [#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) - [@wesen](https://github.com/wesen) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) - [@mwallace582](https://github.com/mwallace582) ([#162](https://github.com/dbeatty10/dbt-mysql/pull/162)) diff --git a/dbt/adapters/mariadb/__version__.py b/dbt/adapters/mariadb/__version__.py index a9fe3c3..70ba273 100644 --- a/dbt/adapters/mariadb/__version__.py +++ b/dbt/adapters/mariadb/__version__.py @@ -1 +1 @@ -version = "1.3.0a1" +version = "1.4.0a1" diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index d85ec29..cd50ea9 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -40,11 +40,11 @@ def __init__(self, **kwargs): def __post_init__(self): # Database and schema are treated as the same thing if self.database is not None and self.database != self.schema: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f" schema: {self.schema} \n" f" database: {self.database} \n" - f"On MariaDB, database must be omitted or have the same value as" - f" schema." + f"On MariaDB, database must be omitted" + f" or have the same value as schema." ) @property @@ -117,7 +117,7 @@ def open(cls, connection): connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectException(str(e)) + raise dbt.exceptions.FailedToConnectError(str(e)) return connection @@ -142,19 +142,19 @@ def exception_handler(self, sql): logger.debug("Failed to release connection!") pass - raise dbt.exceptions.DatabaseException(str(e).strip()) from e + raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() - if isinstance(e, dbt.exceptions.RuntimeException): + if isinstance(e, dbt.exceptions.DbtRuntimeError): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise - raise dbt.exceptions.RuntimeException(e) from e + raise dbt.exceptions.DbtRuntimeError(e) from e @classmethod def get_response(cls, cursor) -> AdapterResponse: @@ -164,8 +164,11 @@ def get_response(cls, cursor) -> AdapterResponse: if cursor is not None and cursor.rowcount is not None: num_rows = cursor.rowcount - # There's no real way to get the status from the mysql-connector-python driver. + # There's no real way to get the status from + # the mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code + _message="{} {}".format(code, num_rows), + rows_affected=num_rows, + code=code ) diff --git a/dbt/adapters/mariadb/impl.py b/dbt/adapters/mariadb/impl.py index 6a16bcb..2557f36 100644 --- a/dbt/adapters/mariadb/impl.py +++ b/dbt/adapters/mariadb/impl.py @@ -32,7 +32,8 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, + col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -43,8 +44,9 @@ def list_relations_without_caching( ) -> List[MariaDBRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) - except dbt.exceptions.RuntimeException as e: + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, + kwargs=kwargs) + except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") if f"MariaDB database '{schema_relation}' not found" in errmsg: return [] @@ -56,7 +58,7 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( "Invalid value from " f'"mariadb__list_relations_without_caching({kwargs})", ' f"got {len(row)} values, expected 4" @@ -69,7 +71,8 @@ def list_relations_without_caching( return relations - def get_columns_in_relation(self, relation: Relation) -> List[MariaDBColumn]: + def get_columns_in_relation(self, + relation: Relation) -> List[MariaDBColumn]: rows: List[agate.Row] = super().get_columns_in_relation(relation) return self.parse_show_columns(relation, rows) @@ -89,7 +92,7 @@ def _get_columns_for_catalog( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: - if not self.Relation.include_policy.database: + if not self.Relation.get_default_include_policy().database: database = None return super().get_relation(database, schema, identifier) @@ -115,7 +118,7 @@ def parse_show_columns( def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) @@ -144,7 +147,7 @@ def _get_one_catalog( manifest, ) -> agate.Table: if len(schemas) != 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one schema in mariadb _get_one_catalog, found " f"{schemas}" ) @@ -156,7 +159,8 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) - return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, + column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro( @@ -199,7 +203,7 @@ def string_add_sql( elif location == "prepend": return f"concat({value}, '{add_to}')" else: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f'Got an unexpected location value of "{location}"' ) diff --git a/dbt/adapters/mariadb/relation.py b/dbt/adapters/mariadb/relation.py index 0b21aa0..fd1e883 100644 --- a/dbt/adapters/mariadb/relation.py +++ b/dbt/adapters/mariadb/relation.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import RuntimeException +from dbt.exceptions import DbtRuntimeError @dataclass @@ -20,13 +20,15 @@ class MariaDBIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MariaDBRelation(BaseRelation): - quote_policy: MariaDBQuotePolicy = MariaDBQuotePolicy() - include_policy: MariaDBIncludePolicy = MariaDBIncludePolicy() + quote_policy: MariaDBQuotePolicy = field( + default_factory=lambda: MariaDBQuotePolicy()) + include_policy: MariaDBIncludePolicy = field( + default_factory=lambda: MariaDBIncludePolicy()) quote_character: str = "`" def __post_init__(self): if self.database != self.schema and self.database: - raise RuntimeException( + raise DbtRuntimeError( f"Cannot set `database` to '{self.database}' in MariaDB!" "You can either unset `database`, or make it match `schema`, " f"currently set to '{self.schema}'" @@ -34,7 +36,7 @@ def __post_init__(self): def render(self): if self.include_policy.database and self.include_policy.schema: - raise RuntimeException( + raise DbtRuntimeError( "Got a MariaDB relation with schema and database set to " "include, but only one can be set" ) diff --git a/dbt/adapters/mysql/__version__.py b/dbt/adapters/mysql/__version__.py index a9fe3c3..70ba273 100644 --- a/dbt/adapters/mysql/__version__.py +++ b/dbt/adapters/mysql/__version__.py @@ -1 +1 @@ -version = "1.3.0a1" +version = "1.4.0a1" diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 6a4e285..42880f6 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -39,7 +39,7 @@ def __init__(self, **kwargs): def __post_init__(self): # mysql classifies database and schema as the same thing if self.database is not None and self.database != self.schema: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f" schema: {self.schema} \n" f" database: {self.database} \n" f"On MySQL, database must be omitted or have the same value as" @@ -113,7 +113,7 @@ def open(cls, connection): connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectException(str(e)) + raise dbt.exceptions.FailedToConnectError(str(e)) return connection @@ -138,19 +138,19 @@ def exception_handler(self, sql): logger.debug("Failed to release connection!") pass - raise dbt.exceptions.DatabaseException(str(e).strip()) from e + raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() - if isinstance(e, dbt.exceptions.RuntimeException): + if isinstance(e, dbt.exceptions.DbtRuntimeError): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise - raise dbt.exceptions.RuntimeException(e) from e + raise dbt.exceptions.DbtRuntimeError(e) from e @classmethod def get_response(cls, cursor) -> AdapterResponse: @@ -160,8 +160,11 @@ def get_response(cls, cursor) -> AdapterResponse: if cursor is not None and cursor.rowcount is not None: num_rows = cursor.rowcount - # There's no real way to get the status from the mysql-connector-python driver. + # There's no real way to get the status from the + # mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code + _message="{} {}".format(code, num_rows), + rows_affected=num_rows, + code=code ) diff --git a/dbt/adapters/mysql/impl.py b/dbt/adapters/mysql/impl.py index f1e11d1..7e449ef 100644 --- a/dbt/adapters/mysql/impl.py +++ b/dbt/adapters/mysql/impl.py @@ -32,7 +32,8 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, + col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -43,8 +44,9 @@ def list_relations_without_caching( ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) - except dbt.exceptions.RuntimeException as e: + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, + kwargs=kwargs) + except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") if f"MySQL database '{schema_relation}' not found" in errmsg: return [] @@ -56,7 +58,7 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( "Invalid value from " f'"mysql__list_relations_without_caching({kwargs})", ' f"got {len(row)} values, expected 4" @@ -89,7 +91,7 @@ def _get_columns_for_catalog( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: - if not self.Relation.include_policy.database: + if not self.Relation.get_default_include_policy().database: database = None return super().get_relation(database, schema, identifier) @@ -116,7 +118,7 @@ def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) @@ -145,7 +147,7 @@ def _get_one_catalog( manifest, ) -> agate.Table: if len(schemas) != 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one schema in mysql _get_one_catalog, found " f"{schemas}" ) @@ -157,7 +159,8 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) - return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, + column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro( @@ -200,7 +203,7 @@ def string_add_sql( elif location == "prepend": return f"concat({value}, '{add_to}')" else: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f'Got an unexpected location value of "{location}"' ) @@ -227,7 +230,8 @@ def get_rows_different_sql( ) first_column = names[0] - # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it + # MySQL doesn't have an EXCEPT or MINUS operator, + # so we need to simulate it COLUMNS_EQUAL_SQL = """ WITH a_except_b as ( diff --git a/dbt/adapters/mysql/relation.py b/dbt/adapters/mysql/relation.py index 859afc1..5170dbd 100644 --- a/dbt/adapters/mysql/relation.py +++ b/dbt/adapters/mysql/relation.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import RuntimeException +from dbt.exceptions import DbtRuntimeError @dataclass @@ -20,13 +20,15 @@ class MySQLIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MySQLRelation(BaseRelation): - quote_policy: MySQLQuotePolicy = MySQLQuotePolicy() - include_policy: MySQLIncludePolicy = MySQLIncludePolicy() + quote_policy: MySQLQuotePolicy = field( + default_factory=lambda: MySQLQuotePolicy()) + include_policy: MySQLIncludePolicy = field( + default_factory=lambda: MySQLIncludePolicy()) quote_character: str = "`" def __post_init__(self): if self.database != self.schema and self.database: - raise RuntimeException( + raise DbtRuntimeError( f"Cannot set `database` to '{self.database}' in mysql!" "You can either unset `database`, or make it match `schema`, " f"currently set to '{self.schema}'" @@ -34,7 +36,7 @@ def __post_init__(self): def render(self): if self.include_policy.database and self.include_policy.schema: - raise RuntimeException( + raise DbtRuntimeError( "Got a mysql relation with schema and database set to " "include, but only one can be set" ) diff --git a/dbt/adapters/mysql5/__version__.py b/dbt/adapters/mysql5/__version__.py index a9fe3c3..70ba273 100644 --- a/dbt/adapters/mysql5/__version__.py +++ b/dbt/adapters/mysql5/__version__.py @@ -1 +1 @@ -version = "1.3.0a1" +version = "1.4.0a1" diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index c8c1d20..6199ff5 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -40,7 +40,7 @@ def __init__(self, **kwargs): def __post_init__(self): # mysql classifies database and schema as the same thing if self.database is not None and self.database != self.schema: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f" schema: {self.schema} \n" f" database: {self.database} \n" f"On MySQL, database must be omitted or have the same value as" @@ -117,7 +117,7 @@ def open(cls, connection): connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectException(str(e)) + raise dbt.exceptions.FailedToConnectError(str(e)) return connection @@ -142,19 +142,19 @@ def exception_handler(self, sql): logger.debug("Failed to release connection!") pass - raise dbt.exceptions.DatabaseException(str(e).strip()) from e + raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() - if isinstance(e, dbt.exceptions.RuntimeException): + if isinstance(e, dbt.exceptions.DbtRuntimeError): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise - raise dbt.exceptions.RuntimeException(e) from e + raise dbt.exceptions.DbtRuntimeError(e) from e @classmethod def get_response(cls, cursor) -> AdapterResponse: @@ -164,8 +164,11 @@ def get_response(cls, cursor) -> AdapterResponse: if cursor is not None and cursor.rowcount is not None: num_rows = cursor.rowcount - # There's no real way to get the status from the mysql-connector-python driver. + # There's no real way to get the status from + # the mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code + _message="{} {}".format(code, num_rows), + rows_affected=num_rows, + code=code ) diff --git a/dbt/adapters/mysql5/impl.py b/dbt/adapters/mysql5/impl.py index 2582c83..e0d61a3 100644 --- a/dbt/adapters/mysql5/impl.py +++ b/dbt/adapters/mysql5/impl.py @@ -32,7 +32,8 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, + col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -43,8 +44,9 @@ def list_relations_without_caching( ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) - except dbt.exceptions.RuntimeException as e: + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, + kwargs=kwargs) + except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") if f"MySQL database '{schema_relation}' not found" in errmsg: return [] @@ -56,7 +58,7 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( "Invalid value from " f'"mysql5__list_relations_without_caching({kwargs})", ' f"got {len(row)} values, expected 4" @@ -89,7 +91,7 @@ def _get_columns_for_catalog( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: - if not self.Relation.include_policy.database: + if not self.Relation.get_default_include_policy().database: database = None return super().get_relation(database, schema, identifier) @@ -115,7 +117,7 @@ def parse_show_columns( def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) @@ -144,7 +146,7 @@ def _get_one_catalog( manifest, ) -> agate.Table: if len(schemas) != 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one schema in mysql5 _get_one_catalog, found " f"{schemas}" ) @@ -156,7 +158,8 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) - return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, + column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro( @@ -199,7 +202,7 @@ def string_add_sql( elif location == "prepend": return f"concat({value}, '{add_to}')" else: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f'Got an unexpected location value of "{location}"' ) @@ -226,7 +229,8 @@ def get_rows_different_sql( ) first_column = names[0] - # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it + # MySQL doesn't have an EXCEPT or MINUS operator, + # so we need to simulate it COLUMNS_EQUAL_SQL = """ SELECT row_count_diff.difference as row_count_difference, diff --git a/dbt/adapters/mysql5/relation.py b/dbt/adapters/mysql5/relation.py index 1b03317..c7a150e 100644 --- a/dbt/adapters/mysql5/relation.py +++ b/dbt/adapters/mysql5/relation.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import RuntimeException +from dbt.exceptions import DbtRuntimeError @dataclass @@ -20,13 +20,15 @@ class MySQLIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MySQLRelation(BaseRelation): - quote_policy: MySQLQuotePolicy = MySQLQuotePolicy() - include_policy: MySQLIncludePolicy = MySQLIncludePolicy() + quote_policy: MySQLQuotePolicy = field( + default_factory=lambda: MySQLQuotePolicy()) + include_policy: MySQLIncludePolicy = field( + default_factory=lambda: MySQLIncludePolicy()) quote_character: str = "`" def __post_init__(self): if self.database != self.schema and self.database: - raise RuntimeException( + raise DbtRuntimeError( f"Cannot set `database` to '{self.database}' in mysql5!" "You can either unset `database`, or make it match `schema`, " f"currently set to '{self.schema}'" @@ -34,7 +36,7 @@ def __post_init__(self): def render(self): if self.include_policy.database and self.include_policy.schema: - raise RuntimeException( + raise DbtRuntimeError( "Got a mysql5 relation with schema and database set to " "include, but only one can be set" ) diff --git a/dev-requirements.txt b/dev-requirements.txt index 75fdb4f..9b92e9f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@1.3.latest#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@1.3.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor @@ -19,6 +19,6 @@ pytest-dotenv~=0.5.2 pytest-logbook~=1.2 pytest-xdist~=3.5 pytz~=2023.3 -tox~=3.28 +tox~=4.11 twine~=4.0 wheel~=0.42 diff --git a/setup.py b/setup.py index 213ac9c..7c67449 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ def _get_dbt_core_version(): package_name = "dbt-mysql" -package_version = "1.3.0a1" +package_version = "1.4.0a1" dbt_core_version = _get_dbt_core_version() description = """The MySQL adapter plugin for dbt""" From 732f2014c2c7c613284a4edcb40a3d781b5eb3dc Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 9 Jan 2024 09:19:05 -0700 Subject: [PATCH 07/24] Bump version to 1.5 --- .bumpversion.toml | 2 +- dbt/adapters/mariadb/__version__.py | 2 +- dbt/adapters/mysql/__version__.py | 2 +- dbt/adapters/mysql5/__version__.py | 2 +- dev-requirements.txt | 4 ++-- setup.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 2a23ab6..9e3c8eb 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "1.4.0a1" +current_version = "1.5.0a1" parse = ''' (?P[\d]+) # major version number \.(?P[\d]+) # minor version number diff --git a/dbt/adapters/mariadb/__version__.py b/dbt/adapters/mariadb/__version__.py index 70ba273..219c289 100644 --- a/dbt/adapters/mariadb/__version__.py +++ b/dbt/adapters/mariadb/__version__.py @@ -1 +1 @@ -version = "1.4.0a1" +version = "1.5.0a1" diff --git a/dbt/adapters/mysql/__version__.py b/dbt/adapters/mysql/__version__.py index 70ba273..219c289 100644 --- a/dbt/adapters/mysql/__version__.py +++ b/dbt/adapters/mysql/__version__.py @@ -1 +1 @@ -version = "1.4.0a1" +version = "1.5.0a1" diff --git a/dbt/adapters/mysql5/__version__.py b/dbt/adapters/mysql5/__version__.py index 70ba273..219c289 100644 --- a/dbt/adapters/mysql5/__version__.py +++ b/dbt/adapters/mysql5/__version__.py @@ -1 +1 @@ -version = "1.4.0a1" +version = "1.5.0a1" diff --git a/dev-requirements.txt b/dev-requirements.txt index 9b92e9f..30b0f34 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@1.5.latest#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@1.5.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor diff --git a/setup.py b/setup.py index 7c67449..32fb208 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ def _get_dbt_core_version(): package_name = "dbt-mysql" -package_version = "1.4.0a1" +package_version = "1.5.0a1" dbt_core_version = _get_dbt_core_version() description = """The MySQL adapter plugin for dbt""" From 49c9e857ac3ca2af55d271c0837162ef766ebe9c Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Thu, 11 Jan 2024 16:42:01 -0700 Subject: [PATCH 08/24] Update dbt-core to 1.5 and implement support for model contracts (#163) Update dbt-core to 1.5.9 and implement support for constraints * Update changelog * Fix unit test error - The error was: `AttributeError: 'Namespace' object has no attribute 'MACRO_DEBUGGING'` * Allow Unix socket connection rather than just TCP (#165) --- CHANGELOG.md | 4 +- dbt/adapters/mariadb/column.py | 8 + dbt/adapters/mariadb/connections.py | 17 +- dbt/adapters/mariadb/impl.py | 20 +- dbt/adapters/mysql/column.py | 8 + dbt/adapters/mysql/connections.py | 17 +- dbt/adapters/mysql/impl.py | 20 +- dbt/adapters/mysql5/column.py | 8 + dbt/adapters/mysql5/connections.py | 17 +- dbt/adapters/mysql5/impl.py | 20 +- dbt/include/mariadb/macros/adapters.sql | 54 ++- dbt/include/mysql/macros/adapters.sql | 49 ++- dbt/include/mysql5/macros/adapters.sql | 47 ++- .../adapter/constraints/fixtures.py | 320 +++++++++++++++ .../adapter/constraints/test_constraints.py | 365 ++++++++++++++++++ tests/unit/test_adapter.py | 1 - tests/unit/utils.py | 9 +- 17 files changed, 963 insertions(+), 21 deletions(-) create mode 100644 tests/functional/adapter/constraints/fixtures.py create mode 100644 tests/functional/adapter/constraints/test_constraints.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b462295..ac88daf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ### Features - Migrate CircleCI to GitHub Actions ([#120](https://github.com/dbeatty10/dbt-mysql/issues/120)) - Support dbt v1.4 ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) +- Support dbt v1.5 ([#145](https://github.com/dbeatty10/dbt-mysql/issues/145)) +- Support connecting via UNIX sockets ([#164](https://github.com/dbeatty10/dbt-mysql/issues/164)) ### Fixes - Fix incremental composite keys ([#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) @@ -11,7 +13,7 @@ - [@lpezet](https://github.com/lpezet) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) - [@moszutij](https://github.com/moszutij) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146), [#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) - [@wesen](https://github.com/wesen) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) -- [@mwallace582](https://github.com/mwallace582) ([#162](https://github.com/dbeatty10/dbt-mysql/pull/162)) +- [@mwallace582](https://github.com/mwallace582) ([#162](https://github.com/dbeatty10/dbt-mysql/pull/162), [#163](https://github.com/dbeatty10/dbt-mysql/pull/163), [#164](https://github.com/dbeatty10/dbt-mysql/issues/164)) ## dbt-mysql 1.1.0 (Feb 5, 2023) diff --git a/dbt/adapters/mariadb/column.py b/dbt/adapters/mariadb/column.py index c7c47eb..adc941e 100644 --- a/dbt/adapters/mariadb/column.py +++ b/dbt/adapters/mariadb/column.py @@ -8,6 +8,14 @@ @dataclass class MariaDBColumn(Column): + TYPE_LABELS = { + "STRING": "TEXT", + "VAR_STRING": "TEXT", + "LONG": "INTEGER", + "LONGLONG": "INTEGER", + "INT": "INTEGER", + "TIMESTAMP": "DATETIME", + } table_database: Optional[str] = None table_schema: Optional[str] = None table_name: Optional[str] = None diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index cd50ea9..6b30f9a 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -1,6 +1,7 @@ from contextlib import contextmanager import mysql.connector +import mysql.connector.constants import dbt.exceptions from dbt.adapters.sql import SQLConnectionManager @@ -16,7 +17,8 @@ @dataclass(init=False) class MariaDBCredentials(Credentials): - server: str + server: Optional[str] = None + unix_socket: Optional[str] = None port: Optional[int] = None database: Optional[str] = None schema: str @@ -61,6 +63,7 @@ def _connection_keys(self): """ return ( "server", + "unix_socket", "port", "database", "schema", @@ -80,7 +83,6 @@ def open(cls, connection): credentials = cls.get_credentials(connection.credentials) kwargs = {} - kwargs["host"] = credentials.server kwargs["user"] = credentials.username kwargs["passwd"] = credentials.password kwargs["buffered"] = True @@ -88,6 +90,11 @@ def open(cls, connection): if credentials.ssl_disabled: kwargs["ssl_disabled"] = credentials.ssl_disabled + if credentials.server: + kwargs["host"] = credentials.server + elif credentials.unix_socket: + kwargs["unix_socket"] = credentials.unix_socket + if credentials.port: kwargs["port"] = credentials.port @@ -172,3 +179,9 @@ def get_response(cls, cursor) -> AdapterResponse: rows_affected=num_rows, code=code ) + + @classmethod + def data_type_code_to_name(cls, type_code: int) -> str: + field_type_values = mysql.connector.constants.FieldType.desc.values() + mapping = {code: name for (code, name) in field_type_values} + return mapping[type_code] diff --git a/dbt/adapters/mariadb/impl.py b/dbt/adapters/mariadb/impl.py index 2557f36..7289e7a 100644 --- a/dbt/adapters/mariadb/impl.py +++ b/dbt/adapters/mariadb/impl.py @@ -12,6 +12,8 @@ from dbt.adapters.mariadb import MariaDBRelation from dbt.adapters.mariadb import MariaDBColumn from dbt.adapters.base import BaseRelation +from dbt.contracts.graph.nodes import ConstraintType +from dbt.adapters.base.impl import ConstraintSupport from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER from dbt.events import AdapterLogger from dbt.utils import executor @@ -27,6 +29,19 @@ class MariaDBAdapter(SQLAdapter): Column = MariaDBColumn ConnectionManager = MariaDBConnectionManager + CONSTRAINT_SUPPORT = { + ConstraintType.check: ConstraintSupport.ENFORCED, + ConstraintType.not_null: ConstraintSupport.ENFORCED, + ConstraintType.unique: ConstraintSupport.ENFORCED, + ConstraintType.primary_key: ConstraintSupport.ENFORCED, + # While Foreign Keys are indeed supported, they're not supported in + # CREATE TABLE AS SELECT statements, which is what DBT uses. + # + # It is possible to use a `post-hook` to add a foreign key after the + # table is created. + ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, + } + @classmethod def date_function(cls): return "current_date()" @@ -36,7 +51,8 @@ def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" - def quote(self, identifier): + @classmethod + def quote(cls, identifier: str) -> str: return "`{}`".format(identifier) def list_relations_without_caching( @@ -157,7 +173,7 @@ def _get_one_catalog( columns: List[Dict[str, Any]] = [] for relation in self.list_relations(database, schema): - logger.debug("Getting table schema for relation {}", relation) + logger.debug("Getting table schema for relation {}", str(relation)) columns.extend(self._get_columns_for_catalog(relation)) return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) diff --git a/dbt/adapters/mysql/column.py b/dbt/adapters/mysql/column.py index 9ce3786..931593c 100644 --- a/dbt/adapters/mysql/column.py +++ b/dbt/adapters/mysql/column.py @@ -8,6 +8,14 @@ @dataclass class MySQLColumn(Column): + TYPE_LABELS = { + "STRING": "TEXT", + "VAR_STRING": "TEXT", + "LONG": "INTEGER", + "LONGLONG": "INTEGER", + "INT": "INTEGER", + "TIMESTAMP": "DATETIME", + } table_database: Optional[str] = None table_schema: Optional[str] = None table_name: Optional[str] = None diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 42880f6..d8932dd 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -1,6 +1,7 @@ from contextlib import contextmanager import mysql.connector +import mysql.connector.constants import dbt.exceptions from dbt.adapters.sql import SQLConnectionManager @@ -16,7 +17,8 @@ @dataclass(init=False) class MySQLCredentials(Credentials): - server: str + server: Optional[str] = None + unix_socket: Optional[str] = None port: Optional[int] = None database: Optional[str] = None schema: str @@ -60,6 +62,7 @@ def _connection_keys(self): """ return ( "server", + "unix_socket", "port", "database", "schema", @@ -79,11 +82,15 @@ def open(cls, connection): credentials = cls.get_credentials(connection.credentials) kwargs = {} - kwargs["host"] = credentials.server kwargs["user"] = credentials.username kwargs["passwd"] = credentials.password kwargs["buffered"] = True + if credentials.server: + kwargs["host"] = credentials.server + elif credentials.unix_socket: + kwargs["unix_socket"] = credentials.unix_socket + if credentials.port: kwargs["port"] = credentials.port @@ -168,3 +175,9 @@ def get_response(cls, cursor) -> AdapterResponse: rows_affected=num_rows, code=code ) + + @classmethod + def data_type_code_to_name(cls, type_code: int) -> str: + field_type_values = mysql.connector.constants.FieldType.desc.values() + mapping = {code: name for (code, name) in field_type_values} + return mapping[type_code] diff --git a/dbt/adapters/mysql/impl.py b/dbt/adapters/mysql/impl.py index 7e449ef..df76878 100644 --- a/dbt/adapters/mysql/impl.py +++ b/dbt/adapters/mysql/impl.py @@ -12,6 +12,8 @@ from dbt.adapters.mysql import MySQLRelation from dbt.adapters.mysql import MySQLColumn from dbt.adapters.base import BaseRelation +from dbt.contracts.graph.nodes import ConstraintType +from dbt.adapters.base.impl import ConstraintSupport from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER from dbt.events import AdapterLogger from dbt.utils import executor @@ -27,6 +29,19 @@ class MySQLAdapter(SQLAdapter): Column = MySQLColumn ConnectionManager = MySQLConnectionManager + CONSTRAINT_SUPPORT = { + ConstraintType.check: ConstraintSupport.ENFORCED, + ConstraintType.not_null: ConstraintSupport.ENFORCED, + ConstraintType.unique: ConstraintSupport.ENFORCED, + ConstraintType.primary_key: ConstraintSupport.ENFORCED, + # While Foreign Keys are indeed supported, they're not supported in + # CREATE TABLE AS SELECT statements, which is what DBT uses. + # + # It is possible to use a `post-hook` to add a foreign key after the + # table is created. + ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, + } + @classmethod def date_function(cls): return "current_date()" @@ -36,7 +51,8 @@ def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" - def quote(self, identifier): + @classmethod + def quote(cls, identifier: str) -> str: return "`{}`".format(identifier) def list_relations_without_caching( @@ -157,7 +173,7 @@ def _get_one_catalog( columns: List[Dict[str, Any]] = [] for relation in self.list_relations(database, schema): - logger.debug("Getting table schema for relation {}", relation) + logger.debug("Getting table schema for relation {}", str(relation)) columns.extend(self._get_columns_for_catalog(relation)) return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) diff --git a/dbt/adapters/mysql5/column.py b/dbt/adapters/mysql5/column.py index 9ce3786..931593c 100644 --- a/dbt/adapters/mysql5/column.py +++ b/dbt/adapters/mysql5/column.py @@ -8,6 +8,14 @@ @dataclass class MySQLColumn(Column): + TYPE_LABELS = { + "STRING": "TEXT", + "VAR_STRING": "TEXT", + "LONG": "INTEGER", + "LONGLONG": "INTEGER", + "INT": "INTEGER", + "TIMESTAMP": "DATETIME", + } table_database: Optional[str] = None table_schema: Optional[str] = None table_name: Optional[str] = None diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index 6199ff5..f1481a2 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -1,6 +1,7 @@ from contextlib import contextmanager import mysql.connector +import mysql.connector.constants import dbt.exceptions from dbt.adapters.sql import SQLConnectionManager @@ -16,7 +17,8 @@ @dataclass(init=False) class MySQLCredentials(Credentials): - server: str + server: Optional[str] = None + unix_socket: Optional[str] = None port: Optional[int] = None database: Optional[str] = None schema: str @@ -61,6 +63,7 @@ def _connection_keys(self): """ return ( "server", + "unix_socket", "port", "database", "schema", @@ -80,7 +83,6 @@ def open(cls, connection): credentials = cls.get_credentials(connection.credentials) kwargs = {} - kwargs["host"] = credentials.server kwargs["user"] = credentials.username kwargs["passwd"] = credentials.password kwargs["buffered"] = True @@ -88,6 +90,11 @@ def open(cls, connection): if credentials.ssl_disabled: kwargs["ssl_disabled"] = credentials.ssl_disabled + if credentials.server: + kwargs["host"] = credentials.server + elif credentials.unix_socket: + kwargs["unix_socket"] = credentials.unix_socket + if credentials.port: kwargs["port"] = credentials.port @@ -172,3 +179,9 @@ def get_response(cls, cursor) -> AdapterResponse: rows_affected=num_rows, code=code ) + + @classmethod + def data_type_code_to_name(cls, type_code: int) -> str: + field_type_values = mysql.connector.constants.FieldType.desc.values() + mapping = {code: name for (code, name) in field_type_values} + return mapping[type_code] diff --git a/dbt/adapters/mysql5/impl.py b/dbt/adapters/mysql5/impl.py index e0d61a3..c5f7aa4 100644 --- a/dbt/adapters/mysql5/impl.py +++ b/dbt/adapters/mysql5/impl.py @@ -12,6 +12,8 @@ from dbt.adapters.mysql5 import MySQLRelation from dbt.adapters.mysql5 import MySQLColumn from dbt.adapters.base import BaseRelation +from dbt.contracts.graph.nodes import ConstraintType +from dbt.adapters.base.impl import ConstraintSupport from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER from dbt.events import AdapterLogger from dbt.utils import executor @@ -27,6 +29,19 @@ class MySQLAdapter(SQLAdapter): Column = MySQLColumn ConnectionManager = MySQLConnectionManager + CONSTRAINT_SUPPORT = { + ConstraintType.check: ConstraintSupport.NOT_SUPPORTED, + ConstraintType.not_null: ConstraintSupport.ENFORCED, + ConstraintType.unique: ConstraintSupport.ENFORCED, + ConstraintType.primary_key: ConstraintSupport.ENFORCED, + # While Foreign Keys are indeed supported, they're not supported in + # CREATE TABLE AS SELECT statements, which is what DBT uses. + # + # It is possible to use a `post-hook` to add a foreign key after the + # table is created. + ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, + } + @classmethod def date_function(cls): return "current_date()" @@ -36,7 +51,8 @@ def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" - def quote(self, identifier): + @classmethod + def quote(cls, identifier: str) -> str: return "`{}`".format(identifier) def list_relations_without_caching( @@ -156,7 +172,7 @@ def _get_one_catalog( columns: List[Dict[str, Any]] = [] for relation in self.list_relations(database, schema): - logger.debug("Getting table schema for relation {}", relation) + logger.debug("Getting table schema for relation {}", str(relation)) columns.extend(self._get_columns_for_catalog(relation)) return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) diff --git a/dbt/include/mariadb/macros/adapters.sql b/dbt/include/mariadb/macros/adapters.sql index 51f7476..90fa9bd 100644 --- a/dbt/include/mariadb/macros/adapters.sql +++ b/dbt/include/mariadb/macros/adapters.sql @@ -39,14 +39,29 @@ create {% if temporary: -%}temporary{%- endif %} table {{ relation.include(database=False) }} - {{ sql }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {{ get_table_columns_and_constraints() }} + {%- set sql = get_select_subquery(sql) %} + ( + {{ sql }} + ) + {% else %} + {{ sql }} + {% endif %} {% endmacro %} {% macro mariadb__create_view_as(relation, sql) -%} {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} - create view {{ relation }} as + create view {{ relation }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} + as {{ sql }} {%- endmacro %} @@ -101,3 +116,38 @@ {% macro mariadb__generate_database_name(custom_database_name=none, node=none) -%} {% do return(None) %} {%- endmacro %} + +{% macro mariadb__get_phony_data_for_type(data_type) %} + {# + The types that MariaDB supports in CAST statements are NOT the same as the + types that are supported in table definitions. This is a bit of a hack to + work around the known mismatches. + #} + {%- if data_type.lower() == 'integer' -%} + 0 + {%- elif data_type.lower() == 'text' -%} + '' + {%- elif data_type.lower() == 'integer unsigned' -%} + cast(null as unsigned) + {%- elif data_type.lower() == 'integer signed' -%} + cast(null as signed) + {%- else -%} + cast(null as {{ data_type }}) + {%- endif -%} +{% endmacro %} + +{% macro mariadb__get_empty_schema_sql(columns) %} + {%- set col_err = [] -%} + select + {% for i in columns %} + {%- set col = columns[i] -%} + {%- if col['data_type'] is not defined -%} + {{ col_err.append(col['name']) }} + {%- endif -%} + {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %} + {{ mariadb__get_phony_data_for_type(col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }} + {%- endfor -%} + {%- if (col_err | length) > 0 -%} + {{ exceptions.column_type_missing(column_names=col_err) }} + {%- endif -%} +{% endmacro %} diff --git a/dbt/include/mysql/macros/adapters.sql b/dbt/include/mysql/macros/adapters.sql index eada89e..3b8b4d5 100644 --- a/dbt/include/mysql/macros/adapters.sql +++ b/dbt/include/mysql/macros/adapters.sql @@ -39,9 +39,17 @@ create {% if temporary: -%}temporary{%- endif %} table {{ relation.include(database=False) }} - as ( - {{ sql }} - ) + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {{ get_table_columns_and_constraints() }} + {%- set sql = get_select_subquery(sql) %} + {% else %} + as + {% endif %} + ( + {{ sql }} + ) {% endmacro %} {% macro mysql__current_timestamp() -%} @@ -95,3 +103,38 @@ {% macro mysql__generate_database_name(custom_database_name=none, node=none) -%} {% do return(None) %} {%- endmacro %} + +{% macro mysql__get_phony_data_for_type(data_type) %} + {# + The types that MySQL supports in CAST statements are NOT the same as the + types that are supported in table definitions. This is a bit of a hack to + work around the known mismatches. + #} + {%- if data_type.lower() == 'integer' -%} + 0 + {%- elif data_type.lower() == 'text' -%} + '' + {%- elif data_type.lower() == 'integer unsigned' -%} + cast(null as unsigned) + {%- elif data_type.lower() == 'integer signed' -%} + cast(null as signed) + {%- else -%} + cast(null as {{ data_type }}) + {%- endif -%} +{% endmacro %} + +{% macro mysql__get_empty_schema_sql(columns) %} + {%- set col_err = [] -%} + select + {% for i in columns %} + {%- set col = columns[i] -%} + {%- if col['data_type'] is not defined -%} + {{ col_err.append(col['name']) }} + {%- endif -%} + {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %} + {{ mysql__get_phony_data_for_type(col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }} + {%- endfor -%} + {%- if (col_err | length) > 0 -%} + {{ exceptions.column_type_missing(column_names=col_err) }} + {%- endif -%} +{% endmacro %} diff --git a/dbt/include/mysql5/macros/adapters.sql b/dbt/include/mysql5/macros/adapters.sql index 624d282..5c0c9bc 100644 --- a/dbt/include/mysql5/macros/adapters.sql +++ b/dbt/include/mysql5/macros/adapters.sql @@ -39,7 +39,17 @@ create {% if temporary: -%}temporary{%- endif %} table {{ relation.include(database=False) }} - {{ sql }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {{ get_table_columns_and_constraints() }} + {%- set sql = get_select_subquery(sql) %} + ( + {{ sql }} + ) + {% else %} + {{ sql }} + {% endif %} {% endmacro %} {% macro mysql5__current_timestamp() -%} @@ -93,3 +103,38 @@ {% macro mysql5__generate_database_name(custom_database_name=none, node=none) -%} {% do return(None) %} {%- endmacro %} + +{% macro mysql5__get_phony_data_for_type(data_type) %} + {# + The types that MySQL supports in CAST statements are NOT the same as the + types that are supported in table definitions. This is a bit of a hack to + work around the known mismatches. + #} + {%- if data_type.lower() == 'integer' -%} + 0 + {%- elif data_type.lower() == 'text' -%} + '' + {%- elif data_type.lower() == 'integer unsigned' -%} + cast(null as unsigned) + {%- elif data_type.lower() == 'integer signed' -%} + cast(null as signed) + {%- else -%} + cast(null as {{ data_type }}) + {%- endif -%} +{% endmacro %} + +{% macro mysql5__get_empty_schema_sql(columns) %} + {%- set col_err = [] -%} + select + {% for i in columns %} + {%- set col = columns[i] -%} + {%- if col['data_type'] is not defined -%} + {{ col_err.append(col['name']) }} + {%- endif -%} + {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %} + {{ mysql5__get_phony_data_for_type(col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }} + {%- endfor -%} + {%- if (col_err | length) > 0 -%} + {{ exceptions.column_type_missing(column_names=col_err) }} + {%- endif -%} +{% endmacro %} diff --git a/tests/functional/adapter/constraints/fixtures.py b/tests/functional/adapter/constraints/fixtures.py new file mode 100644 index 0000000..2b860f4 --- /dev/null +++ b/tests/functional/adapter/constraints/fixtures.py @@ -0,0 +1,320 @@ +# model breaking constraints +my_model_with_nulls_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + -- null value for 'id' + CAST(null AS UNSIGNED) as id, + -- change the color as well (to test rollback) + 'red' as color, + '2019-01-01' as date_day +""" + + +my_model_view_with_nulls_sql = """ +{{ + config( + materialized = "view" + ) +}} + +select + -- null value for 'id' + CAST(null AS UNSIGNED) as id, + -- change the color as well (to test rollback) + 'red' as color, + '2019-01-01' as date_day +""" + +my_model_incremental_with_nulls_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' ) +}} + +select + -- null value for 'id' + CAST(null AS UNSIGNED) as id, + -- change the color as well (to test rollback) + 'red' as color, + '2019-01-01' as date_day +""" + +# model columns data types different to schema definitions +my_model_contract_sql_header_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select 'Kolkata' as column_name +""" + +my_model_incremental_contract_sql_header_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change="append_new_columns" + ) +}} + +select 'Kolkata' as column_name +""" + +constrained_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: check + expression: (id > 0) + - type: check + expression: id >= 1 + - type: primary_key + columns: [ id ] + - type: unique + columns: [ color(10), date_day(20) ] + name: strange_uniqueness_requirement + - type: foreign_key + columns: [ id ] + expression: {schema}.foreign_key_model (id) + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: foreign_key_model + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + constraints: + - type: unique + - type: primary_key +""" + +model_quoted_column_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + materialized: table + constraints: + - type: check + # this one is the on the user + expression: (`from` = 'blue') + columns: [ '`from`' ] + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + tests: + - unique + - name: from # reserved word + quote: true + data_type: text + constraints: + - type: not_null + - name: date_day + data_type: text +""" + +# MariaDB does not support multiple column-level CHECK constraints +mariadb_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: id > 0 AND id >= 1 + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_wrong_order + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text +""" + +# MariaDB does not support multiple column-level CHECK constraints +# Additionally, MariaDB requires CHECK constraints to come last +mariadb_model_fk_constraint_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: foreign_key + expression: {schema}.foreign_key_model (id) + - type: unique + - type: check + expression: id > 0 AND id >= 1 + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_wrong_order + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: text + - name: foreign_key_model + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + constraints: + - type: unique + - type: primary_key +""" diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py new file mode 100644 index 0000000..78b6710 --- /dev/null +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -0,0 +1,365 @@ +import pytest + +from dbt.tests.adapter.constraints.test_constraints import ( + BaseTableConstraintsColumnsEqual, + BaseViewConstraintsColumnsEqual, + BaseTableContractSqlHeader, + BaseIncrementalContractSqlHeader, + BaseIncrementalConstraintsColumnsEqual, + BaseConstraintsRuntimeDdlEnforcement, + BaseConstraintsRollback, + BaseIncrementalConstraintsRuntimeDdlEnforcement, + BaseIncrementalConstraintsRollback, + BaseModelConstraintsRuntimeEnforcement, + BaseConstraintQuotedColumn, +) + +from dbt.tests.adapter.constraints.fixtures import ( + my_incremental_model_sql, + model_contract_header_schema_yml, + model_schema_yml, + my_model_wrong_order_depends_on_fk_sql, + foreign_key_model_sql, + my_model_with_quoted_column_name_sql, + my_model_incremental_wrong_order_depends_on_fk_sql, + model_fk_constraint_schema_yml, +) + +from tests.functional.adapter.constraints.fixtures import ( + my_model_with_nulls_sql, + my_model_incremental_with_nulls_sql, + my_model_contract_sql_header_sql, + my_model_incremental_contract_sql_header_sql, + mariadb_model_schema_yml, + mariadb_model_fk_constraint_schema_yml, + constrained_model_schema_yml, + model_quoted_column_schema_yml, +) + + +class MySQLColumnEqualSetup: + @pytest.fixture + def int_type(self): + return "INTEGER" + + @pytest.fixture + def schema_int_type(self): + return "INTEGER" + + @pytest.fixture + def data_types(self, int_type, schema_int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ["'str'", string_type, string_type], + ["cast('2019-01-01' as date)", "date", "DATE"], + ["cast('2013-11-03 00:00:00' as datetime)", "datetime", "DATETIME"], + ] + + +class TestMySQLTableConstraintsColumnsEqual( + MySQLColumnEqualSetup, BaseTableConstraintsColumnsEqual +): + pass + + +class TestMySQLViewConstraintsColumnsEqual( + MySQLColumnEqualSetup, BaseViewConstraintsColumnsEqual +): + pass + + +class TestMySQLIncrementalConstraintsColumnsEqual( + MySQLColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual +): + pass + + +class TestMySQLTableContractsSqlHeader(BaseTableContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +class TestMySQLIncrementalContractsSqlHeader(BaseIncrementalContractSqlHeader): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_contract_sql_header.sql": my_model_incremental_contract_sql_header_sql, + "constraints_schema.yml": model_contract_header_schema_yml, + } + + +# MySQL 5 does not support CHECK constraints +_expected_mysql5_ddl_enforcement_sql = """ + create table ( + id integer not null primary key unique, + color text, + date_day text + ) + ( + select id, color, date_day + from ( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq + ) +""" + +# MariaDB does not support multiple column-level CHECK constraints +# Additionally, MariaDB requires CHECK constraints to come last +_expected_mariadb_ddl_enforcement_sql = """ + create table ( + id integer not null primary key unique check (id > 0 AND id >= 1), + color text, + date_day text + ) + ( + select id, color, date_day + from ( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq + ) +""" + +_expected_mysql_ddl_enforcement_sql = """ + create table ( + id integer not null primary key check ((id > 0)) check (id >= 1) unique, + color text, + date_day text + ) + ( + select id, color, date_day + from ( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq + ) +""" + + +class TestMySQLTableConstraintsDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + @pytest.fixture(scope="class") + def models(self, dbt_profile_target): + if dbt_profile_target["type"] == "mariadb": + return { + "my_model.sql": my_model_incremental_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": mariadb_model_fk_constraint_schema_yml, + } + else: + return { + "my_model.sql": my_model_incremental_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self, dbt_profile_target): + if dbt_profile_target["type"] == "mysql5": + return _expected_mysql5_ddl_enforcement_sql + elif dbt_profile_target["type"] == "mariadb": + return _expected_mariadb_ddl_enforcement_sql + else: + return _expected_mysql_ddl_enforcement_sql + + +class TestMySQLIncrementalConstraintsDdlEnforcement( + BaseIncrementalConstraintsRuntimeDdlEnforcement +): + @pytest.fixture(scope="class") + def models(self, dbt_profile_target): + if dbt_profile_target["type"] == "mariadb": + return { + "my_model.sql": my_model_incremental_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": mariadb_model_fk_constraint_schema_yml, + } + else: + return { + "my_model.sql": my_model_incremental_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self, dbt_profile_target): + if dbt_profile_target["type"] == "mysql5": + return _expected_mysql5_ddl_enforcement_sql + elif dbt_profile_target["type"] == "mariadb": + return _expected_mariadb_ddl_enforcement_sql + else: + return _expected_mysql_ddl_enforcement_sql + + +class TestMySQLTableConstraintsRollback(BaseConstraintsRollback): + @pytest.fixture(scope="class") + def models(self, dbt_profile_target): + if dbt_profile_target["type"] == "mariadb": + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": mariadb_model_schema_yml, + } + else: + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["Column 'id' cannot be null"] + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_with_nulls_sql + + +class TestMySQLIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): + @pytest.fixture(scope="class") + def models(self, dbt_profile_target): + if dbt_profile_target["type"] == "mariadb": + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": mariadb_model_schema_yml, + } + else: + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["Column 'id' cannot be null"] + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_incremental_with_nulls_sql + + +# MySQL 5 does not support CHECK constraints +_expected_mysql5_runtime_enforcement_sql = """ + create table ( + id integer not null, + color text, + date_day text, + primary key (id), + constraint strange_uniqueness_requirement unique (color(10), date_day(20)) + ) + ( + select id, color, date_day + from ( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq + ) +""" + +_expected_mysql_runtime_enforcement_sql = """ + create table ( + id integer not null, + color text, + date_day text, + check ((id > 0)), + check (id >= 1), + primary key (id), + constraint strange_uniqueness_requirement unique (color(10), date_day(20)) + ) + ( + select id, color, date_day + from ( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq + ) +""" + + +class TestMySQLModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": constrained_model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self, dbt_profile_target): + if dbt_profile_target["type"] == "mysql5": + return _expected_mysql5_runtime_enforcement_sql + else: + return _expected_mysql_runtime_enforcement_sql + + +# MySQL 5 does not support CHECK constraints +_expected_mysql5_quoted_column_sql = """ + create table ( + id integer not null, + `from` text not null, + date_day text + ) + ( + select id, `from`, date_day + from ( + select + 'blue' as `from`, + 1 as id, + '2019-01-01' as date_day + ) as model_subq + ) +""" + +_expected_mysql_quoted_column_sql = """ + create table ( + id integer not null, + `from` text not null, + date_day text, + check ((`from` = 'blue')) + ) + ( + select id, `from`, date_day + from ( + select + 'blue' as `from`, + 1 as id, + '2019-01-01' as date_day + ) as model_subq + ) +""" + + +class TestMySQLConstraintQuotedColumn(BaseConstraintQuotedColumn): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_with_quoted_column_name_sql, + "constraints_schema.yml": model_quoted_column_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self, dbt_profile_target): + if dbt_profile_target["type"] == "mysql5": + return _expected_mysql5_quoted_column_sql + else: + return _expected_mysql_quoted_column_sql diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 8c499d5..06af385 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -8,7 +8,6 @@ class TestMySQLAdapter(unittest.TestCase): def setUp(self): - pass flags.STRICT_MODE = True profile_cfg = { diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 07371d3..74d8483 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -44,6 +44,13 @@ def profile_from_dict(profile, profile_name, cli_vars="{}"): cli_vars = parse_cli_vars(cli_vars) renderer = ProfileRenderer(cli_vars) + + # in order to call dbt's internal profile rendering, we need to set the + # flags global. This is a bit of a hack, but it's the best way to do it. + from dbt.flags import set_from_args + from argparse import Namespace + + set_from_args(Namespace(), None) return Profile.from_raw_profile_info( profile, profile_name, @@ -72,7 +79,7 @@ def project_from_dict(project, profile, packages=None, selectors=None, cli_vars= def config_from_parts_or_dicts( - project, profile, packages=None, selectors=None, cli_vars="{}" + project, profile, packages=None, selectors=None, cli_vars={} ): from dbt.config import Project, Profile, RuntimeConfig from copy import deepcopy From e45c48c318a9b7aad0d136cbaa9411f42fbbc009 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 14:44:31 -0700 Subject: [PATCH 09/24] Support Black & MyPy pre-commit hooks (#167) * Add black and mypy as pre-commit hooks * Run black formatter on all files * Add MyPy configuration & make tweaks and ignore errors to make MyPy pass * Add .git-blame-ignore-revs to ignore `black` changes in git blame * Update changelog --- .git-blame-ignore-revs | 2 ++ .github/workflows/main.yml | 1 + .pre-commit-config.yaml | 45 +++++++++++++++++++++++-- CHANGELOG.md | 3 +- Makefile | 19 +++++++++-- dbt/adapters/mariadb/__init__.py | 2 +- dbt/adapters/mariadb/connections.py | 11 +++---- dbt/adapters/mariadb/impl.py | 51 ++++++++++++----------------- dbt/adapters/mysql/__init__.py | 4 ++- dbt/adapters/mysql/connections.py | 11 +++---- dbt/adapters/mysql/impl.py | 48 +++++++++++---------------- dbt/adapters/mysql5/__init__.py | 4 ++- dbt/adapters/mysql5/connections.py | 11 +++---- dbt/adapters/mysql5/impl.py | 49 +++++++++++---------------- dev-requirements.txt | 2 ++ mypy.ini | 2 ++ tests/unit/utils.py | 4 +-- 17 files changed, 148 insertions(+), 121 deletions(-) create mode 100644 .git-blame-ignore-revs create mode 100644 mypy.ini diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..4323bf5 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Ran `black` on all files +99a125c82b846fd25aed432ed67a9ad982bbe0ad diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2d3db96..66093db 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,6 +58,7 @@ jobs: python -m pip install -r dev-requirements.txt python -m pip --version pre-commit --version + mypy --version dbt --version - name: Run pre-commit hooks diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5e392cb..3d80b95 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,12 @@ # For more on configuring pre-commit hooks (see https://pre-commit.com/) +# Force all unspecified python hooks to run python 3.8 default_language_version: python: python3 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.4.0 hooks: - id: check-yaml args: [--unsafe] @@ -13,10 +14,50 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - id: check-case-conflict +- repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + additional_dependencies: ['click~=8.1'] + args: + - "--line-length=99" + - "--target-version=py38" + - id: black + alias: black-check + stages: [manual] + additional_dependencies: ['click~=8.1'] + args: + - "--line-length=99" + - "--target-version=py38" + - "--check" + - "--diff" - repo: https://github.com/pycqa/flake8 - rev: 4.0.1 + rev: 6.0.0 hooks: - id: flake8 - id: flake8 alias: flake8-check stages: [manual] +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.1.1 + hooks: + - id: mypy + # N.B.: Mypy is... a bit fragile. + # + # By using `language: system` we run this hook in the local + # environment instead of a pre-commit isolated one. This is needed + # to ensure mypy correctly parses the project. + + # It may cause trouble in that it adds environmental variables out + # of our control to the mix. Unfortunately, there's nothing we can + # do about per pre-commit's author. + # See https://github.com/pre-commit/pre-commit/issues/730 for details. + args: [--show-error-codes, --ignore-missing-imports, --explicit-package-bases] + files: ^dbt/adapters/.* + language: system + - id: mypy + alias: mypy-check + stages: [manual] + args: [--show-error-codes, --pretty, --ignore-missing-imports, --explicit-package-bases] + files: ^dbt/adapters + language: system diff --git a/CHANGELOG.md b/CHANGELOG.md index 85d01e2..0186095 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features - Migrate CircleCI to GitHub Actions ([#120](https://github.com/dbeatty10/dbt-mysql/issues/120)) +- Support Black & MyPy pre-commit hooks ([#138](https://github.com/dbeatty10/dbt-mysql/issues/138)) ### Fixes - Fix incremental composite keys ([#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) @@ -9,7 +10,7 @@ ### Contributors - [@moszutij](https://github.com/moszutij) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146), [#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) - [@wesen](https://github.com/wesen) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) -- [@mwallace582](https://github.com/mwallace582) ([#162](https://github.com/dbeatty10/dbt-mysql/pull/162)) +- [@mwallace582](https://github.com/mwallace582) ([#162](https://github.com/dbeatty10/dbt-mysql/pull/162), [#138](https://github.com/dbeatty10/dbt-mysql/issues/138)) ## dbt-mysql 1.1.0 (Feb 5, 2023) diff --git a/Makefile b/Makefile index d6aa5b5..e59c742 100644 --- a/Makefile +++ b/Makefile @@ -11,15 +11,26 @@ dev-uninstall: ## Uninstalls all packages while maintaining the virtual environm pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y pip uninstall -y dbt-mysql +.PHONY: mypy +mypy: ## Runs mypy against staged changes for static type checking. + @\ + pre-commit run --hook-stage manual mypy-check | grep -v "INFO" + .PHONY: flake8 flake8: ## Runs flake8 against staged changes to enforce style guide. @\ pre-commit run --hook-stage manual flake8-check | grep -v "INFO" +.PHONY: black +black: ## Runs black against staged changes to enforce style guide. + @\ + pre-commit run --hook-stage manual black-check -v | grep -v "INFO" + .PHONY: lint -lint: ## Runs flake8 code checks against staged changes. +lint: ## Runs flake8 and mypy code checks against staged changes. @\ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; + pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run mypy-check --hook-stage manual | grep -v "INFO" .PHONY: linecheck linecheck: ## Checks for all Python lines 100 characters or more @@ -35,7 +46,9 @@ unit: ## Runs unit tests with py38. test: ## Runs unit tests with py38 and code checks against staged changes. @\ tox -p -e py38; \ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; + pre-commit run black-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run mypy-check --hook-stage manual | grep -v "INFO" .PHONY: integration integration: ## Runs mysql integration tests with py38. diff --git a/dbt/adapters/mariadb/__init__.py b/dbt/adapters/mariadb/__init__.py index 820e9d1..f6f09ad 100644 --- a/dbt/adapters/mariadb/__init__.py +++ b/dbt/adapters/mariadb/__init__.py @@ -9,7 +9,7 @@ Plugin = AdapterPlugin( - adapter=MariaDBAdapter, + adapter=MariaDBAdapter, # type: ignore[arg-type] credentials=MariaDBCredentials, include_path=mariadb.PACKAGE_PATH, ) diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index d85ec29..879b260 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -16,10 +16,10 @@ @dataclass(init=False) class MariaDBCredentials(Credentials): - server: str + server: str = "" port: Optional[int] = None - database: Optional[str] = None - schema: str + database: str = "" + schema: str = "" username: Optional[str] = None password: Optional[str] = None charset: Optional[str] = None @@ -95,7 +95,6 @@ def open(cls, connection): connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" except mysql.connector.Error: - try: logger.debug( "Failed connection without supplying the `database`. " @@ -108,10 +107,8 @@ def open(cls, connection): connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" except mysql.connector.Error as e: - logger.debug( - "Got an error when attempting to open a MariaDB " - "connection: '{}'".format(e) + "Got an error when attempting to open a MariaDB " "connection: '{}'".format(e) ) connection.handle = None diff --git a/dbt/adapters/mariadb/impl.py b/dbt/adapters/mariadb/impl.py index 6a16bcb..cdb6273 100644 --- a/dbt/adapters/mariadb/impl.py +++ b/dbt/adapters/mariadb/impl.py @@ -1,6 +1,6 @@ from concurrent.futures import Future from dataclasses import asdict -from typing import Optional, List, Dict, Any, Iterable +from typing import Optional, List, Dict, Any, Iterable, Tuple import agate import dbt @@ -12,6 +12,7 @@ from dbt.adapters.mariadb import MariaDBRelation from dbt.adapters.mariadb import MariaDBColumn from dbt.adapters.base import BaseRelation +from dbt.contracts.graph.manifest import Manifest from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER from dbt.events import AdapterLogger from dbt.utils import executor @@ -38,8 +39,8 @@ def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: def quote(self, identifier): return "`{}`".format(identifier) - def list_relations_without_caching( - self, schema_relation: MariaDBRelation + def list_relations_without_caching( # type: ignore[override] + self, schema_relation: MariaDBRelation # type: ignore[override] ) -> List[MariaDBRelation]: kwargs = {"schema_relation": schema_relation} try: @@ -62,20 +63,16 @@ def list_relations_without_caching( f"got {len(row)} values, expected 4" ) _, name, _schema, relation_type = row - relation = self.Relation.create( - schema=_schema, identifier=name, type=relation_type - ) + relation = self.Relation.create(schema=_schema, identifier=name, type=relation_type) relations.append(relation) return relations - def get_columns_in_relation(self, relation: Relation) -> List[MariaDBColumn]: + def get_columns_in_relation(self, relation: MariaDBRelation) -> List[MariaDBColumn]: rows: List[agate.Row] = super().get_columns_in_relation(relation) return self.parse_show_columns(relation, rows) - def _get_columns_for_catalog( - self, relation: MariaDBRelation - ) -> Iterable[Dict[str, Any]]: + def _get_columns_for_catalog(self, relation: MariaDBRelation) -> Iterable[Dict[str, Any]]: columns = self.get_columns_in_relation(relation) for column in columns: @@ -87,7 +84,7 @@ def _get_columns_for_catalog( yield as_dict def get_relation( - self, database: str, schema: str, identifier: str + self, database: Optional[str], schema: str, identifier: str ) -> Optional[BaseRelation]: if not self.Relation.include_policy.database: database = None @@ -95,7 +92,7 @@ def get_relation( return super().get_relation(database, schema, identifier) def parse_show_columns( - self, relation: Relation, raw_rows: List[agate.Row] + self, relation: MariaDBRelation, raw_rows: List[agate.Row] ) -> List[MariaDBColumn]: return [ MariaDBColumn( @@ -112,12 +109,12 @@ def parse_show_columns( for idx, column in enumerate(raw_rows) ] - def get_catalog(self, manifest): + def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]: schema_map = self._get_catalog_schemas(manifest) + if len(schema_map) > 1: dbt.exceptions.raise_compiler_error( - f"Expected only one database in get_catalog, found " - f"{list(schema_map)}" + f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) with executor(self.config) as tpe: @@ -145,8 +142,7 @@ def _get_one_catalog( ) -> agate.Table: if len(schemas) != 1: dbt.exceptions.raise_compiler_error( - f"Expected only one schema in mariadb _get_one_catalog, found " - f"{schemas}" + f"Expected only one schema in mariadb_get_one_catalog, found " f"{schemas}" ) database = information_schema.database @@ -155,13 +151,11 @@ def _get_one_catalog( columns: List[Dict[str, Any]] = [] for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) - columns.extend(self._get_columns_for_catalog(relation)) + columns.extend(self._get_columns_for_catalog(relation)) # type: ignore[arg-type] return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): - results = self.execute_macro( - LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database} - ) + results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) exists = True if schema in [row[0] for row in results] else False return exists @@ -179,9 +173,7 @@ def update_column_sql( clause += f" where {where_clause}" return clause - def timestamp_add_sql( - self, add_to: str, number: int = 1, interval: str = "hour" - ) -> str: + def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: # for backwards compatibility, we're compelled to set some sort of # default. A lot of searching has lead me to believe that the # '+ interval' syntax used in postgres/redshift is relatively common @@ -205,9 +197,10 @@ def string_add_sql( def get_rows_different_sql( self, - relation_a: MariaDBRelation, - relation_b: MariaDBRelation, + relation_a: MariaDBRelation, # type: ignore[override] + relation_b: MariaDBRelation, # type: ignore[override] column_names: Optional[List[str]] = None, + except_operator: str = "", # Required to match BaseRelation.get_rows_different_sql() ) -> str: # This method only really exists for test reasons names: List[str] @@ -221,12 +214,10 @@ def get_rows_different_sql( alias_b = "B" columns_csv_a = ", ".join([f"{alias_a}.{name}" for name in names]) columns_csv_b = ", ".join([f"{alias_b}.{name}" for name in names]) - join_condition = " AND ".join( - [f"{alias_a}.{name} = {alias_b}.{name}" for name in names] - ) + join_condition = " AND ".join([f"{alias_a}.{name} = {alias_b}.{name}" for name in names]) first_column = names[0] - # There is no EXCEPT or MINUS operator, so we need to simulate it + # MariaDB doesn't have an EXCEPT or MINUS operator, so we need to simulate it COLUMNS_EQUAL_SQL = """ SELECT row_count_diff.difference as row_count_difference, diff --git a/dbt/adapters/mysql/__init__.py b/dbt/adapters/mysql/__init__.py index 654b023..1357d08 100644 --- a/dbt/adapters/mysql/__init__.py +++ b/dbt/adapters/mysql/__init__.py @@ -9,5 +9,7 @@ Plugin = AdapterPlugin( - adapter=MySQLAdapter, credentials=MySQLCredentials, include_path=mysql.PACKAGE_PATH + adapter=MySQLAdapter, # type: ignore[arg-type] + credentials=MySQLCredentials, + include_path=mysql.PACKAGE_PATH, ) diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 6a4e285..087aa4a 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -16,10 +16,10 @@ @dataclass(init=False) class MySQLCredentials(Credentials): - server: str + server: str = "" port: Optional[int] = None - database: Optional[str] = None - schema: str + database: str = "" + schema: str = "" username: Optional[str] = None password: Optional[str] = None charset: Optional[str] = None @@ -91,7 +91,6 @@ def open(cls, connection): connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" except mysql.connector.Error: - try: logger.debug( "Failed connection without supplying the `database`. " @@ -104,10 +103,8 @@ def open(cls, connection): connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" except mysql.connector.Error as e: - logger.debug( - "Got an error when attempting to open a mysql " - "connection: '{}'".format(e) + "Got an error when attempting to open a mysql " "connection: '{}'".format(e) ) connection.handle = None diff --git a/dbt/adapters/mysql/impl.py b/dbt/adapters/mysql/impl.py index f1e11d1..41afacf 100644 --- a/dbt/adapters/mysql/impl.py +++ b/dbt/adapters/mysql/impl.py @@ -1,6 +1,6 @@ from concurrent.futures import Future from dataclasses import asdict -from typing import Optional, List, Dict, Any, Iterable +from typing import Optional, List, Dict, Any, Iterable, Tuple import agate import dbt @@ -12,6 +12,7 @@ from dbt.adapters.mysql import MySQLRelation from dbt.adapters.mysql import MySQLColumn from dbt.adapters.base import BaseRelation +from dbt.contracts.graph.manifest import Manifest from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER from dbt.events import AdapterLogger from dbt.utils import executor @@ -38,8 +39,8 @@ def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: def quote(self, identifier): return "`{}`".format(identifier) - def list_relations_without_caching( - self, schema_relation: MySQLRelation + def list_relations_without_caching( # type: ignore[override] + self, schema_relation: MySQLRelation # type: ignore[override] ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: @@ -62,20 +63,16 @@ def list_relations_without_caching( f"got {len(row)} values, expected 4" ) _, name, _schema, relation_type = row - relation = self.Relation.create( - schema=_schema, identifier=name, type=relation_type - ) + relation = self.Relation.create(schema=_schema, identifier=name, type=relation_type) relations.append(relation) return relations - def get_columns_in_relation(self, relation: Relation) -> List[MySQLColumn]: + def get_columns_in_relation(self, relation: MySQLRelation) -> List[MySQLColumn]: rows: List[agate.Row] = super().get_columns_in_relation(relation) return self.parse_show_columns(relation, rows) - def _get_columns_for_catalog( - self, relation: MySQLRelation - ) -> Iterable[Dict[str, Any]]: + def _get_columns_for_catalog(self, relation: MySQLRelation) -> Iterable[Dict[str, Any]]: columns = self.get_columns_in_relation(relation) for column in columns: @@ -87,7 +84,7 @@ def _get_columns_for_catalog( yield as_dict def get_relation( - self, database: str, schema: str, identifier: str + self, database: Optional[str], schema: str, identifier: str ) -> Optional[BaseRelation]: if not self.Relation.include_policy.database: database = None @@ -95,7 +92,7 @@ def get_relation( return super().get_relation(database, schema, identifier) def parse_show_columns( - self, relation: Relation, raw_rows: List[agate.Row] + self, relation: MySQLRelation, raw_rows: List[agate.Row] ) -> List[MySQLColumn]: return [ MySQLColumn( @@ -112,13 +109,12 @@ def parse_show_columns( for idx, column in enumerate(raw_rows) ] - def get_catalog(self, manifest): + def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]: schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: dbt.exceptions.raise_compiler_error( - f"Expected only one database in get_catalog, found " - f"{list(schema_map)}" + f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) with executor(self.config) as tpe: @@ -146,8 +142,7 @@ def _get_one_catalog( ) -> agate.Table: if len(schemas) != 1: dbt.exceptions.raise_compiler_error( - f"Expected only one schema in mysql _get_one_catalog, found " - f"{schemas}" + f"Expected only one schema in mysql_get_one_catalog, found " f"{schemas}" ) database = information_schema.database @@ -156,13 +151,11 @@ def _get_one_catalog( columns: List[Dict[str, Any]] = [] for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) - columns.extend(self._get_columns_for_catalog(relation)) + columns.extend(self._get_columns_for_catalog(relation)) # type: ignore[arg-type] return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): - results = self.execute_macro( - LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database} - ) + results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) exists = True if schema in [row[0] for row in results] else False return exists @@ -180,9 +173,7 @@ def update_column_sql( clause += f" where {where_clause}" return clause - def timestamp_add_sql( - self, add_to: str, number: int = 1, interval: str = "hour" - ) -> str: + def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: # for backwards compatibility, we're compelled to set some sort of # default. A lot of searching has lead me to believe that the # '+ interval' syntax used in postgres/redshift is relatively common @@ -206,9 +197,10 @@ def string_add_sql( def get_rows_different_sql( self, - relation_a: MySQLRelation, - relation_b: MySQLRelation, + relation_a: MySQLRelation, # type: ignore[override] + relation_b: MySQLRelation, # type: ignore[override] column_names: Optional[List[str]] = None, + except_operator: str = "", # Required to match BaseRelation.get_rows_different_sql() ) -> str: # This method only really exists for test reasons names: List[str] @@ -222,9 +214,7 @@ def get_rows_different_sql( alias_b = "B" columns_csv_a = ", ".join([f"{alias_a}.{name}" for name in names]) columns_csv_b = ", ".join([f"{alias_b}.{name}" for name in names]) - join_condition = " AND ".join( - [f"{alias_a}.{name} = {alias_b}.{name}" for name in names] - ) + join_condition = " AND ".join([f"{alias_a}.{name} = {alias_b}.{name}" for name in names]) first_column = names[0] # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it diff --git a/dbt/adapters/mysql5/__init__.py b/dbt/adapters/mysql5/__init__.py index 8f23e58..00d41e6 100644 --- a/dbt/adapters/mysql5/__init__.py +++ b/dbt/adapters/mysql5/__init__.py @@ -9,5 +9,7 @@ Plugin = AdapterPlugin( - adapter=MySQLAdapter, credentials=MySQLCredentials, include_path=mysql5.PACKAGE_PATH + adapter=MySQLAdapter, # type: ignore[arg-type] + credentials=MySQLCredentials, + include_path=mysql5.PACKAGE_PATH, ) diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index c8c1d20..e6c6ed8 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -16,10 +16,10 @@ @dataclass(init=False) class MySQLCredentials(Credentials): - server: str + server: str = "" port: Optional[int] = None - database: Optional[str] = None - schema: str + database: str = "" + schema: str = "" username: Optional[str] = None password: Optional[str] = None charset: Optional[str] = None @@ -95,7 +95,6 @@ def open(cls, connection): connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" except mysql.connector.Error: - try: logger.debug( "Failed connection without supplying the `database`. " @@ -108,10 +107,8 @@ def open(cls, connection): connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" except mysql.connector.Error as e: - logger.debug( - "Got an error when attempting to open a mysql " - "connection: '{}'".format(e) + "Got an error when attempting to open a mysql " "connection: '{}'".format(e) ) connection.handle = None diff --git a/dbt/adapters/mysql5/impl.py b/dbt/adapters/mysql5/impl.py index 2582c83..65f84b4 100644 --- a/dbt/adapters/mysql5/impl.py +++ b/dbt/adapters/mysql5/impl.py @@ -1,6 +1,6 @@ from concurrent.futures import Future from dataclasses import asdict -from typing import Optional, List, Dict, Any, Iterable +from typing import Optional, List, Dict, Any, Iterable, Tuple import agate import dbt @@ -12,6 +12,7 @@ from dbt.adapters.mysql5 import MySQLRelation from dbt.adapters.mysql5 import MySQLColumn from dbt.adapters.base import BaseRelation +from dbt.contracts.graph.manifest import Manifest from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER from dbt.events import AdapterLogger from dbt.utils import executor @@ -38,8 +39,8 @@ def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: def quote(self, identifier): return "`{}`".format(identifier) - def list_relations_without_caching( - self, schema_relation: MySQLRelation + def list_relations_without_caching( # type: ignore[override] + self, schema_relation: MySQLRelation # type: ignore[override] ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: @@ -62,20 +63,16 @@ def list_relations_without_caching( f"got {len(row)} values, expected 4" ) _, name, _schema, relation_type = row - relation = self.Relation.create( - schema=_schema, identifier=name, type=relation_type - ) + relation = self.Relation.create(schema=_schema, identifier=name, type=relation_type) relations.append(relation) return relations - def get_columns_in_relation(self, relation: Relation) -> List[MySQLColumn]: + def get_columns_in_relation(self, relation: MySQLRelation) -> List[MySQLColumn]: rows: List[agate.Row] = super().get_columns_in_relation(relation) return self.parse_show_columns(relation, rows) - def _get_columns_for_catalog( - self, relation: MySQLRelation - ) -> Iterable[Dict[str, Any]]: + def _get_columns_for_catalog(self, relation: MySQLRelation) -> Iterable[Dict[str, Any]]: columns = self.get_columns_in_relation(relation) for column in columns: @@ -87,7 +84,7 @@ def _get_columns_for_catalog( yield as_dict def get_relation( - self, database: str, schema: str, identifier: str + self, database: Optional[str], schema: str, identifier: str ) -> Optional[BaseRelation]: if not self.Relation.include_policy.database: database = None @@ -95,7 +92,7 @@ def get_relation( return super().get_relation(database, schema, identifier) def parse_show_columns( - self, relation: Relation, raw_rows: List[agate.Row] + self, relation: MySQLRelation, raw_rows: List[agate.Row] ) -> List[MySQLColumn]: return [ MySQLColumn( @@ -112,12 +109,12 @@ def parse_show_columns( for idx, column in enumerate(raw_rows) ] - def get_catalog(self, manifest): + def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]: schema_map = self._get_catalog_schemas(manifest) + if len(schema_map) > 1: dbt.exceptions.raise_compiler_error( - f"Expected only one database in get_catalog, found " - f"{list(schema_map)}" + f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) with executor(self.config) as tpe: @@ -145,8 +142,7 @@ def _get_one_catalog( ) -> agate.Table: if len(schemas) != 1: dbt.exceptions.raise_compiler_error( - f"Expected only one schema in mysql5 _get_one_catalog, found " - f"{schemas}" + f"Expected only one schema in mysql5 _get_one_catalog, found " f"{schemas}" ) database = information_schema.database @@ -155,13 +151,11 @@ def _get_one_catalog( columns: List[Dict[str, Any]] = [] for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) - columns.extend(self._get_columns_for_catalog(relation)) + columns.extend(self._get_columns_for_catalog(relation)) # type: ignore[arg-type] return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): - results = self.execute_macro( - LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database} - ) + results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) exists = True if schema in [row[0] for row in results] else False return exists @@ -179,9 +173,7 @@ def update_column_sql( clause += f" where {where_clause}" return clause - def timestamp_add_sql( - self, add_to: str, number: int = 1, interval: str = "hour" - ) -> str: + def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: # for backwards compatibility, we're compelled to set some sort of # default. A lot of searching has lead me to believe that the # '+ interval' syntax used in postgres/redshift is relatively common @@ -205,9 +197,10 @@ def string_add_sql( def get_rows_different_sql( self, - relation_a: MySQLRelation, - relation_b: MySQLRelation, + relation_a: MySQLRelation, # type: ignore[override] + relation_b: MySQLRelation, # type: ignore[override] column_names: Optional[List[str]] = None, + except_operator: str = "", # Required to match BaseRelation.get_rows_different_sql() ) -> str: # This method only really exists for test reasons names: List[str] @@ -221,9 +214,7 @@ def get_rows_different_sql( alias_b = "B" columns_csv_a = ", ".join([f"{alias_a}.{name}" for name in names]) columns_csv_b = ", ".join([f"{alias_b}.{name}" for name in names]) - join_condition = " AND ".join( - [f"{alias_a}.{name} = {alias_b}.{name}" for name in names] - ) + join_condition = " AND ".join([f"{alias_a}.{name} = {alias_b}.{name}" for name in names]) first_column = names[0] # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it diff --git a/dev-requirements.txt b/dev-requirements.txt index b52651f..7acf8ba 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,12 +5,14 @@ git+https://github.com/dbt-labs/dbt-core.git@1.2.latest#egg=dbt-tests-adapter&su # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor +black~=22.12 bumpversion~=0.6.0 ddtrace~=2.3 flake8~=6.1 flaky~=3.7 freezegun~=1.3 ipdb~=0.13.13 +mypy==1.7.1 # patch updates have historically introduced breaking changes pre-commit~=3.5 pre-commit-hooks~=4.5 pytest~=7.4 diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..b6e6035 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +namespace_packages = True diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 07371d3..bac51d6 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -71,9 +71,7 @@ def project_from_dict(project, profile, packages=None, selectors=None, cli_vars= return partial.render(renderer) -def config_from_parts_or_dicts( - project, profile, packages=None, selectors=None, cli_vars="{}" -): +def config_from_parts_or_dicts(project, profile, packages=None, selectors=None, cli_vars="{}"): from dbt.config import Project, Profile, RuntimeConfig from copy import deepcopy From f0507baaa4279575831ad06452757666d25bd90b Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 15:22:15 -0700 Subject: [PATCH 10/24] Run black formatter on all files --- dbt/adapters/mariadb/connections.py | 4 +--- dbt/adapters/mariadb/impl.py | 15 +++++---------- dbt/adapters/mysql/connections.py | 4 +--- dbt/adapters/mysql/impl.py | 15 +++++---------- dbt/adapters/mysql5/connections.py | 4 +--- dbt/adapters/mysql5/impl.py | 15 +++++---------- 6 files changed, 18 insertions(+), 39 deletions(-) diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index 7effe96..a8fcb74 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -165,7 +165,5 @@ def get_response(cls, cursor) -> AdapterResponse: # the mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), - rows_affected=num_rows, - code=code + _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code ) diff --git a/dbt/adapters/mariadb/impl.py b/dbt/adapters/mariadb/impl.py index 09870b2..c04c19c 100644 --- a/dbt/adapters/mariadb/impl.py +++ b/dbt/adapters/mariadb/impl.py @@ -33,8 +33,7 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, - col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -45,8 +44,7 @@ def list_relations_without_caching( # type: ignore[override] ) -> List[MariaDBRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, - kwargs=kwargs) + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") if f"MariaDB database '{schema_relation}' not found" in errmsg: @@ -116,8 +114,7 @@ def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]] if len(schema_map) > 1: raise dbt.exceptions.CompilationError( - f"Expected only one database in get_catalog, found " - f"{list(schema_map)}" + f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) with executor(self.config) as tpe: @@ -145,8 +142,7 @@ def _get_one_catalog( ) -> agate.Table: if len(schemas) != 1: raise dbt.exceptions.CompilationError( - f"Expected only one schema in mariadb _get_one_catalog, found " - f"{schemas}" + f"Expected only one schema in mariadb _get_one_catalog, found " f"{schemas}" ) database = information_schema.database @@ -156,8 +152,7 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) # type: ignore[arg-type] - return agate.Table.from_object(columns, - column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 676c818..45329e3 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -161,7 +161,5 @@ def get_response(cls, cursor) -> AdapterResponse: # mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), - rows_affected=num_rows, - code=code + _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code ) diff --git a/dbt/adapters/mysql/impl.py b/dbt/adapters/mysql/impl.py index b599008..39d385d 100644 --- a/dbt/adapters/mysql/impl.py +++ b/dbt/adapters/mysql/impl.py @@ -33,8 +33,7 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, - col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -45,8 +44,7 @@ def list_relations_without_caching( # type: ignore[override] ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, - kwargs=kwargs) + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") if f"MySQL database '{schema_relation}' not found" in errmsg: @@ -116,8 +114,7 @@ def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]] if len(schema_map) > 1: raise dbt.exceptions.CompilationError( - f"Expected only one database in get_catalog, found " - f"{list(schema_map)}" + f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) with executor(self.config) as tpe: @@ -145,8 +142,7 @@ def _get_one_catalog( ) -> agate.Table: if len(schemas) != 1: raise dbt.exceptions.CompilationError( - f"Expected only one schema in mysql _get_one_catalog, found " - f"{schemas}" + f"Expected only one schema in mysql _get_one_catalog, found " f"{schemas}" ) database = information_schema.database @@ -156,8 +152,7 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) # type: ignore[arg-type] - return agate.Table.from_object(columns, - column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index 0c8254a..a9c58e9 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -165,7 +165,5 @@ def get_response(cls, cursor) -> AdapterResponse: # the mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), - rows_affected=num_rows, - code=code + _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code ) diff --git a/dbt/adapters/mysql5/impl.py b/dbt/adapters/mysql5/impl.py index 2de83ed..88675b5 100644 --- a/dbt/adapters/mysql5/impl.py +++ b/dbt/adapters/mysql5/impl.py @@ -33,8 +33,7 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, - col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -45,8 +44,7 @@ def list_relations_without_caching( # type: ignore[override] ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, - kwargs=kwargs) + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") if f"MySQL database '{schema_relation}' not found" in errmsg: @@ -116,8 +114,7 @@ def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]] if len(schema_map) > 1: raise dbt.exceptions.CompilationError( - f"Expected only one database in get_catalog, found " - f"{list(schema_map)}" + f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) with executor(self.config) as tpe: @@ -145,8 +142,7 @@ def _get_one_catalog( ) -> agate.Table: if len(schemas) != 1: raise dbt.exceptions.CompilationError( - f"Expected only one schema in mysql5 _get_one_catalog, found " - f"{schemas}" + f"Expected only one schema in mysql5 _get_one_catalog, found " f"{schemas}" ) database = information_schema.database @@ -156,8 +152,7 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) # type: ignore[arg-type] - return agate.Table.from_object(columns, - column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) From 147cd7da31a298ad730c6575371b7621df8073af Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 15:22:39 -0700 Subject: [PATCH 11/24] Add black commit to .git-blame-ignore-revs --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 4323bf5..24654af 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,2 +1,4 @@ # Ran `black` on all files 99a125c82b846fd25aed432ed67a9ad982bbe0ad +# Ran `black` on all files +f0507baaa4279575831ad06452757666d25bd90b From 870bbbf3c55081c436da7acb67a28e05c7484a44 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 15:23:32 -0700 Subject: [PATCH 12/24] Run black formatter on all files --- dbt/adapters/mariadb/relation.py | 6 ++---- dbt/adapters/mysql/relation.py | 6 ++---- dbt/adapters/mysql5/relation.py | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/dbt/adapters/mariadb/relation.py b/dbt/adapters/mariadb/relation.py index fd1e883..c8f1a24 100644 --- a/dbt/adapters/mariadb/relation.py +++ b/dbt/adapters/mariadb/relation.py @@ -20,10 +20,8 @@ class MariaDBIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MariaDBRelation(BaseRelation): - quote_policy: MariaDBQuotePolicy = field( - default_factory=lambda: MariaDBQuotePolicy()) - include_policy: MariaDBIncludePolicy = field( - default_factory=lambda: MariaDBIncludePolicy()) + quote_policy: MariaDBQuotePolicy = field(default_factory=lambda: MariaDBQuotePolicy()) + include_policy: MariaDBIncludePolicy = field(default_factory=lambda: MariaDBIncludePolicy()) quote_character: str = "`" def __post_init__(self): diff --git a/dbt/adapters/mysql/relation.py b/dbt/adapters/mysql/relation.py index 5170dbd..734a785 100644 --- a/dbt/adapters/mysql/relation.py +++ b/dbt/adapters/mysql/relation.py @@ -20,10 +20,8 @@ class MySQLIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MySQLRelation(BaseRelation): - quote_policy: MySQLQuotePolicy = field( - default_factory=lambda: MySQLQuotePolicy()) - include_policy: MySQLIncludePolicy = field( - default_factory=lambda: MySQLIncludePolicy()) + quote_policy: MySQLQuotePolicy = field(default_factory=lambda: MySQLQuotePolicy()) + include_policy: MySQLIncludePolicy = field(default_factory=lambda: MySQLIncludePolicy()) quote_character: str = "`" def __post_init__(self): diff --git a/dbt/adapters/mysql5/relation.py b/dbt/adapters/mysql5/relation.py index c7a150e..a94b446 100644 --- a/dbt/adapters/mysql5/relation.py +++ b/dbt/adapters/mysql5/relation.py @@ -20,10 +20,8 @@ class MySQLIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MySQLRelation(BaseRelation): - quote_policy: MySQLQuotePolicy = field( - default_factory=lambda: MySQLQuotePolicy()) - include_policy: MySQLIncludePolicy = field( - default_factory=lambda: MySQLIncludePolicy()) + quote_policy: MySQLQuotePolicy = field(default_factory=lambda: MySQLQuotePolicy()) + include_policy: MySQLIncludePolicy = field(default_factory=lambda: MySQLIncludePolicy()) quote_character: str = "`" def __post_init__(self): From a74fe8264bbd6820dd5a5f20e2e3dd0bdf42f1a1 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 15:24:02 -0700 Subject: [PATCH 13/24] Add black commit to .git-blame-ignore-revs --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 24654af..907a6cc 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -2,3 +2,5 @@ 99a125c82b846fd25aed432ed67a9ad982bbe0ad # Ran `black` on all files f0507baaa4279575831ad06452757666d25bd90b +# Ran `black` on all files +870bbbf3c55081c436da7acb67a28e05c7484a44 From fc06ff22ea5db5726ba7d990a77330b37d4fc49d Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 15:30:30 -0700 Subject: [PATCH 14/24] Fix MyPy & black errors --- dbt/adapters/mariadb/connections.py | 4 ++-- dbt/adapters/mariadb/impl.py | 3 +-- dbt/adapters/mysql/connections.py | 4 ++-- dbt/adapters/mysql/impl.py | 3 +-- dbt/adapters/mysql5/connections.py | 4 ++-- dbt/adapters/mysql5/impl.py | 3 +-- tests/functional/adapter/constraints/test_constraints.py | 4 +--- tests/unit/utils.py | 4 +--- 8 files changed, 11 insertions(+), 18 deletions(-) diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index 860afd0..5cc5397 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -10,7 +10,7 @@ from dbt.contracts.connection import Credentials from dbt.events import AdapterLogger from dataclasses import dataclass -from typing import Optional +from typing import Optional, Union logger = AdapterLogger("mysql") @@ -176,7 +176,7 @@ def get_response(cls, cursor) -> AdapterResponse: ) @classmethod - def data_type_code_to_name(cls, type_code: int) -> str: + def data_type_code_to_name(cls, type_code: Union[int, str]) -> str: field_type_values = mysql.connector.constants.FieldType.desc.values() mapping = {code: name for (code, name) in field_type_values} return mapping[type_code] diff --git a/dbt/adapters/mariadb/impl.py b/dbt/adapters/mariadb/impl.py index 77ac486..b08527e 100644 --- a/dbt/adapters/mariadb/impl.py +++ b/dbt/adapters/mariadb/impl.py @@ -168,8 +168,7 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", str(relation)) columns.extend(self._get_columns_for_catalog(relation)) # type: ignore[arg-type] - return agate.Table.from_object(columns, - column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 5587f26..cf23701 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -10,7 +10,7 @@ from dbt.contracts.connection import Credentials from dbt.events import AdapterLogger from dataclasses import dataclass -from typing import Optional +from typing import Optional, Union logger = AdapterLogger("mysql") @@ -172,7 +172,7 @@ def get_response(cls, cursor) -> AdapterResponse: ) @classmethod - def data_type_code_to_name(cls, type_code: int) -> str: + def data_type_code_to_name(cls, type_code: Union[int, str]) -> str: field_type_values = mysql.connector.constants.FieldType.desc.values() mapping = {code: name for (code, name) in field_type_values} return mapping[type_code] diff --git a/dbt/adapters/mysql/impl.py b/dbt/adapters/mysql/impl.py index 696e9ae..9029e86 100644 --- a/dbt/adapters/mysql/impl.py +++ b/dbt/adapters/mysql/impl.py @@ -168,8 +168,7 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", str(relation)) columns.extend(self._get_columns_for_catalog(relation)) # type: ignore[arg-type] - return agate.Table.from_object(columns, - column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index 050596e..c370c5a 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -10,7 +10,7 @@ from dbt.contracts.connection import Credentials from dbt.events import AdapterLogger from dataclasses import dataclass -from typing import Optional +from typing import Optional, Union logger = AdapterLogger("mysql") @@ -176,7 +176,7 @@ def get_response(cls, cursor) -> AdapterResponse: ) @classmethod - def data_type_code_to_name(cls, type_code: int) -> str: + def data_type_code_to_name(cls, type_code: Union[int, str]) -> str: field_type_values = mysql.connector.constants.FieldType.desc.values() mapping = {code: name for (code, name) in field_type_values} return mapping[type_code] diff --git a/dbt/adapters/mysql5/impl.py b/dbt/adapters/mysql5/impl.py index a7beaf9..59cc362 100644 --- a/dbt/adapters/mysql5/impl.py +++ b/dbt/adapters/mysql5/impl.py @@ -168,8 +168,7 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", str(relation)) columns.extend(self._get_columns_for_catalog(relation)) # type: ignore[arg-type] - return agate.Table.from_object(columns, - column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}) diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py index 78b6710..7e433db 100644 --- a/tests/functional/adapter/constraints/test_constraints.py +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -63,9 +63,7 @@ class TestMySQLTableConstraintsColumnsEqual( pass -class TestMySQLViewConstraintsColumnsEqual( - MySQLColumnEqualSetup, BaseViewConstraintsColumnsEqual -): +class TestMySQLViewConstraintsColumnsEqual(MySQLColumnEqualSetup, BaseViewConstraintsColumnsEqual): pass diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 74d8483..7a9cadb 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -78,9 +78,7 @@ def project_from_dict(project, profile, packages=None, selectors=None, cli_vars= return partial.render(renderer) -def config_from_parts_or_dicts( - project, profile, packages=None, selectors=None, cli_vars={} -): +def config_from_parts_or_dicts(project, profile, packages=None, selectors=None, cli_vars={}): from dbt.config import Project, Profile, RuntimeConfig from copy import deepcopy From befc595d02612ceff1bf2ab4ad105dc49ee7fd23 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 15:38:21 -0700 Subject: [PATCH 15/24] Make the `database` config field nullable again --- dbt/adapters/mariadb/connections.py | 2 +- dbt/adapters/mysql/connections.py | 2 +- dbt/adapters/mysql5/connections.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index 879b260..c67a56a 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -18,7 +18,7 @@ class MariaDBCredentials(Credentials): server: str = "" port: Optional[int] = None - database: str = "" + database: Optional[str] = None schema: str = "" username: Optional[str] = None password: Optional[str] = None diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 087aa4a..8c00e6b 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -18,7 +18,7 @@ class MySQLCredentials(Credentials): server: str = "" port: Optional[int] = None - database: str = "" + database: Optional[str] = None schema: str = "" username: Optional[str] = None password: Optional[str] = None diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index e6c6ed8..2eb7e96 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -18,7 +18,7 @@ class MySQLCredentials(Credentials): server: str = "" port: Optional[int] = None - database: str = "" + database: Optional[str] = None schema: str = "" username: Optional[str] = None password: Optional[str] = None From 3ab4e1a03a245452a962de06f1b8041cbef999e5 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 16:24:46 -0700 Subject: [PATCH 16/24] Fix mypy --- dbt/adapters/mariadb/connections.py | 2 +- dbt/adapters/mysql/connections.py | 2 +- dbt/adapters/mysql5/connections.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index c67a56a..af2de84 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -18,7 +18,7 @@ class MariaDBCredentials(Credentials): server: str = "" port: Optional[int] = None - database: Optional[str] = None + database: Optional[str] = None # type: ignore[assignment] schema: str = "" username: Optional[str] = None password: Optional[str] = None diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 8c00e6b..7ddec32 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -18,7 +18,7 @@ class MySQLCredentials(Credentials): server: str = "" port: Optional[int] = None - database: Optional[str] = None + database: Optional[str] = None # type: ignore[assignment] schema: str = "" username: Optional[str] = None password: Optional[str] = None diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index 2eb7e96..b884079 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -18,7 +18,7 @@ class MySQLCredentials(Credentials): server: str = "" port: Optional[int] = None - database: Optional[str] = None + database: Optional[str] = None # type: ignore[assignment] schema: str = "" username: Optional[str] = None password: Optional[str] = None From 5c2ed3dec5c60aad4134befc22055c6cde60d5eb Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 16:29:34 -0700 Subject: [PATCH 17/24] Bump version to 1.6 --- .bumpversion.toml | 2 +- dbt/adapters/mariadb/__version__.py | 2 +- dbt/adapters/mysql/__version__.py | 2 +- dbt/adapters/mysql5/__version__.py | 2 +- dev-requirements.txt | 4 ++-- setup.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 9e3c8eb..797e0d9 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "1.5.0a1" +current_version = "1.6.0a1" parse = ''' (?P[\d]+) # major version number \.(?P[\d]+) # minor version number diff --git a/dbt/adapters/mariadb/__version__.py b/dbt/adapters/mariadb/__version__.py index 219c289..07fc02e 100644 --- a/dbt/adapters/mariadb/__version__.py +++ b/dbt/adapters/mariadb/__version__.py @@ -1 +1 @@ -version = "1.5.0a1" +version = "1.6.0a1" diff --git a/dbt/adapters/mysql/__version__.py b/dbt/adapters/mysql/__version__.py index 219c289..07fc02e 100644 --- a/dbt/adapters/mysql/__version__.py +++ b/dbt/adapters/mysql/__version__.py @@ -1 +1 @@ -version = "1.5.0a1" +version = "1.6.0a1" diff --git a/dbt/adapters/mysql5/__version__.py b/dbt/adapters/mysql5/__version__.py index 219c289..07fc02e 100644 --- a/dbt/adapters/mysql5/__version__.py +++ b/dbt/adapters/mysql5/__version__.py @@ -1 +1 @@ -version = "1.5.0a1" +version = "1.6.0a1" diff --git a/dev-requirements.txt b/dev-requirements.txt index 0f0e60c..cac7d99 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@1.5.latest#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@1.5.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@1.6.latest#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@1.6.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor diff --git a/setup.py b/setup.py index 32fb208..76fa1d3 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ def _get_dbt_core_version(): package_name = "dbt-mysql" -package_version = "1.5.0a1" +package_version = "1.6.0a1" dbt_core_version = _get_dbt_core_version() description = """The MySQL adapter plugin for dbt""" From 8b3246197bc4a07ec5aaa2e1112f5387775153de Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 16:39:44 -0700 Subject: [PATCH 18/24] Bump version to 1.7 --- .bumpversion.toml | 2 +- dbt/adapters/mariadb/__version__.py | 2 +- dbt/adapters/mysql/__version__.py | 2 +- dbt/adapters/mysql5/__version__.py | 2 +- dev-requirements.txt | 4 ++-- setup.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 797e0d9..7c1129d 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "1.6.0a1" +current_version = "1.7.0a1" parse = ''' (?P[\d]+) # major version number \.(?P[\d]+) # minor version number diff --git a/dbt/adapters/mariadb/__version__.py b/dbt/adapters/mariadb/__version__.py index 07fc02e..874bd74 100644 --- a/dbt/adapters/mariadb/__version__.py +++ b/dbt/adapters/mariadb/__version__.py @@ -1 +1 @@ -version = "1.6.0a1" +version = "1.7.0a1" diff --git a/dbt/adapters/mysql/__version__.py b/dbt/adapters/mysql/__version__.py index 07fc02e..874bd74 100644 --- a/dbt/adapters/mysql/__version__.py +++ b/dbt/adapters/mysql/__version__.py @@ -1 +1 @@ -version = "1.6.0a1" +version = "1.7.0a1" diff --git a/dbt/adapters/mysql5/__version__.py b/dbt/adapters/mysql5/__version__.py index 07fc02e..874bd74 100644 --- a/dbt/adapters/mysql5/__version__.py +++ b/dbt/adapters/mysql5/__version__.py @@ -1 +1 @@ -version = "1.6.0a1" +version = "1.7.0a1" diff --git a/dev-requirements.txt b/dev-requirements.txt index cac7d99..161b0a0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@1.6.latest#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@1.6.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@1.7.latest#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@1.7.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor diff --git a/setup.py b/setup.py index 76fa1d3..f257d6f 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ def _get_dbt_core_version(): package_name = "dbt-mysql" -package_version = "1.6.0a1" +package_version = "1.7.0a1" dbt_core_version = _get_dbt_core_version() description = """The MySQL adapter plugin for dbt""" From 53e6c19621bf983d1c594e3fcdde787076376367 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Tue, 16 Jan 2024 16:48:16 -0700 Subject: [PATCH 19/24] Fix import error --- tests/unit/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 7a9cadb..e246138 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -7,7 +7,6 @@ from unittest import mock from unittest import TestCase -from hologram import ValidationError from dbt.config.project import PartialProject @@ -146,13 +145,6 @@ def assert_symmetric(self, obj, dct, cls=None): self.assert_to_dict(obj, dct) self.assert_from_dict(obj, dct, cls) - def assert_fails_validation(self, dct, cls=None): - if cls is None: - cls = self.ContractType - - with self.assertRaises(ValidationError): - cls.from_dict(dct) - def generate_name_macros(package): from dbt.contracts.graph.parsed import ParsedMacro From dcb472e3ddc54524333a591da4dacb01e203630a Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Fri, 26 Apr 2024 08:57:43 -0600 Subject: [PATCH 20/24] Fix unicode decode error (#169) * Update setup.py Explicitly defining utf-8 encoding to avoid UnicodeDecodeError on setup.py in Windows environment * Update CHANGELOG.md --------- Co-authored-by: Scott Gunn <99990724+sagunn-echo@users.noreply.github.com> --- CHANGELOG.md | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0186095..43451a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,13 @@ ### Fixes - Fix incremental composite keys ([#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) +- Fix UnicodeDecodeErorr on setup.py ([#160](https://github.com/dbeatty10/dbt-mysql/issues/160)) ### Contributors - [@moszutij](https://github.com/moszutij) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146), [#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) - [@wesen](https://github.com/wesen) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) - [@mwallace582](https://github.com/mwallace582) ([#162](https://github.com/dbeatty10/dbt-mysql/pull/162), [#138](https://github.com/dbeatty10/dbt-mysql/issues/138)) +- [@sagunn-echo](https://github.com/sagunn-echo) ([#160](https://github.com/dbeatty10/dbt-mysql/issues/160)) ## dbt-mysql 1.1.0 (Feb 5, 2023) diff --git a/setup.py b/setup.py index c176ec4..3e60e3d 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ # pull long description from README this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, "README.md")) as f: +with open(os.path.join(this_directory, "README.md"), encoding="utf-8") as f: long_description = f.read() From 1ad3bf1bd11ff74d03431a00a1700a330069e749 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Fri, 26 Apr 2024 09:32:52 -0600 Subject: [PATCH 21/24] Pin jsonschema version to fix tests (#174) --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 161b0a0..c33b7d6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,6 +8,7 @@ git+https://github.com/dbt-labs/dbt-core.git@1.7.latest#egg=dbt-tests-adapter&su black~=22.12 bumpversion~=0.6.0 ddtrace~=2.3 +jsonschema<=4.17 flake8~=6.1 flaky~=3.7 freezegun~=1.3 From 149157d6bbbf8392453dc92e4c0dae481b6fd44b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85czkowski?= <65259263+kzajaczkowski@users.noreply.github.com> Date: Fri, 26 Apr 2024 17:55:13 +0200 Subject: [PATCH 22/24] Add collation support in profiles.yml / Fix incorrect collation for utf8mb4 (#173) * Add support for charset and collation in profile.yml * Change mysql-connect-python version * Update readme with new profile options * Update setup.py with less restricting mysql-connector-python version after code review Co-authored-by: Matthew Wallace * Add collation and charset support for mariadb and mysql5 --------- Co-authored-by: Matthew Wallace --- README.md | 6 +++++- dbt/adapters/mariadb/connections.py | 7 +++++++ dbt/adapters/mysql/connections.py | 7 +++++++ dbt/adapters/mysql5/connections.py | 7 +++++++ setup.py | 2 +- 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b901ce4..5a305a7 100644 --- a/README.md +++ b/README.md @@ -86,17 +86,21 @@ your_profile_name: username: your_mysql_username password: your_mysql_password ssl_disabled: True + charset: utf8mb4 + collation: utf8mb4_0900_ai_ci ``` | Option | Description | Required? | Example | | --------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------------ | ---------------------------------------------- | -| type | The specific adapter to use | Required | `mysql`, `mysql5` or `mariadb` | +| type | The specific adapter to use | Required | `mysql`, `mysql5` or `mariadb` | | server | The server (hostname) to connect to | Required | `yourorg.mysqlhost.com` | | port | The port to use | Optional | `3306` | | schema | Specify the schema (database) to build models into | Required | `analytics` | | username | The username to use to connect to the server | Required | `dbt_admin` | | password | The password to use for authenticating to the server | Required | `correct-horse-battery-staple` | | ssl_disabled | Set to enable or disable TLS connectivity to mysql5.x | Optional | `True` or `False` | +| charset | Specify charset to be used by a connection | Optional | `utf8mb4` | +| collation | Set to enable or disable TLS connectivity to mysql5.x | Optional | `utf8mb4_0900_ai_ci` | ### Notes diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index 94c9428..6e463de 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -26,6 +26,7 @@ class MariaDBCredentials(Credentials): password: Optional[str] = None charset: Optional[str] = None ssl_disabled: Optional[bool] = None + collation: Optional[str] = None _ALIASES = { "UID": "username", @@ -98,6 +99,12 @@ def open(cls, connection): if credentials.port: kwargs["port"] = credentials.port + if credentials.charset: + kwargs["charset"] = credentials.charset + + if credentials.collation: + kwargs["collation"] = credentials.collation + try: connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 353d5ce..0e2af00 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -25,6 +25,7 @@ class MySQLCredentials(Credentials): username: Optional[str] = None password: Optional[str] = None charset: Optional[str] = None + collation: Optional[str] = None _ALIASES = { "UID": "username", @@ -94,6 +95,12 @@ def open(cls, connection): if credentials.port: kwargs["port"] = credentials.port + if credentials.charset: + kwargs["charset"] = credentials.charset + + if credentials.collation: + kwargs["collation"] = credentials.collation + try: connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index 0160bd9..ee7cbc5 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -26,6 +26,7 @@ class MySQLCredentials(Credentials): password: Optional[str] = None charset: Optional[str] = None ssl_disabled: Optional[bool] = None + collation: Optional[str] = None _ALIASES = { "UID": "username", @@ -98,6 +99,12 @@ def open(cls, connection): if credentials.port: kwargs["port"] = credentials.port + if credentials.charset: + kwargs["charset"] = credentials.charset + + if credentials.collation: + kwargs["collation"] = credentials.collation + try: connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" diff --git a/setup.py b/setup.py index f257d6f..6d08cd4 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def _get_dbt_core_version(): include_package_data=True, install_requires=[ "dbt-core~={}".format(dbt_core_version), - "mysql-connector-python>=8.0.0,<8.1", + "mysql-connector-python>=8.0.0", ], zip_safe=False, classifiers=[ From ec1889cdf5c81873eb4e7f2913b5ea54a1d9badc Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Fri, 26 Apr 2024 09:26:36 -0600 Subject: [PATCH 23/24] Pin jsonschema version to fix tests --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 6fff712..bff3b49 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,6 +8,7 @@ git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-tests-adapter&su black~=22.12 bumpversion~=0.6.0 ddtrace~=2.3 +jsonschema<=4.17 flake8~=6.1 flaky~=3.7 freezegun~=1.3 From 05979edd2f3e8e6213a7ce2b01f74201c966f555 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Fri, 26 Apr 2024 10:18:29 -0600 Subject: [PATCH 24/24] Update black to resolve security alert https://github.com/dbeatty10/dbt-mysql/security/dependabot/5 --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0194aaf..d4f4356 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,7 +5,7 @@ git+https://github.com/dbt-labs/dbt-core.git@1.5.latest#egg=dbt-tests-adapter&su # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor -black~=22.12 +black~=24.3 bumpversion~=0.6.0 ddtrace~=2.3 jsonschema<=4.17