Merge branch 'update-to-dbt-core-1-5-9' of https://github.com/mwallac…

…e582/dbt-mysql into mwallace/support-dbt-1-5

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Features
 - Migrate CircleCI to GitHub Actions ([#120](https://github.com/dbeatty10/dbt-mysql/issues/120))
 - Support dbt v1.4 ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146))
+- Support dbt v1.5 ([#159](https://github.com/dbeatty10/dbt-mysql/pull/159))
 
 ### Fixes
 - Fix incremental composite keys ([#144](https://github.com/dbeatty10/dbt-mysql/issues/144))

dbt/adapters/mariadb/column.py

@@ -8,6 +8,14 @@
 
 @dataclass
 class MariaDBColumn(Column):
+    TYPE_LABELS = {
+        "STRING": "TEXT",
+        "VAR_STRING": "TEXT",
+        "LONG": "INTEGER",
+        "LONGLONG": "INTEGER",
+        "INT": "INTEGER",
+        "TIMESTAMP": "DATETIME",
+    }
     table_database: Optional[str] = None
     table_schema: Optional[str] = None
     table_name: Optional[str] = None

dbt/adapters/mariadb/connections.py

@@ -1,6 +1,7 @@
 from contextlib import contextmanager
 
 import mysql.connector
+import mysql.connector.constants
 
 import dbt.exceptions
 from dbt.adapters.sql import SQLConnectionManager
@@ -172,3 +173,9 @@ def get_response(cls, cursor) -> AdapterResponse:
             rows_affected=num_rows,
             code=code
         )
+
+    @classmethod
+    def data_type_code_to_name(cls, type_code: int) -> str:
+        field_type_values = mysql.connector.constants.FieldType.desc.values()
+        mapping = {code: name for (code, name) in field_type_values}
+        return mapping[type_code]

dbt/adapters/mariadb/impl.py

@@ -12,6 +12,8 @@
 from dbt.adapters.mariadb import MariaDBRelation
 from dbt.adapters.mariadb import MariaDBColumn
 from dbt.adapters.base import BaseRelation
+from dbt.contracts.graph.nodes import ConstraintType
+from dbt.adapters.base.impl import ConstraintSupport
 from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER
 from dbt.events import AdapterLogger
 from dbt.utils import executor
@@ -27,6 +29,19 @@ class MariaDBAdapter(SQLAdapter):
     Column = MariaDBColumn
     ConnectionManager = MariaDBConnectionManager
 
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.ENFORCED,
+        ConstraintType.not_null: ConstraintSupport.ENFORCED,
+        ConstraintType.unique: ConstraintSupport.ENFORCED,
+        ConstraintType.primary_key: ConstraintSupport.ENFORCED,
+        # While Foreign Keys are indeed supported, they're not supported in
+        # CREATE TABLE AS SELECT statements, which is what DBT uses.
+        #
+        # It is possible to use a `post-hook` to add a foreign key after the
+        # table is created.
+        ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,
+    }
+
     @classmethod
     def date_function(cls):
         return "current_date()"
@@ -36,7 +51,8 @@ def convert_datetime_type(cls, agate_table: agate.Table,
                               col_idx: int) -> str:
         return "timestamp"
 
-    def quote(self, identifier):
+    @classmethod
+    def quote(cls, identifier: str) -> str:
         return "`{}`".format(identifier)
 
     def list_relations_without_caching(
@@ -157,7 +173,7 @@ def _get_one_catalog(
 
         columns: List[Dict[str, Any]] = []
         for relation in self.list_relations(database, schema):
-            logger.debug("Getting table schema for relation {}", relation)
+            logger.debug("Getting table schema for relation {}", str(relation))
             columns.extend(self._get_columns_for_catalog(relation))
         return agate.Table.from_object(columns,
                                        column_types=DEFAULT_TYPE_TESTER)

dbt/adapters/mysql/column.py

@@ -8,6 +8,14 @@
 
 @dataclass
 class MySQLColumn(Column):
+    TYPE_LABELS = {
+        "STRING": "TEXT",
+        "VAR_STRING": "TEXT",
+        "LONG": "INTEGER",
+        "LONGLONG": "INTEGER",
+        "INT": "INTEGER",
+        "TIMESTAMP": "DATETIME",
+    }
     table_database: Optional[str] = None
     table_schema: Optional[str] = None
     table_name: Optional[str] = None

dbt/adapters/mysql/connections.py

@@ -1,6 +1,7 @@
 from contextlib import contextmanager
 
 import mysql.connector
+import mysql.connector.constants
 
 import dbt.exceptions
 from dbt.adapters.sql import SQLConnectionManager
@@ -168,3 +169,9 @@ def get_response(cls, cursor) -> AdapterResponse:
             rows_affected=num_rows,
             code=code
         )
+
+    @classmethod
+    def data_type_code_to_name(cls, type_code: int) -> str:
+        field_type_values = mysql.connector.constants.FieldType.desc.values()
+        mapping = {code: name for (code, name) in field_type_values}
+        return mapping[type_code]

dbt/adapters/mysql/impl.py

@@ -12,6 +12,8 @@
 from dbt.adapters.mysql import MySQLRelation
 from dbt.adapters.mysql import MySQLColumn
 from dbt.adapters.base import BaseRelation
+from dbt.contracts.graph.nodes import ConstraintType
+from dbt.adapters.base.impl import ConstraintSupport
 from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER
 from dbt.events import AdapterLogger
 from dbt.utils import executor
@@ -27,6 +29,19 @@ class MySQLAdapter(SQLAdapter):
     Column = MySQLColumn
     ConnectionManager = MySQLConnectionManager
 
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.ENFORCED,
+        ConstraintType.not_null: ConstraintSupport.ENFORCED,
+        ConstraintType.unique: ConstraintSupport.ENFORCED,
+        ConstraintType.primary_key: ConstraintSupport.ENFORCED,
+        # While Foreign Keys are indeed supported, they're not supported in
+        # CREATE TABLE AS SELECT statements, which is what DBT uses.
+        #
+        # It is possible to use a `post-hook` to add a foreign key after the
+        # table is created.
+        ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,
+    }
+
     @classmethod
     def date_function(cls):
         return "current_date()"
@@ -36,7 +51,8 @@ def convert_datetime_type(cls, agate_table: agate.Table,
                               col_idx: int) -> str:
         return "timestamp"
 
-    def quote(self, identifier):
+    @classmethod
+    def quote(cls, identifier: str) -> str:
         return "`{}`".format(identifier)
 
     def list_relations_without_caching(
@@ -157,7 +173,7 @@ def _get_one_catalog(
 
         columns: List[Dict[str, Any]] = []
         for relation in self.list_relations(database, schema):
-            logger.debug("Getting table schema for relation {}", relation)
+            logger.debug("Getting table schema for relation {}", str(relation))
             columns.extend(self._get_columns_for_catalog(relation))
         return agate.Table.from_object(columns,
                                        column_types=DEFAULT_TYPE_TESTER)

dbt/adapters/mysql5/column.py

@@ -8,6 +8,14 @@
 
 @dataclass
 class MySQLColumn(Column):
+    TYPE_LABELS = {
+        "STRING": "TEXT",
+        "VAR_STRING": "TEXT",
+        "LONG": "INTEGER",
+        "LONGLONG": "INTEGER",
+        "INT": "INTEGER",
+        "TIMESTAMP": "DATETIME",
+    }
     table_database: Optional[str] = None
     table_schema: Optional[str] = None
     table_name: Optional[str] = None

dbt/adapters/mysql5/connections.py

@@ -1,6 +1,7 @@
 from contextlib import contextmanager
 
 import mysql.connector
+import mysql.connector.constants
 
 import dbt.exceptions
 from dbt.adapters.sql import SQLConnectionManager
@@ -172,3 +173,9 @@ def get_response(cls, cursor) -> AdapterResponse:
             rows_affected=num_rows,
             code=code
         )
+
+    @classmethod
+    def data_type_code_to_name(cls, type_code: int) -> str:
+        field_type_values = mysql.connector.constants.FieldType.desc.values()
+        mapping = {code: name for (code, name) in field_type_values}
+        return mapping[type_code]

dbt/adapters/mysql5/impl.py

@@ -12,6 +12,8 @@
 from dbt.adapters.mysql5 import MySQLRelation
 from dbt.adapters.mysql5 import MySQLColumn
 from dbt.adapters.base import BaseRelation
+from dbt.contracts.graph.nodes import ConstraintType
+from dbt.adapters.base.impl import ConstraintSupport
 from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER
 from dbt.events import AdapterLogger
 from dbt.utils import executor
@@ -27,6 +29,19 @@ class MySQLAdapter(SQLAdapter):
     Column = MySQLColumn
     ConnectionManager = MySQLConnectionManager
 
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,
+        ConstraintType.not_null: ConstraintSupport.ENFORCED,
+        ConstraintType.unique: ConstraintSupport.ENFORCED,
+        ConstraintType.primary_key: ConstraintSupport.ENFORCED,
+        # While Foreign Keys are indeed supported, they're not supported in
+        # CREATE TABLE AS SELECT statements, which is what DBT uses.
+        #
+        # It is possible to use a `post-hook` to add a foreign key after the
+        # table is created.
+        ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,
+    }
+
     @classmethod
     def date_function(cls):
         return "current_date()"
@@ -36,7 +51,8 @@ def convert_datetime_type(cls, agate_table: agate.Table,
                               col_idx: int) -> str:
         return "timestamp"
 
-    def quote(self, identifier):
+    @classmethod
+    def quote(cls, identifier: str) -> str:
         return "`{}`".format(identifier)
 
     def list_relations_without_caching(
@@ -156,7 +172,7 @@ def _get_one_catalog(
 
         columns: List[Dict[str, Any]] = []
         for relation in self.list_relations(database, schema):
-            logger.debug("Getting table schema for relation {}", relation)
+            logger.debug("Getting table schema for relation {}", str(relation))
             columns.extend(self._get_columns_for_catalog(relation))
         return agate.Table.from_object(columns,
                                        column_types=DEFAULT_TYPE_TESTER)

dbt/include/mariadb/macros/adapters.sql

@@ -39,14 +39,29 @@
 
   create {% if temporary: -%}temporary{%- endif %} table
     {{ relation.include(database=False) }}
-    {{ sql }}
+    {% set contract_config = config.get('contract') %}
+    {% if contract_config.enforced %}
+      {{ get_assert_columns_equivalent(sql) }}
+      {{ get_table_columns_and_constraints() }}
+      {%- set sql = get_select_subquery(sql) %}
+      (
+        {{ sql }}
+      )
+    {% else %}
+      {{ sql }}
+    {% endif %}
 {% endmacro %}
 
 {% macro mariadb__create_view_as(relation, sql) -%}
   {%- set sql_header = config.get('sql_header', none) -%}
 
   {{ sql_header if sql_header is not none }}
-  create view {{ relation }} as
+  create view {{ relation }}
+  {% set contract_config = config.get('contract') %}
+  {% if contract_config.enforced %}
+    {{ get_assert_columns_equivalent(sql) }}
+  {%- endif %}
+  as
     {{ sql }}
 {%- endmacro %}
 
@@ -101,3 +116,38 @@
 {% macro mariadb__generate_database_name(custom_database_name=none, node=none) -%}
   {% do return(None) %}
 {%- endmacro %}
+
+{% macro mariadb__get_phony_data_for_type(data_type) %}
+  {#
+    The types that MariaDB supports in CAST statements are NOT the same as the
+    types that are supported in table definitions. This is a bit of a hack to
+    work around the known mismatches.
+  #}
+  {%- if data_type.lower() == 'integer' -%}
+     0
+  {%- elif data_type.lower() == 'text' -%}
+     ''
+  {%- elif data_type.lower() == 'integer unsigned' -%}
+     cast(null as unsigned)
+  {%- elif data_type.lower() == 'integer signed' -%}
+     cast(null as signed)
+  {%- else -%}
+     cast(null as {{ data_type }})
+  {%- endif -%}
+{% endmacro %}
+
+{% macro mariadb__get_empty_schema_sql(columns) %}
+    {%- set col_err = [] -%}
+    select
+    {% for i in columns %}
+      {%- set col = columns[i] -%}
+      {%- if col['data_type'] is not defined -%}
+        {{ col_err.append(col['name']) }}
+      {%- endif -%}
+      {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}
+      {{ mariadb__get_phony_data_for_type(col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }}
+    {%- endfor -%}
+    {%- if (col_err | length) > 0 -%}
+      {{ exceptions.column_type_missing(column_names=col_err) }}
+    {%- endif -%}
+{% endmacro %}

dbt/include/mysql/macros/adapters.sql

@@ -39,9 +39,17 @@
 
   create {% if temporary: -%}temporary{%- endif %} table
     {{ relation.include(database=False) }}
-  as (
-    {{ sql }}
-  )
+    {% set contract_config = config.get('contract') %}
+    {% if contract_config.enforced %}
+      {{ get_assert_columns_equivalent(sql) }}
+      {{ get_table_columns_and_constraints() }}
+      {%- set sql = get_select_subquery(sql) %}
+    {% else %}
+      as
+    {% endif %}
+    (
+      {{ sql }}
+    )
 {% endmacro %}
 
 {% macro mysql__current_timestamp() -%}
@@ -95,3 +103,38 @@
 {% macro mysql__generate_database_name(custom_database_name=none, node=none) -%}
   {% do return(None) %}
 {%- endmacro %}
+
+{% macro mysql__get_phony_data_for_type(data_type) %}
+  {#
+    The types that MySQL supports in CAST statements are NOT the same as the
+    types that are supported in table definitions. This is a bit of a hack to
+    work around the known mismatches.
+  #}
+  {%- if data_type.lower() == 'integer' -%}
+     0
+  {%- elif data_type.lower() == 'text' -%}
+     ''
+  {%- elif data_type.lower() == 'integer unsigned' -%}
+     cast(null as unsigned)
+  {%- elif data_type.lower() == 'integer signed' -%}
+     cast(null as signed)
+  {%- else -%}
+     cast(null as {{ data_type }})
+  {%- endif -%}
+{% endmacro %}
+
+{% macro mysql__get_empty_schema_sql(columns) %}
+    {%- set col_err = [] -%}
+    select
+    {% for i in columns %}
+      {%- set col = columns[i] -%}
+      {%- if col['data_type'] is not defined -%}
+        {{ col_err.append(col['name']) }}
+      {%- endif -%}
+      {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}
+      {{ mysql__get_phony_data_for_type(col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }}
+    {%- endfor -%}
+    {%- if (col_err | length) > 0 -%}
+      {{ exceptions.column_type_missing(column_names=col_err) }}
+    {%- endif -%}
+{% endmacro %}