diff --git a/.changes/unreleased/Fixes-20250124-140300.yaml b/.changes/unreleased/Fixes-20250124-140300.yaml new file mode 100644 index 0000000000..aa2ec73478 --- /dev/null +++ b/.changes/unreleased/Fixes-20250124-140300.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix `mf tutorial` experience. +time: 2025-01-24T14:03:00.410289-08:00 +custom: + Author: plypaul + Issue: "1631" diff --git a/.changes/unreleased/Fixes-20250124-181618.yaml b/.changes/unreleased/Fixes-20250124-181618.yaml new file mode 100644 index 0000000000..eaad03b5f3 --- /dev/null +++ b/.changes/unreleased/Fixes-20250124-181618.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Compatibility Issue with dbt-core 1.9.0 and dbt-metricflow 0.7.1. +time: 2025-01-24T18:16:18.536201-08:00 +custom: + Author: plypaul + Issue: "1589" diff --git a/.changes/unreleased/Fixes-20250124-181706.yaml b/.changes/unreleased/Fixes-20250124-181706.yaml new file mode 100644 index 0000000000..272f10135f --- /dev/null +++ b/.changes/unreleased/Fixes-20250124-181706.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: dbt-core dependency issue with metricflow==0.207.0. +time: 2025-01-24T18:17:06.936807-08:00 +custom: + Author: plypaul + Issue: "1632" diff --git a/.github/actions/setup-python-env/action.yaml b/.github/actions/setup-python-env/action.yaml index 92da587441..cad2c19865 100644 --- a/.github/actions/setup-python-env/action.yaml +++ b/.github/actions/setup-python-env/action.yaml @@ -5,7 +5,7 @@ inputs: python-version: description: "Version of Python to use for testing" required: false - default: "3.8" + default: "3.9" hatch-environment-cache-config-json: description: "Configuration JSON to be passed into the script to install `hatch` environments for caching." required: false diff --git a/.github/workflows/cd-push-dbt-metricflow-to-pypi.yaml b/.github/workflows/cd-push-dbt-metricflow-to-pypi.yaml index b8f94f53f9..93012ac732 100644 --- a/.github/workflows/cd-push-dbt-metricflow-to-pypi.yaml +++ b/.github/workflows/cd-push-dbt-metricflow-to-pypi.yaml @@ -7,7 +7,7 @@ on: - "dbt-metricflow/v[0-9]+.[0-9]+.[0-9]+*" env: - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" jobs: pypi-publish: diff --git a/.github/workflows/cd-push-metricflow-to-pypi.yaml b/.github/workflows/cd-push-metricflow-to-pypi.yaml index a5d9080555..f64dab0c5e 100644 --- a/.github/workflows/cd-push-metricflow-to-pypi.yaml +++ b/.github/workflows/cd-push-metricflow-to-pypi.yaml @@ -8,7 +8,7 @@ on: - "v[0-9]+.[0-9]+.[0-9]+*" env: - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" jobs: pypi-publish: diff --git a/dbt-metricflow/dbt_metricflow/__about__.py b/dbt-metricflow/dbt_metricflow/__about__.py index dd2d37b852..eb7f82b1f7 100644 --- a/dbt-metricflow/dbt_metricflow/__about__.py +++ b/dbt-metricflow/dbt_metricflow/__about__.py @@ -1,3 +1,3 @@ from __future__ import annotations -__version__ = "0.7.0" +__version__ = "0.8.0" diff --git a/dbt-metricflow/dbt_metricflow/cli/__init__.py b/dbt-metricflow/dbt_metricflow/cli/__init__.py index e83dcca000..52afecf18d 100644 --- a/dbt-metricflow/dbt_metricflow/cli/__init__.py +++ b/dbt-metricflow/dbt_metricflow/cli/__init__.py @@ -1,3 +1,3 @@ from __future__ import annotations -PACKAGE_NAME = "metricflow" +PACKAGE_NAME = "dbt-metricflow" diff --git a/dbt-metricflow/dbt_metricflow/cli/cli_context.py b/dbt-metricflow/dbt_metricflow/cli/cli_configuration.py similarity index 76% rename from dbt-metricflow/dbt_metricflow/cli/cli_context.py rename to dbt-metricflow/dbt_metricflow/cli/cli_configuration.py index 35016f4607..1e627e2399 100644 --- a/dbt-metricflow/dbt_metricflow/cli/cli_context.py +++ b/dbt-metricflow/dbt_metricflow/cli/cli_configuration.py @@ -9,6 +9,7 @@ from metricflow_semantics.mf_logging.lazy_formattable import LazyFormat from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup +from dbt_metricflow.cli import PACKAGE_NAME from dbt_metricflow.cli.dbt_connectors.adapter_backed_client import AdapterBackedSqlClient from dbt_metricflow.cli.dbt_connectors.dbt_config_accessor import dbtArtifacts, dbtProjectMetadata from metricflow.engine.metricflow_engine import MetricFlowEngine @@ -17,24 +18,47 @@ logger = logging.getLogger(__name__) -class CLIContext: - """Context for the MetricFlow CLI.""" +class CLIConfiguration: + """Configuration object used for the MetricFlow CLI.""" - def __init__(self) -> None: - """Initialize the CLI context for executing commands. - - The dbt_artifacts construct must be loaded in order for logging configuration to work correctly. - """ + def __init__(self) -> None: # noqa: D107 self.verbose = False - self._dbt_project_metadata: dbtProjectMetadata = dbtProjectMetadata.load_from_project_path(pathlib.Path.cwd()) + self._dbt_project_metadata: Optional[dbtProjectMetadata] = None self._dbt_artifacts: Optional[dbtArtifacts] = None self._mf: Optional[MetricFlowEngine] = None self._sql_client: Optional[SqlClient] = None self._semantic_manifest: Optional[SemanticManifest] = None self._semantic_manifest_lookup: Optional[SemanticManifestLookup] = None - # self.log_file_path invokes the dbtRunner. If this is done after the configure_logging call all of the - # dbt CLI logging configuration could be overridden, resulting in lots of things printing to console - self._configure_logging(log_file_path=self.log_file_path) + + def setup(self) -> None: + """Setup this configuration for executing commands. + + The dbt_artifacts construct must be loaded in order for logging configuration to work correctly. + """ + dbt_project_path = pathlib.Path.cwd() + try: + self._dbt_project_metadata = dbtProjectMetadata.load_from_project_path(dbt_project_path) + + # self.log_file_path invokes the dbtRunner. If this is done after the configure_logging call all of the + # dbt CLI logging configuration could be overridden, resulting in lots of things printing to console + self._configure_logging(log_file_path=self.log_file_path) + except Exception as e: + exception_message = str(e) + if exception_message.find("Could not find adapter type") != -1: + raise RuntimeError( + f"Got an error during setup, potentially due to a missing adapter package. Has the appropriate " + f"adapter package (`{PACKAGE_NAME}[dbt-*]`) for the dbt project {str(dbt_project_path)!r} been " + f"installed? If not please install it (e.g. `pip install '{PACKAGE_NAME}[dbt-duckdb]')." + ) from e + else: + raise e + + def _get_dbt_project_metadata(self) -> dbtProjectMetadata: + if self._dbt_project_metadata is None: + raise RuntimeError( + f"{self.__class__.__name__}.setup() should have been called before accessing the configuration." + ) + return self._dbt_project_metadata def _configure_logging(self, log_file_path: pathlib.Path) -> None: """Initialize the logging spec for the CLI. @@ -70,13 +94,13 @@ def _configure_logging(self, log_file_path: pathlib.Path) -> None: @property def dbt_project_metadata(self) -> dbtProjectMetadata: """Property accessor for dbt project metadata, useful in cases where the full manifest load is not needed.""" - return self._dbt_project_metadata + return self._get_dbt_project_metadata() @property def dbt_artifacts(self) -> dbtArtifacts: """Property accessor for all dbt artifacts, used for powering the sql client (among other things).""" if self._dbt_artifacts is None: - self._dbt_artifacts = dbtArtifacts.load_from_project_metadata(self._dbt_project_metadata) + self._dbt_artifacts = dbtArtifacts.load_from_project_metadata(self.dbt_project_metadata) return self._dbt_artifacts @property @@ -85,7 +109,7 @@ def log_file_path(self) -> pathlib.Path: # The dbt Project.log_path attribute is currently sourced from the final runtime config value accessible # through the CLI state flags. As such, it will deviate from the default based on the DBT_LOG_PATH environment # variable. Should this behavior change, we will need to update this call. - return pathlib.Path(self._dbt_project_metadata.project.log_path, "metricflow.log") + return pathlib.Path(self.dbt_project_metadata.project.log_path, "metricflow.log") @property def sql_client(self) -> SqlClient: diff --git a/dbt-metricflow/dbt_metricflow/cli/main.py b/dbt-metricflow/dbt_metricflow/cli/main.py index 7c248654bc..8643da58cf 100644 --- a/dbt-metricflow/dbt_metricflow/cli/main.py +++ b/dbt-metricflow/dbt_metricflow/cli/main.py @@ -2,7 +2,9 @@ import csv as csv_module import datetime as dt +import importlib.metadata import logging +import os import pathlib import signal import sys @@ -15,6 +17,7 @@ import click import jinja2 +import packaging.version from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest from dbt_semantic_interfaces.validations.semantic_manifest_validator import SemanticManifestValidator from dbt_semantic_interfaces.validations.validator_helpers import SemanticManifestValidationResults @@ -25,7 +28,7 @@ import dbt_metricflow.cli.custom_click_types as click_custom from dbt_metricflow.cli import PACKAGE_NAME -from dbt_metricflow.cli.cli_context import CLIContext +from dbt_metricflow.cli.cli_configuration import CLIConfiguration from dbt_metricflow.cli.constants import DEFAULT_RESULT_DECIMAL_PLACES, MAX_LIST_OBJECT_ELEMENTS from dbt_metricflow.cli.dbt_connectors.dbt_config_accessor import dbtArtifacts from dbt_metricflow.cli.tutorial import ( @@ -45,7 +48,7 @@ logger = logging.getLogger(__name__) -pass_config = click.make_pass_decorator(CLIContext, ensure=True) +pass_config = click.make_pass_decorator(CLIConfiguration, ensure=True) _telemetry_reporter = TelemetryReporter(report_levels_higher_or_equal_to=TelemetryLevel.USAGE) _telemetry_reporter.add_python_log_handler() @@ -53,10 +56,9 @@ @click.group() @click.option("-v", "--verbose", is_flag=True) @click.version_option() -@error_if_not_in_dbt_project @pass_config @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) -def cli(cfg: CLIContext, verbose: bool) -> None: # noqa: D103 +def cli(cfg: CLIConfiguration, verbose: bool) -> None: # noqa: D103 # Some HTTP logging callback somewhere is failing to close its SSL connections correctly. # For now, filter those warnings so they don't pop up in CLI stderr # note - this should be addressed as adapter connection issues might produce these as well @@ -103,47 +105,141 @@ def exit_signal_handler(signal_type: int, frame) -> None: # type: ignore @cli.command() -@click.option("-m", "--msg", is_flag=True, help="Output the final steps dialogue") +@click.option("-m", "--message", is_flag=True, help="Output the final steps dialogue") # @click.option("--skip-dw", is_flag=True, help="Skip the data warehouse health checks") # TODO: re-enable this @click.option("--clean", is_flag=True, help="Remove sample model files.") +@click.option("--yes", is_flag=True, help="Respond yes to all questions (for scripting).") @pass_config @click.pass_context @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) -def tutorial(ctx: click.core.Context, cfg: CLIContext, msg: bool, clean: bool) -> None: +def tutorial(ctx: click.core.Context, cfg: CLIConfiguration, message: bool, clean: bool, yes: bool) -> None: """Run user through a tutorial.""" + # Needed to handle the backslash outside f-string + complex_query = ( + """mf query \\ + --metrics transactions,transaction_usd_na \\ + --group-by metric_time,transaction__is_large \\ + --order metric_time \\ + --start-time 2022-03-20 --end-time 2022-04-01 + """ + ).rstrip() + # dbt_core_version = importlib.metadata.version("dbt-core") + dbt_core_package_name = "dbt-core" + dbt_core_version = None + try: + dbt_core_version = packaging.version.parse(importlib.metadata.version(dbt_core_package_name)) + except Exception as e: + click.secho( + textwrap.dedent( + f""" + Unable to determine the version of package {dbt_core_package_name!r}: + { str(e).splitlines()[0]} + Displayed links to dbt docs may not reflect the correct version for your installed version of {dbt_core_package_name!r} + """ + ), + fg="yellow", + ) + time_spine_docs_link = "https://docs.getdbt.com/docs/build/metricflow-time-spine" + if dbt_core_version is not None: + time_spine_docs_link = time_spine_docs_link + f"?version={dbt_core_version.major}.{dbt_core_version.minor}" help_msg = textwrap.dedent( - """\ - πŸ€“ Please run the following steps, - 1. Verify that your adapter credentials are correct in `profiles.yml` - 2. Add time spine model to the models directory (https://docs.getdbt.com/docs/build/metricflow-time-spine) - 3. Run `dbt seed`, check to see that the steps related to countries, transactions, customers are passing. - 4. Try validating your data model: `mf validate-configs` - 5. Check out your metrics: `mf list metrics` - 6. Check out dimensions for your metric `mf list dimensions --metrics transactions` - 7. Query your first metric: `mf query --metrics transactions --group-by metric_time --order metric_time` - 8. Show the SQL MetricFlow generates: - `mf query --metrics transactions --group-by metric_time --order metric_time --explain` - 9. Visualize the plan: - `mf query --metrics transactions --group-by metric_time --order metric_time --explain --display-plans` - * This only works if you have graphviz installed - see README. - 10. Add another dimension: - `mf query --metrics transactions --group-by metric_time,customer__customer_country --order metric_time` - 11. Add a coarser time granularity: - `mf query --metrics transactions --group-by metric_time__week --order metric_time__week` - 12. Try a more complicated query: mf query --metrics transactions,transaction_usd_na --group-by metric_time,is_large --order metric_time --start-time 2022-03-20 --end-time 2022-04-01. - 13. When you're done with the tutorial, run mf tutorial --clean to delete sample models and seeds. + f"""\ + πŸ€“ {click.style("Please run the following steps:", bold=True)} + + 1. If you're using the tutorial-generated dbt project, switch to the root directory of the project. + 2. Otherwise, if you're using your own dbt project: + * Verify that your adapter credentials are correct in `profiles.yml`. + * Add a time spine model to the model directory. See (try +Left Click on the link): + {click.style(time_spine_docs_link, fg="blue", bold=True)} + 3. Run {click.style("`dbt seed`", bold=True)} and check that the steps related to countries, transactions, customers are passing. + 4. Run {click.style("`dbt build`", bold=True)} to produce the model tables. + 4. Try validating your data model: {click.style("`mf validate-configs`", bold=True)} + 5. Check out your metrics: {click.style("`mf list metrics`", bold=True)} + 6. Check out dimensions for your metric {click.style("`mf list dimensions --metrics transactions`", bold=True)} + 7. Query your first metric: + {click.style("mf query --metrics transactions --group-by metric_time --order metric_time", bold=True)} + 8. Show the SQL MetricFlow generates: + {click.style("mf query --metrics transactions --group-by metric_time --order metric_time --explain", bold=True)} + 9. Visualize the plan: + {click.style("mf query --metrics transactions --group-by metric_time --order metric_time --explain --display-plans", bold=True)} + * This only works if you have graphviz installed - see README. + 10. Add another dimension: + {click.style("mf query --metrics transactions --group-by metric_time,customer__customer_country --order metric_time", bold=True)} + 11. Add a coarser time granularity: + {click.style("mf query --metrics transactions --group-by metric_time__week --order metric_time__week", bold=True)} + 12. Try a more complicated query: + {click.style(complex_query, bold=True)} + 13. When you're done with the tutorial, run mf tutorial --clean to delete sample models and seeds. + * If a sample project was created, it wil remain. """ ) - if msg: + if message: click.echo(help_msg) exit() + current_directory = pathlib.Path.cwd() + project_path = current_directory if not dbt_project_file_exists(): - click.echo( - "Unable to detect dbt project. Please ensure that your current working directory is at the root of the dbt project." + tutorial_project_name = "mf_tutorial_project" + + sample_dbt_project_path = (current_directory / tutorial_project_name).absolute() + click.secho( + "Unable to detect a dbt project. Please run `mf tutorial` from the root directory of your dbt project.", + fg="yellow", ) - exit() + yes or click.confirm( + textwrap.dedent( + f"""\ + + Alternatively, this tutorial can create a sample dbt project with a `profiles.yml` configured to + use DuckDB. This will allow you to run the tutorial as a self-contained experience. The sample project + will be created at: + + {click.style(str(sample_dbt_project_path), bold=True)} + + Do you want to create the sample project now? + """ + ).rstrip(), + abort=True, + ) + + _check_duckdb_package_installed_for_sample_project(yes) + + if dbtMetricFlowTutorialHelper.check_if_path_exists([sample_dbt_project_path]): + yes or click.confirm( + click.style( + textwrap.dedent( + f"""\ + + The path {str(sample_dbt_project_path)!r} already exists. + Do you want to overwrite it? + """ + ).rstrip(), + fg="yellow", + ), + abort=True, + ) + dbtMetricFlowTutorialHelper.remove_files(sample_dbt_project_path) + spinner = Halo(text=f"Generating {repr(tutorial_project_name)} files...", spinner="dots") + spinner.start() + + dbtMetricFlowTutorialHelper.generate_dbt_project(sample_dbt_project_path) + spinner.succeed("πŸ“¦ Sample dbt project has been generated.") + click.secho( + textwrap.dedent( + """\ + + Before running the steps in the tutorial, be sure to switch to the sample project directory. + """ + ), + bold=True, + ) + os.chdir(sample_dbt_project_path.as_posix()) + project_path = sample_dbt_project_path + + click.echo(f"Using the project in {str(project_path)!r}\n") + cfg.setup() # TODO: Health checks @@ -160,7 +256,7 @@ def tutorial(ctx: click.core.Context, cfg: CLIContext, msg: bool, clean: bool) - # Remove sample files from dbt project if clean: - click.confirm("Would you like to remove all the sample files?", abort=True) + yes or click.confirm("Would you like to remove all the sample files?", abort=True) spinner = Halo(text="Removing sample files...", spinner="dots") spinner.start() try: @@ -171,25 +267,32 @@ def tutorial(ctx: click.core.Context, cfg: CLIContext, msg: bool, clean: bool) - spinner.fail(f"❌ Unable to remove sample files.\nERROR: {str(e)}") exit(1) + # TODO: Why a JSON file for the manifest? click.echo( textwrap.dedent( f"""\ - To begin building and querying metrics, you must define semantic models and - metric configuration files in your dbt project. dbt will use these files to generate a - semantic manifest artifact, which MetricFlow will use to create a semantic graph for querying. - As part of this tutorial, we will generate the following files to help you get started: - - πŸ“œ model files -> {model_path.absolute().as_posix()} - 🌱 seed files -> {seed_path.absolute().as_posix()} - βœ… semantic manifest json file -> {manifest_path.absolute().as_posix()} - """ + To begin building and querying metrics, you must define semantic models and + metric configuration files in your dbt project. dbt will use these files to generate a + semantic manifest artifact, which MetricFlow will use to create a semantic graph for querying. + As part of this tutorial, we will generate the following files to help you get started: + + πŸ“œ Model Files + -> {model_path.absolute().as_posix()} + 🌱 Seed Files + -> {seed_path.absolute().as_posix()} + βœ… Semantic Manifest JSON File + -> {manifest_path.absolute().as_posix()} + """ ) ) - click.confirm("Continue and generate the files?", abort=True) + yes or click.confirm("Continue and generate the files?", abort=True) # Generate sample files into dbt project if dbtMetricFlowTutorialHelper.check_if_path_exists([model_path, seed_path]): - click.confirm("There are existing files in the paths above, would you like to overwrite them?", abort=True) + yes or click.confirm( + click.style("There are existing files in the paths above, would you like to overwrite them?", fg="yellow"), + abort=True, + ) dbtMetricFlowTutorialHelper.remove_sample_files(model_path=model_path, seed_path=seed_path) spinner = Halo(text="Generating sample files...", spinner="dots") @@ -200,11 +303,61 @@ def tutorial(ctx: click.core.Context, cfg: CLIContext, msg: bool, clean: bool) - spinner.succeed("πŸ“œ Sample files has been generated.") - click.echo(help_msg) - click.echo("πŸ’‘ Run `mf tutorial --msg` to see this message again without executing everything else") + click.echo("\n" + help_msg) + click.echo("πŸ’‘ Run `mf tutorial --message` to see this message again without executing everything else") exit() +def _check_duckdb_package_installed_for_sample_project(yes: bool) -> None: + """Check if the DuckDB adapter package is installed and prompt user to install it if not. + + If `yes` is set, the prompt to exit if the package is not installed will be skipped. + """ + click.echo( + textwrap.dedent( + """\ + + Since the sample project uses DuckDB, the `dbt-metricflow[dbt-duckdb]` package must be installed beforehand. + """ + ).rstrip(), + ) + + duckdb_adapter_package_name = "dbt-duckdb" + dbt_duckdb_package_version = None + try: + dbt_duckdb_package_version = importlib.metadata.version(duckdb_adapter_package_name) + except importlib.metadata.PackageNotFoundError: + pass + + if dbt_duckdb_package_version is not None: + click.secho( + f"* Detected installed package {duckdb_adapter_package_name!r} {dbt_duckdb_package_version}", + bold=True, + fg="green", + ) + return + + click.secho("* Did not detect package is installed", bold=True, fg="red") + + if yes: + return + + exit_tutorial = click.confirm( + textwrap.dedent( + """\ + + As the package was not detected as installed, it's recommended to install the package first before + generating the sample project. e.g. `pip install 'dbt-metricflow[dbt-duckdb]'`. Otherwise, there may be + errors that prevent you from generating the sample project and running the tutorial. + + Do you want to exit the tutorial so that you can install the package? + """ + ).rstrip(), + ) + if exit_tutorial: + exit(0) + + @cli.command() @query_options @click.option( @@ -252,9 +405,10 @@ def tutorial(ctx: click.core.Context, cfg: CLIContext, msg: bool, clean: bool) - ) @pass_config @exception_handler +@error_if_not_in_dbt_project @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) def query( - cfg: CLIContext, + cfg: CLIConfiguration, metrics: Optional[Sequence[str]] = None, group_by: Optional[Sequence[str]] = None, where: Optional[str] = None, @@ -271,6 +425,7 @@ def query( saved_query: Optional[str] = None, ) -> None: """Create a new query with MetricFlow and assembles a MetricFlowQueryResult.""" + cfg.setup() start = time.time() spinner = Halo(text="Initiating query…", spinner="dots") spinner.start() @@ -322,6 +477,7 @@ def query( else: click.echo( "πŸ”Ž SQL (remove --explain to see data or add --show-dataflow-plan to see the generated dataflow plan):" + "\n" ) click.echo(sql) if display_plans: @@ -356,8 +512,10 @@ def query( @cli.group() @pass_config @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) -def list(cfg: CLIContext) -> None: +@error_if_not_in_dbt_project +def list(cfg: CLIConfiguration) -> None: """Retrieve metadata values about metrics/dimensions/entities/dimension values.""" + pass @list.command() @@ -368,11 +526,13 @@ def list(cfg: CLIContext) -> None: @pass_config @exception_handler @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) -def metrics(cfg: CLIContext, show_all_dimensions: bool = False, search: Optional[str] = None) -> None: +@error_if_not_in_dbt_project +def metrics(cfg: CLIConfiguration, show_all_dimensions: bool = False, search: Optional[str] = None) -> None: """List the metrics with their available dimensions. Automatically truncates long lists of dimensions, pass --show-all-dims to see all. """ + cfg.setup() spinner = Halo(text="πŸ” Looking for all available metrics...", spinner="dots") spinner.start() @@ -411,8 +571,10 @@ def metrics(cfg: CLIContext, show_all_dimensions: bool = False, search: Optional @pass_config @exception_handler @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) -def dimensions(cfg: CLIContext, metrics: List[str]) -> None: +@error_if_not_in_dbt_project +def dimensions(cfg: CLIConfiguration, metrics: List[str]) -> None: """List all unique dimensions.""" + cfg.setup() spinner = Halo( text="πŸ” Looking for all available dimensions...", spinner="dots", @@ -438,8 +600,10 @@ def dimensions(cfg: CLIContext, metrics: List[str]) -> None: @pass_config @exception_handler @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) -def entities(cfg: CLIContext, metrics: List[str]) -> None: +@error_if_not_in_dbt_project +def entities(cfg: CLIConfiguration, metrics: List[str]) -> None: """List all unique entities.""" + cfg.setup() spinner = Halo( text="πŸ” Looking for all available entities...", spinner="dots", @@ -459,8 +623,10 @@ def entities(cfg: CLIContext, metrics: List[str]) -> None: @pass_config @exception_handler @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) -def health_checks(cfg: CLIContext) -> None: +@error_if_not_in_dbt_project +def health_checks(cfg: CLIConfiguration) -> None: """Performs a health check against the DW provided in the configs.""" + cfg.setup() spinner = Halo( text="πŸ₯ Running health checks against your data warehouse... (This should not take longer than 30s for a successful connection)", spinner="dots", @@ -488,14 +654,16 @@ def health_checks(cfg: CLIContext) -> None: @pass_config @exception_handler @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) +@error_if_not_in_dbt_project def dimension_values( - cfg: CLIContext, + cfg: CLIConfiguration, metrics: List[str], dimension: str, start_time: Optional[dt.datetime] = None, end_time: Optional[dt.datetime] = None, ) -> None: """List all dimension values with the corresponding metrics.""" + cfg.setup() spinner = Halo( text=f"πŸ” Retrieving dimension values for dimension '{dimension}' of metrics '{', '.join(metrics)}'...", spinner="dots", @@ -612,8 +780,9 @@ def _data_warehouse_validations_runner( @pass_config @exception_handler @log_call(module_name=__name__, telemetry_reporter=_telemetry_reporter) +@error_if_not_in_dbt_project def validate_configs( - cfg: CLIContext, + cfg: CLIConfiguration, dw_timeout: Optional[int] = None, skip_dw: bool = False, show_all: bool = False, @@ -621,6 +790,8 @@ def validate_configs( semantic_validation_workers: int = 1, ) -> None: """Perform validations against the defined model configurations.""" + cfg.setup() + cfg.verbose = True if not show_all: diff --git a/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/dbt_project.yml b/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/dbt_project.yml new file mode 100644 index 0000000000..a69fbafdf9 --- /dev/null +++ b/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/dbt_project.yml @@ -0,0 +1,8 @@ +name: mf_tutorial_project +version: 1.0.0 +config-version: 2 + +profile: mf_tutorial + +model-paths: ["models"] +seed-paths: ["seeds"] diff --git a/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/models/_models.yml b/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/models/_models.yml new file mode 100644 index 0000000000..402775aab1 --- /dev/null +++ b/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/models/_models.yml @@ -0,0 +1,11 @@ +# See: https://docs.getdbt.com/docs/build/metricflow-time-spine +# For the page above, be sure to select the correct version of dbt at the top. + +models: + - name: all_days + description: A time spine with one row per day. + time_spine: + standard_granularity_column: date_day # Column for the standard grain of your table + columns: + - name: date_day + granularity: day # Set the granularity of the column diff --git a/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/models/all_days.sql b/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/models/all_days.sql new file mode 100644 index 0000000000..4ed312d99e --- /dev/null +++ b/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/models/all_days.sql @@ -0,0 +1,28 @@ +{{ + config( + materialized = 'table', + ) +}} + +with days as ( + -- Only generate dates for 2022 since that's what's in the seed data. + -- Note that `date_spine` does not include the end date. + {{ + dbt.date_spine( + 'day', + "make_date(2022, 1, 1)", + "make_date(2023, 1, 1)" + ) + }} + +), + +final as ( + select cast(date_day as date) as date_day + from days +) + + +select * from final +where date_day >= DATE '2022-01-01' +and date_day < DATE '2023-01-01' diff --git a/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/profiles.yml b/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/profiles.yml new file mode 100644 index 0000000000..8514a4a6ba --- /dev/null +++ b/dbt-metricflow/dbt_metricflow/cli/mf_tutorial_project/profiles.yml @@ -0,0 +1,6 @@ +mf_tutorial: + target: dev + outputs: + dev: + type: duckdb + path: 'mf_tutorial.duckdb' diff --git a/dbt-metricflow/dbt_metricflow/cli/sample_dbt_models/semantic_manifest.json b/dbt-metricflow/dbt_metricflow/cli/sample_dbt_models/semantic_manifest.json index 28032c8865..cd8e597a9c 100644 --- a/dbt-metricflow/dbt_metricflow/cli/sample_dbt_models/semantic_manifest.json +++ b/dbt-metricflow/dbt_metricflow/cli/sample_dbt_models/semantic_manifest.json @@ -1 +1,521 @@ -{"semantic_models": [{"name": "customers", "defaults": {"agg_time_dimension": "ds"}, "description": "Location and description of all customers.", "node_relation": {"alias": "customers", "schema_name": "dbt_test", "database": "metricflow", "relation_name": "\"metricflow\".\"dbt_test\".\"customers\""}, "entities": [{"name": "customer", "description": null, "type": "primary", "role": null, "expr": "id_customer", "metadata": null}, {"name": "country", "description": null, "type": "foreign", "role": null, "expr": null, "metadata": null}], "measures": [{"name": "new_customers", "agg": "sum", "description": null, "create_metric": false, "expr": "1", "agg_params": null, "metadata": null, "non_additive_dimension": null, "agg_time_dimension": null}], "dimensions": [{"name": "ds", "description": null, "type": "time", "is_partition": false, "type_params": {"time_granularity": "day", "validity_params": null}, "expr": null, "metadata": null}, {"name": "customer_country", "description": null, "type": "categorical", "is_partition": false, "type_params": null, "expr": "country", "metadata": null}], "metadata": null}, {"name": "countries", "defaults": null, "description": "Regions mapped to countries.", "node_relation": {"alias": "countries", "schema_name": "dbt_test", "database": "metricflow", "relation_name": "\"metricflow\".\"dbt_test\".\"countries\""}, "entities": [{"name": "country", "description": null, "type": "primary", "role": null, "expr": null, "metadata": null}], "measures": [], "dimensions": [{"name": "region", "description": "The region associated with the country.", "type": "categorical", "is_partition": false, "type_params": null, "expr": null, "metadata": null}], "metadata": null}, {"name": "transactions", "defaults": {"agg_time_dimension": "ds"}, "description": "Each row represents one transaction. There\nwill be a new row for any cancellations or alterations.\nThere is a transaction, order, and customer id for\nevery transaction. There is only one transaction id per\ntransaction, but there can be many rows per order id and\ncustomer id. The `ds` or date is reflected in UTC.\n", "node_relation": {"alias": "transactions", "schema_name": "dbt_test", "database": "metricflow", "relation_name": "\"metricflow\".\"dbt_test\".\"transactions\""}, "entities": [{"name": "transaction", "description": null, "type": "primary", "role": null, "expr": "id_transaction", "metadata": null}, {"name": "customer", "description": null, "type": "foreign", "role": null, "expr": "id_customer", "metadata": null}, {"name": "id_order", "description": null, "type": "foreign", "role": null, "expr": null, "metadata": null}], "measures": [{"name": "transaction_amount_usd", "agg": "sum", "description": "The total USD value of the transaction.", "create_metric": false, "expr": null, "agg_params": null, "metadata": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "transactions", "agg": "sum", "description": "The total number of transactions.", "create_metric": false, "expr": "1", "agg_params": null, "metadata": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "quick_buy_amount_usd", "agg": "sum", "description": "The total USD value of the transactions that were purchased using the \"quick buy\" button.", "create_metric": false, "expr": "CASE WHEN transaction_type_name = 'quick-buy' THEN transaction_amount_usd ELSE 0 END", "agg_params": null, "metadata": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "quick_buy_transactions", "agg": "sum_boolean", "description": "The total transactions bought as quick buy.", "create_metric": false, "expr": "CASE WHEN transaction_type_name = 'quick-buy' THEN TRUE ELSE FALSE END", "agg_params": null, "metadata": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "cancellations_usd", "agg": "sum", "description": "The total USD value of the transactions that were cancelled.", "create_metric": false, "expr": "CASE WHEN transaction_type_name = 'cancellation' THEN transaction_amount_usd ELSE 0 END", "agg_params": null, "metadata": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "alterations_usd", "agg": "sum", "description": "The total USD value of the transactions that were altered.", "create_metric": false, "expr": "CASE WHEN transaction_type_name = 'alteration' THEN transaction_amount_usd ELSE 0 END", "agg_params": null, "metadata": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "transacting_customers", "agg": "count_distinct", "description": "The distinct count of customers transacting on any given day.", "create_metric": false, "expr": "id_customer", "agg_params": null, "metadata": null, "non_additive_dimension": null, "agg_time_dimension": null}], "dimensions": [{"name": "ds", "description": null, "type": "time", "is_partition": false, "type_params": {"time_granularity": "day", "validity_params": null}, "expr": null, "metadata": null}, {"name": "is_large", "description": null, "type": "categorical", "is_partition": false, "type_params": null, "expr": "CASE WHEN transaction_amount_usd >= 30 THEN TRUE ELSE FALSE END", "metadata": null}, {"name": "quick_buy_transaction", "description": null, "type": "categorical", "is_partition": false, "type_params": null, "expr": "CASE\n WHEN transaction_type_name = 'quick-buy' THEN 'Quick Buy'\n ELSE 'Not Quick Buy'\nEND", "metadata": null}], "metadata": null}], "metrics": [{"name": "new_customers", "description": "", "type": "simple", "type_params": {"measure": {"name": "new_customers", "filter": null, "alias": null}, "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": null, "metadata": null}, {"name": "quick_buy_transactions", "description": "", "type": "simple", "type_params": {"measure": {"name": "quick_buy_transactions", "filter": null, "alias": null}, "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": null, "metadata": null}, {"name": "quick_buy_amount_usd", "description": "", "type": "simple", "type_params": {"measure": {"name": "quick_buy_amount_usd", "filter": null, "alias": null}, "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": null, "metadata": null}, {"name": "cancellations", "description": "", "type": "simple", "type_params": {"measure": {"name": "cancellations_usd", "filter": null, "alias": null}, "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": null, "metadata": null}, {"name": "transactions", "description": "", "type": "simple", "type_params": {"measure": {"name": "transactions", "filter": null, "alias": null}, "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": null, "metadata": null}, {"name": "alterations", "description": "", "type": "simple", "type_params": {"measure": {"name": "alterations_usd", "filter": null, "alias": null}, "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": null, "metadata": null}, {"name": "transaction_amount", "description": "", "type": "simple", "type_params": {"measure": {"name": "transaction_amount_usd", "filter": null, "alias": null}, "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": null, "metadata": null}, {"name": "cancellation_rate", "description": "", "type": "ratio", "type_params": {"measure": null, "numerator": {"name": "cancellations", "filter": null, "alias": null, "offset_window": null, "offset_to_grain": null}, "denominator": {"name": "transaction_amount", "filter": null, "alias": null, "offset_window": null, "offset_to_grain": null}, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": null, "metadata": null}, {"name": "revenue_usd", "description": "", "type": "derived", "type_params": {"measure": null, "numerator": null, "denominator": null, "expr": "transaction_amount - cancellations + alterations", "window": null, "grain_to_date": null, "metrics": [{"name": "transaction_amount", "filter": null, "alias": null, "offset_window": null, "offset_to_grain": null}, {"name": "cancellations", "filter": null, "alias": null, "offset_window": null, "offset_to_grain": null}, {"name": "alterations", "filter": null, "alias": null, "offset_window": null, "offset_to_grain": null}], "input_measures": []}, "filter": null, "metadata": null}, {"name": "cancellations_mx", "description": "", "type": "simple", "type_params": {"measure": {"name": "cancellations_usd", "filter": null, "alias": null}, "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": {"where_sql_template": "{{ dimension('customer_country', entity_path=['customer']) }} = 'MX'\n"}, "metadata": null}, {"name": "transaction_usd_na", "description": "", "type": "simple", "type_params": {"measure": {"name": "transaction_amount_usd", "filter": null, "alias": null}, "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "input_measures": []}, "filter": {"where_sql_template": "{{ dimension('region', entity_path=['customer', 'country']) }} = 'NA'\n"}, "metadata": null}], "interfaces_version": "0.1.0.dev7"} +{ + "semantic_models": [ + { + "name": "customers", + "defaults": { + "agg_time_dimension": "ds" + }, + "description": "Location and description of all customers.", + "node_relation": { + "alias": "customers", + "schema_name": "dbt_test", + "database": "metricflow", + "relation_name": "\"metricflow\".\"dbt_test\".\"customers\"" + }, + "entities": [ + { + "name": "customer", + "description": null, + "type": "primary", + "role": null, + "expr": "id_customer", + "metadata": null + }, + { + "name": "country", + "description": null, + "type": "foreign", + "role": null, + "expr": null, + "metadata": null + } + ], + "measures": [ + { + "name": "new_customers", + "agg": "sum", + "description": null, + "create_metric": false, + "expr": "1", + "agg_params": null, + "metadata": null, + "non_additive_dimension": null, + "agg_time_dimension": null + } + ], + "dimensions": [ + { + "name": "ds", + "description": null, + "type": "time", + "is_partition": false, + "type_params": { + "time_granularity": "day", + "validity_params": null + }, + "expr": null, + "metadata": null + }, + { + "name": "customer_country", + "description": null, + "type": "categorical", + "is_partition": false, + "type_params": null, + "expr": "country", + "metadata": null + } + ], + "metadata": null + }, + { + "name": "countries", + "defaults": null, + "description": "Regions mapped to countries.", + "node_relation": { + "alias": "countries", + "schema_name": "dbt_test", + "database": "metricflow", + "relation_name": "\"metricflow\".\"dbt_test\".\"countries\"" + }, + "entities": [ + { + "name": "country", + "description": null, + "type": "primary", + "role": null, + "expr": null, + "metadata": null + } + ], + "measures": [], + "dimensions": [ + { + "name": "region", + "description": "The region associated with the country.", + "type": "categorical", + "is_partition": false, + "type_params": null, + "expr": null, + "metadata": null + } + ], + "metadata": null + }, + { + "name": "transactions", + "defaults": { + "agg_time_dimension": "ds" + }, + "description": "Each row represents one transaction. There\nwill be a new row for any cancellations or alterations.\nThere is a transaction, order, and customer id for\nevery transaction. There is only one transaction id per\ntransaction, but there can be many rows per order id and\ncustomer id. The `ds` or date is reflected in UTC.\n", + "node_relation": { + "alias": "transactions", + "schema_name": "dbt_test", + "database": "metricflow", + "relation_name": "\"metricflow\".\"dbt_test\".\"transactions\"" + }, + "entities": [ + { + "name": "transaction", + "description": null, + "type": "primary", + "role": null, + "expr": "id_transaction", + "metadata": null + }, + { + "name": "customer", + "description": null, + "type": "foreign", + "role": null, + "expr": "id_customer", + "metadata": null + }, + { + "name": "id_order", + "description": null, + "type": "foreign", + "role": null, + "expr": null, + "metadata": null + } + ], + "measures": [ + { + "name": "transaction_amount_usd", + "agg": "sum", + "description": "The total USD value of the transaction.", + "create_metric": false, + "expr": null, + "agg_params": null, + "metadata": null, + "non_additive_dimension": null, + "agg_time_dimension": null + }, + { + "name": "transactions", + "agg": "sum", + "description": "The total number of transactions.", + "create_metric": false, + "expr": "1", + "agg_params": null, + "metadata": null, + "non_additive_dimension": null, + "agg_time_dimension": null + }, + { + "name": "quick_buy_amount_usd", + "agg": "sum", + "description": "The total USD value of the transactions that were purchased using the \"quick buy\" button.", + "create_metric": false, + "expr": "CASE WHEN transaction_type_name = 'quick-buy' THEN transaction_amount_usd ELSE 0 END", + "agg_params": null, + "metadata": null, + "non_additive_dimension": null, + "agg_time_dimension": null + }, + { + "name": "quick_buy_transactions", + "agg": "sum_boolean", + "description": "The total transactions bought as quick buy.", + "create_metric": false, + "expr": "CASE WHEN transaction_type_name = 'quick-buy' THEN TRUE ELSE FALSE END", + "agg_params": null, + "metadata": null, + "non_additive_dimension": null, + "agg_time_dimension": null + }, + { + "name": "cancellations_usd", + "agg": "sum", + "description": "The total USD value of the transactions that were cancelled.", + "create_metric": false, + "expr": "CASE WHEN transaction_type_name = 'cancellation' THEN transaction_amount_usd ELSE 0 END", + "agg_params": null, + "metadata": null, + "non_additive_dimension": null, + "agg_time_dimension": null + }, + { + "name": "alterations_usd", + "agg": "sum", + "description": "The total USD value of the transactions that were altered.", + "create_metric": false, + "expr": "CASE WHEN transaction_type_name = 'alteration' THEN transaction_amount_usd ELSE 0 END", + "agg_params": null, + "metadata": null, + "non_additive_dimension": null, + "agg_time_dimension": null + }, + { + "name": "transacting_customers", + "agg": "count_distinct", + "description": "The distinct count of customers transacting on any given day.", + "create_metric": false, + "expr": "id_customer", + "agg_params": null, + "metadata": null, + "non_additive_dimension": null, + "agg_time_dimension": null + } + ], + "dimensions": [ + { + "name": "ds", + "description": null, + "type": "time", + "is_partition": false, + "type_params": { + "time_granularity": "day", + "validity_params": null + }, + "expr": null, + "metadata": null + }, + { + "name": "is_large", + "description": null, + "type": "categorical", + "is_partition": false, + "type_params": null, + "expr": "CASE WHEN transaction_amount_usd >= 30 THEN TRUE ELSE FALSE END", + "metadata": null + }, + { + "name": "quick_buy_transaction", + "description": null, + "type": "categorical", + "is_partition": false, + "type_params": null, + "expr": "CASE\n WHEN transaction_type_name = 'quick-buy' THEN 'Quick Buy'\n ELSE 'Not Quick Buy'\nEND", + "metadata": null + } + ], + "metadata": null + } + ], + "metrics": [ + { + "name": "new_customers", + "description": "", + "type": "simple", + "type_params": { + "measure": { + "name": "new_customers", + "filter": null, + "alias": null + }, + "numerator": null, + "denominator": null, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": null, + "metadata": null + }, + { + "name": "quick_buy_transactions", + "description": "", + "type": "simple", + "type_params": { + "measure": { + "name": "quick_buy_transactions", + "filter": null, + "alias": null + }, + "numerator": null, + "denominator": null, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": null, + "metadata": null + }, + { + "name": "quick_buy_amount_usd", + "description": "", + "type": "simple", + "type_params": { + "measure": { + "name": "quick_buy_amount_usd", + "filter": null, + "alias": null + }, + "numerator": null, + "denominator": null, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": null, + "metadata": null + }, + { + "name": "cancellations", + "description": "", + "type": "simple", + "type_params": { + "measure": { + "name": "cancellations_usd", + "filter": null, + "alias": null + }, + "numerator": null, + "denominator": null, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": null, + "metadata": null + }, + { + "name": "transactions", + "description": "", + "type": "simple", + "type_params": { + "measure": { + "name": "transactions", + "filter": null, + "alias": null + }, + "numerator": null, + "denominator": null, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": null, + "metadata": null + }, + { + "name": "alterations", + "description": "", + "type": "simple", + "type_params": { + "measure": { + "name": "alterations_usd", + "filter": null, + "alias": null + }, + "numerator": null, + "denominator": null, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": null, + "metadata": null + }, + { + "name": "transaction_amount", + "description": "", + "type": "simple", + "type_params": { + "measure": { + "name": "transaction_amount_usd", + "filter": null, + "alias": null + }, + "numerator": null, + "denominator": null, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": null, + "metadata": null + }, + { + "name": "cancellation_rate", + "description": "", + "type": "ratio", + "type_params": { + "measure": null, + "numerator": { + "name": "cancellations", + "filter": null, + "alias": null, + "offset_window": null, + "offset_to_grain": null + }, + "denominator": { + "name": "transaction_amount", + "filter": null, + "alias": null, + "offset_window": null, + "offset_to_grain": null + }, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": null, + "metadata": null + }, + { + "name": "revenue_usd", + "description": "", + "type": "derived", + "type_params": { + "measure": null, + "numerator": null, + "denominator": null, + "expr": "transaction_amount - cancellations + alterations", + "window": null, + "grain_to_date": null, + "metrics": [ + { + "name": "transaction_amount", + "filter": null, + "alias": null, + "offset_window": null, + "offset_to_grain": null + }, + { + "name": "cancellations", + "filter": null, + "alias": null, + "offset_window": null, + "offset_to_grain": null + }, + { + "name": "alterations", + "filter": null, + "alias": null, + "offset_window": null, + "offset_to_grain": null + } + ], + "input_measures": [] + }, + "filter": null, + "metadata": null + }, + { + "name": "cancellations_mx", + "description": "", + "type": "simple", + "type_params": { + "measure": { + "name": "cancellations_usd", + "filter": null, + "alias": null + }, + "numerator": null, + "denominator": null, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": { + "where_sql_template": "{{ dimension('customer_country', entity_path=['customer']) }} = 'MX'\n" + }, + "metadata": null + }, + { + "name": "transaction_usd_na", + "description": "", + "type": "simple", + "type_params": { + "measure": { + "name": "transaction_amount_usd", + "filter": null, + "alias": null + }, + "numerator": null, + "denominator": null, + "expr": null, + "window": null, + "grain_to_date": null, + "metrics": [], + "input_measures": [] + }, + "filter": { + "where_sql_template": "{{ dimension('region', entity_path=['customer', 'country']) }} = 'NA'\n" + }, + "metadata": null + } + ], + "interfaces_version": "0.1.0.dev7" +} diff --git a/dbt-metricflow/dbt_metricflow/cli/tutorial.py b/dbt-metricflow/dbt_metricflow/cli/tutorial.py index 15ad2e1f2f..1529290fc1 100644 --- a/dbt-metricflow/dbt_metricflow/cli/tutorial.py +++ b/dbt-metricflow/dbt_metricflow/cli/tutorial.py @@ -13,6 +13,7 @@ class dbtMetricFlowTutorialHelper: SAMPLE_MODELS_DIRECTORY = SAMPLE_DBT_MODEL_DIRECTORY + "/sample_models" SAMPLE_SEED_DIRECTORY = SAMPLE_DBT_MODEL_DIRECTORY + "/seeds" SAMPLE_SEMANTIC_MANIFEST = SAMPLE_DBT_MODEL_DIRECTORY + "/semantic_manifest.json" + SAMPLE_DBT_PROJECT_DIRECTORY = "mf_tutorial_project" SAMPLE_SOURCES_FILE = "sources.yml" @staticmethod @@ -52,12 +53,22 @@ def generate_semantic_manifest_file(manifest_path: pathlib.Path) -> None: @staticmethod def remove_sample_files(model_path: pathlib.Path, seed_path: pathlib.Path) -> None: """Remove the sample files generated.""" - if model_path.exists(): - shutil.rmtree(model_path) - if seed_path.exists(): - shutil.rmtree(seed_path) + dbtMetricFlowTutorialHelper.remove_files(model_path) + dbtMetricFlowTutorialHelper.remove_files(seed_path) + + @staticmethod + def remove_files(path: pathlib.Path) -> None: + """Remove the sample files generated.""" + if path.exists(): + shutil.rmtree(path) @staticmethod def check_if_path_exists(paths: Sequence[pathlib.Path]) -> bool: """Check if the given set of paths already exists, return True if any of the paths exists.""" return any(p.exists() for p in paths) + + @staticmethod + def generate_dbt_project(project_path: pathlib.Path) -> None: + """Generate a sample dbt project using a self-contained DuckDB instance into the given directory.""" + sample_project_path = pathlib.Path(__file__).parent / dbtMetricFlowTutorialHelper.SAMPLE_DBT_PROJECT_DIRECTORY + shutil.copytree(src=sample_project_path, dst=project_path) diff --git a/dbt-metricflow/dbt_metricflow/cli/utils.py b/dbt-metricflow/dbt_metricflow/cli/utils.py index d3e329a6a7..ddba75eabb 100644 --- a/dbt-metricflow/dbt_metricflow/cli/utils.py +++ b/dbt-metricflow/dbt_metricflow/cli/utils.py @@ -12,7 +12,7 @@ from metricflow_semantics.mf_logging.lazy_formattable import LazyFormat import dbt_metricflow.cli.custom_click_types as click_custom -from dbt_metricflow.cli.cli_context import CLIContext +from dbt_metricflow.cli.cli_configuration import CLIConfiguration logger = logging.getLogger(__name__) @@ -117,14 +117,14 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: # type: ignore[misc] logging.exception("Got an exception in the exception handler.") # Checks if CLIContext has verbose flag set - if isinstance(args[0], CLIContext): - cli_context: CLIContext = args[0] + if isinstance(args[0], CLIConfiguration): + cli_context: CLIConfiguration = args[0] click.echo(f"\nERROR: {str(e)}\nLog file: {cli_context.log_file_path}") else: - if not isinstance(args[0], CLIContext): + if not isinstance(args[0], CLIConfiguration): logger.error( LazyFormat( - lambda: f"Missing {CLIContext.__name__} as the first argument to the function " + lambda: f"Missing {CLIConfiguration.__name__} as the first argument to the function " f"{getattr(func, '__name__', repr(func))}" ) ) diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-cli.txt b/dbt-metricflow/extra-hatch-configuration/requirements-cli.txt index ead6c13150..0e9e7943b4 100644 --- a/dbt-metricflow/extra-hatch-configuration/requirements-cli.txt +++ b/dbt-metricflow/extra-hatch-configuration/requirements-cli.txt @@ -1,5 +1,5 @@ # Internal dependencies -dbt-core @ git+https://github.com/dbt-labs/dbt-core@e53420c1d073dc81609ae7aa84cef6ee09650576#subdirectory=core +dbt-core>=1.9.0, <1.10.0 # dsi version should be fixed by MetricFlow/dbt-core, not set here dbt-semantic-interfaces diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt index 3ef6a2994c..39624dac32 100644 --- a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-bigquery.txt @@ -1 +1 @@ -dbt-bigquery>=1.8.0, <1.9.0 +dbt-bigquery>=1.9.0, <1.10.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt index ab8e6417ea..d3395282e1 100644 --- a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-databricks.txt @@ -1 +1 @@ -dbt-databricks>=1.8.0, <1.9.0 +dbt-databricks>=1.9.0, <1.10.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt index a91d11d94d..68e9ae91cc 100644 --- a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-duckdb.txt @@ -1 +1 @@ -dbt-duckdb>=1.8.0, <1.9.0 +dbt-duckdb>=1.9.0, <1.10.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt index 832aa9e91f..937aa33ada 100644 --- a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-postgres.txt @@ -1 +1 @@ -dbt-postgres>=1.8.0, <1.9.0 +dbt-postgres>=1.9.0, <1.10.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt index 7658ca24f1..8c426e4a6e 100644 --- a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-redshift.txt @@ -1 +1 @@ -dbt-redshift>=1.8.0, <1.9.0 +dbt-redshift>=1.9.0, <1.10.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt index 937f1b52fc..82770c83ae 100644 --- a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-snowflake.txt @@ -1 +1 @@ -dbt-snowflake>=1.8.0, <1.9.0 +dbt-snowflake>=1.9.0, <1.10.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt index bc3390eacb..97ba0ea7a2 100644 --- a/dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt +++ b/dbt-metricflow/extra-hatch-configuration/requirements-dbt-trino.txt @@ -1 +1 @@ -dbt-trino>=1.8.0, <1.9.0 +dbt-trino>=1.9.0, <1.10.0 diff --git a/dbt-metricflow/extra-hatch-configuration/requirements-metricflow.txt b/dbt-metricflow/extra-hatch-configuration/requirements-metricflow.txt index 05293d9c09..368a70605b 100644 --- a/dbt-metricflow/extra-hatch-configuration/requirements-metricflow.txt +++ b/dbt-metricflow/extra-hatch-configuration/requirements-metricflow.txt @@ -1 +1 @@ -metricflow>=0.206.0, <0.207.0 +metricflow>=0.207.1, <0.208.0 diff --git a/dbt-metricflow/pyproject.toml b/dbt-metricflow/pyproject.toml index 2e2c737d20..e48ed6f340 100644 --- a/dbt-metricflow/pyproject.toml +++ b/dbt-metricflow/pyproject.toml @@ -1,12 +1,12 @@ [build-system] -requires = ["hatchling~=1.14.0", "hatch-requirements-txt >= 0.4.1, <0.5.0"] +requires = ["hatchling>=1.27.0,<1.28.0", "hatch-requirements-txt >= 0.4.1, <0.5.0"] build-backend = "hatchling.build" [project] name = "dbt-metricflow" description = "Execute commands against the MetricFlow semantic layer with dbt." readme = "README.md" -requires-python = ">=3.8,<3.13" +requires-python = ">=3.9,<3.13" license = "BUSL-1.1" authors = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, @@ -15,7 +15,6 @@ authors = [ classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/metricflow-semantics/pyproject.toml b/metricflow-semantics/pyproject.toml index b231925a93..088782988e 100644 --- a/metricflow-semantics/pyproject.toml +++ b/metricflow-semantics/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "metricflow-semantics" -version = "0.1.0" +version = "0.2.0" description = "Modules for semantic understanding of a MetricFlow query." readme = "README.md" requires-python = ">=3.9,<3.13" diff --git a/pyproject.toml b/pyproject.toml index 7f1373c5e8..4282bed823 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling~=1.14.0", "hatch-requirements-txt >= 0.4.1, <0.5.0"] +requires = ["hatchling>=1.27.0,<1.28.0", "hatch-requirements-txt >= 0.4.1, <0.5.0"] build-backend = "hatchling.build" [project] diff --git a/tests_metricflow/cli/test_cli.py b/tests_metricflow/cli/test_cli.py index 7152d83641..f19a8497f0 100644 --- a/tests_metricflow/cli/test_cli.py +++ b/tests_metricflow/cli/test_cli.py @@ -19,7 +19,7 @@ EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE, ) -from dbt_metricflow.cli.cli_context import CLIContext +from dbt_metricflow.cli.cli_configuration import CLIConfiguration from dbt_metricflow.cli.main import ( dimension_values, dimensions, @@ -79,7 +79,7 @@ def create_directory(directory_path: str) -> Iterator[None]: shutil.rmtree(path) -def test_validate_configs(cli_context: CLIContext) -> None: +def test_validate_configs(cli_context: CLIConfiguration) -> None: """Tests config validation from a manifest stored on the filesystem. This test is special, because the CLI bypasses the semantic manifest read into the CLIContext and diff --git a/tests_metricflow/fixtures/cli_fixtures.py b/tests_metricflow/fixtures/cli_fixtures.py index 6a77b576a3..2e886212c9 100644 --- a/tests_metricflow/fixtures/cli_fixtures.py +++ b/tests_metricflow/fixtures/cli_fixtures.py @@ -12,7 +12,7 @@ from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from typing_extensions import override -from dbt_metricflow.cli.cli_context import CLIContext +from dbt_metricflow.cli.cli_configuration import CLIConfiguration from dbt_metricflow.cli.dbt_connectors.dbt_config_accessor import dbtArtifacts, dbtProjectMetadata from metricflow.engine.metricflow_engine import MetricFlowEngine from metricflow.protocols.sql_client import SqlClient @@ -20,7 +20,7 @@ from tests_metricflow.fixtures.setup_fixtures import dbt_project_dir -class FakeCLIContext(CLIContext): +class FakeCLIConfiguration(CLIConfiguration): """Fake context for testing. Manually initialize or override params used by CLIContext as appropriate. Note - this construct should not exist. It bypasses a fundamental initialization process within the CLI which @@ -78,9 +78,9 @@ def cli_context( # noqa: D103 sql_client: SqlClient, mf_engine_test_fixture_mapping: Mapping[SemanticManifestSetup, MetricFlowEngineTestFixture], create_source_tables: bool, -) -> Generator[CLIContext, None, None]: +) -> Generator[CLIConfiguration, None, None]: engine_test_fixture = mf_engine_test_fixture_mapping[SemanticManifestSetup.SIMPLE_MANIFEST] - context = FakeCLIContext() + context = FakeCLIConfiguration() context._mf = engine_test_fixture.metricflow_engine context._sql_client = sql_client context._semantic_manifest = engine_test_fixture.semantic_manifest @@ -93,7 +93,7 @@ def cli_context( # noqa: D103 class MetricFlowCliRunner(CliRunner): """Custom CliRunner class to handle passing context.""" - def __init__(self, cli_context: CLIContext, project_path: str) -> None: # noqa: D107 + def __init__(self, cli_context: CLIConfiguration, project_path: str) -> None: # noqa: D107 self.cli_context = cli_context self.project_path = project_path super().__init__() @@ -107,5 +107,5 @@ def run(self, cli: click.BaseCommand, args: Optional[Sequence[str]] = None) -> R @pytest.fixture(scope="session") -def cli_runner(cli_context: CLIContext) -> MetricFlowCliRunner: # noqa: D103 +def cli_runner(cli_context: CLIConfiguration) -> MetricFlowCliRunner: # noqa: D103 return MetricFlowCliRunner(cli_context=cli_context, project_path=dbt_project_dir())