Merge pull request #102 from dlt-hub/feat/cli-updates

Cli updates - normalize paths, add _pipeline suffix to main pipeline file, add source sections to config and secrets toml

.gitignore

@@ -31,7 +31,9 @@ custom-e2e/
 
 client/
 *-pipeline/
+*_pipeline/
 tests/_local/
 .DS_Store
 !hackathon/*-pipeline/
+!hackathon/*_pipeline/
 rest_api

Makefile

@@ -45,7 +45,7 @@ create-e2e-pokemon-pipeline:
 	poetry run dlt-init-openapi init pokemon --path tests/cases/e2e_specs/pokeapi.yml --global-limit 2 --no-interactive
 
 run-pokemon-pipeline:
-	cd pokemon-pipeline && poetry run python pipeline.py
+	cd pokemon_pipeline && poetry run python pokemon_pipeline.py
 
 check-pokemon-pipeline:
 	poetry run pytest tests/e2e

dlt_init_openapi/cli/__init__.py

@@ -10,7 +10,7 @@
 from dlt_init_openapi.config import Config
 from dlt_init_openapi.utils import update_rest_api
 
-app = typer.Typer()
+app = typer.Typer(add_completion=False)
 
 
 def _print_version(value: bool) -> None:

dlt_init_openapi/config.py

@@ -7,6 +7,8 @@
 import yaml
 from pydantic import BaseModel
 
+from dlt_init_openapi.utils.misc import snake_case
+
 from .typing import TEndpointFilter
 
 REST_API_SOURCE_LOCATION = str(pathlib.Path(__file__).parent.resolve() / "../rest_api")
@@ -31,8 +33,10 @@ class Config(BaseModel):
     """HTTP methods to render from OpenAPI spec"""
     fallback_openapi_title: str = "openapi"
     """Fallback title when openapi info.title is missing or empty"""
-    project_folder_suffix: str = "-pipeline"
+    project_folder_suffix: str = "_pipeline"
     """Suffix for project name"""
+    pipeline_file_suffix: str = "_pipeline.py"
+    """Suffix for pipeline file name"""
     dataset_name_suffix: str = "_data"
     """Suffix for dataset"""
     endpoint_filter: Optional[TEndpointFilter] = None
@@ -50,16 +54,22 @@ class Config(BaseModel):
 
     # internal, do not set via config file
     project_dir: Path = None
+    pipeline_file_name: str = None
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super(Config, self).__init__(*args, **kwargs)
         self.prepare()
 
     def prepare(self) -> None:
+        # normalize a couple of vars
+        self.project_name = snake_case(self.project_name)
+        self.package_name = snake_case(self.package_name)
+
         if self.project_name and self.project_folder_suffix:
             base_dir = Path.cwd() if not self.output_path else Path.cwd() / self.output_path
             project_folder = self.project_name + self.project_folder_suffix
             self.project_dir = base_dir / project_folder
+            self.pipeline_file_name = self.project_name + self.pipeline_file_suffix
 
     @staticmethod
     def load_from_path(path: Path, *args: Any, **kwargs: Any) -> "Config":

dlt_init_openapi/renderer/default/__init__.py

@@ -139,7 +139,7 @@ def _render_source(self) -> str:
         )
 
     def _build_pipeline(self) -> None:
-        module_path = self.config.project_dir / "pipeline.py"
+        module_path = self.config.project_dir / self.config.pipeline_file_name
 
         template = self.env.get_template("pipeline.py.j2")
         module_path.write_text(

dlt_init_openapi/renderer/default/templates/dlt_config.toml.j2

@@ -1,7 +1,7 @@
 [runtime]
 log_level="INFO"
 
-[sources]
+[sources.{{project_name}}]
 # Base URL for the API
 {% if first_server %}
 {% if first_server.description %}

dlt_init_openapi/renderer/default/templates/dlt_secrets.toml.j2

@@ -1,3 +1,4 @@
 {% if credentials %}
+[sources.{{project_name}}]
 {{ credentials.detected_secret_name }} = "FILL ME OUT"  # TODO: fill in your credentials
 {% endif %}

dlt_init_openapi/renderer/default/templates/pipeline.py.j2

@@ -8,7 +8,7 @@ if __name__ == "__main__":
         pipeline_name="{{ package_name}}_pipeline",
         destination='duckdb',
         dataset_name="{{ dataset_name }}",
-        full_refresh=False,
+        progress="log",
         export_schema_path="schemas/export"
     )
     source = {{ source_name }}()

dlt_init_openapi/utils/misc.py

@@ -58,6 +58,8 @@ def fix_reserved_words(value: str) -> str:
 
 def snake_case(value: str) -> str:
     """Converts to snake_case"""
+    if not value:
+        return value
     words = split_words(sanitize(value))
     return "_".join(words).lower()
 

tests/e2e/test_pokemon_pipeline.py

@@ -1,6 +1,6 @@
 import duckdb
 
-POKE_DUCKDB_FILE = "./pokemon-pipeline/pokemon_pipeline.duckdb"
+POKE_DUCKDB_FILE = "./pokemon_pipeline/pokemon_pipeline.duckdb"
 
 
 def test_pokemon_pipeline() -> None:

tests/test_config.py

@@ -38,5 +38,5 @@ def test_load_from_path(tmp_path: Path, filename, dump, relative):
     yml_file.write_text(dump(data))
 
     config = Config.load_from_path(yml_file)
-    assert config.project_name == "project-name"
+    assert config.project_name == "project_name"
     assert config.package_name == "package_name"