fix: silverback build missing files during generate (#151)

* fix: silverback build missing files during generate

* fix: black

* feat: proper bot bots and python file treatment

* fix: black

* fix: flake8 issue

* refactor: cleanup the build helper function placement

* fix: pipe only the stdout

* feat: allow generate flag to also build

* feat: add build_utils file

* feat: add note about subclassing

* feat: move image generate to build utils

* refactor: clean up some code

* refactor: more cleanup

* fix: mypy error

* fix: use clean_path for home directory

* refactorb(build): apply suggestions from code review

---------

Co-authored-by: El De-dog-lo <[email protected]>

silverback/_build_utils.py

@@ -0,0 +1,144 @@
+import shlex
+import subprocess
+from functools import singledispatchmethod
+from pathlib import Path
+from typing import Union
+
+import click
+from ape.utils.os import clean_path
+
+DOCKERFILE_CONTENT = """
+FROM ghcr.io/apeworx/silverback:stable
+USER root
+WORKDIR /app
+RUN chown harambe:harambe /app
+USER harambe
+"""
+
+
+# Note: Python3.12 supports subclassing pathlib.Path
+class BasePath(Path):
+    _flavour = type(Path())._flavour  # type: ignore
+
+
+class FilePath(BasePath):
+    """A subclass of Path representing a file."""
+
+
+class DirPath(BasePath):
+    """A subclass of Path representing a path"""
+
+
+def get_path(path: Path):
+    if path.is_file():
+        return FilePath(str(path))
+    elif path.is_dir():
+        return DirPath(str(path))
+    else:
+        raise ValueError(f"{path} is neither a file nor a directory")
+
+
+PathType = Union["FilePath", "DirPath"]
+
+
+def generate_dockerfiles(path: Path):
+    path = get_path(path)
+    dg = DockerfileGenerator()
+    dg.generate_dockerfiles(path)
+
+
+def build_docker_images(path: Path):
+    DockerfileGenerator.build_images(path)
+
+
+class DockerfileGenerator:
+
+    @property
+    def dockerfile_name(self):
+        return self._dockerfile_name
+
+    @dockerfile_name.setter
+    def dockerfile_name(self, name):
+        self._dockerfile_name = name
+
+    @singledispatchmethod
+    def generate_dockerfiles(self, path: PathType):
+        """
+        Will generate a file based on path type
+        """
+        raise NotImplementedError(f"Path type {type(path)} not supported")
+
+    @generate_dockerfiles.register
+    def _(self, path: FilePath):
+        dockerfile_content = self._check_for_requirements(DOCKERFILE_CONTENT)
+        self.dockerfile_name = f"Dockerfile.{path.parent.name}-bot"
+        dockerfile_content += f"COPY {path.name}/ /app/bot.py\n"
+        self._build_helper(dockerfile_content)
+
+    @generate_dockerfiles.register
+    def _(self, path: DirPath):
+        bots = self._get_all_bot_files(path)
+        for bot in bots:
+            dockerfile_content = self._check_for_requirements(DOCKERFILE_CONTENT)
+            if bot.name == "__init__.py" or bot.name == "bot.py":
+                self.dockerfile_name = f"Dockerfile.{bot.parent.parent.name}-bot"
+                dockerfile_content += f"COPY {path.name}/ /app/bot\n"
+            else:
+                self.dockerfile_name = f"Dockerfile.{bot.name.replace('.py', '')}"
+                dockerfile_content += f"COPY {path.name}/{bot.name} /app/bot.py\n"
+            self._build_helper(dockerfile_content)
+
+    def _build_helper(self, dockerfile_c: str):
+        """
+        Used in multiple places in build.
+        """
+        dockerfile_path = Path.cwd() / ".silverback-images" / self.dockerfile_name
+        dockerfile_path.parent.mkdir(exist_ok=True)
+        dockerfile_path.write_text(dockerfile_c.strip() + "\n")
+        click.echo(f"Generated {clean_path(dockerfile_path)}")
+
+    def _check_for_requirements(self, dockerfile_content):
+        if (Path.cwd() / "requirements.txt").exists():
+            dockerfile_content += "COPY requirements.txt .\n"
+            dockerfile_content += (
+                "RUN pip install --upgrade pip && pip install -r requirements.txt\n"
+            )
+
+        if (Path.cwd() / "ape-config.yaml").exists():
+            dockerfile_content += "COPY ape-config.yaml .\n"
+            dockerfile_content += "RUN ape plugins install -U .\n"
+
+        return dockerfile_content
+
+    def _get_all_bot_files(self, path: DirPath):
+        files = sorted({file for file in path.iterdir() if file.is_file()}, reverse=True)
+        bots = []
+        for file in files:
+            if file.name == "__init__.py" or file.name == "bot.py":
+                bots = [file]
+                break
+            bots.append(file)
+        return bots
+
+    @staticmethod
+    def build_images(path: Path):
+        dockerfiles = {file for file in path.iterdir() if file.is_file()}
+        for file in dockerfiles:
+            try:
+                command = shlex.split(
+                    "docker build -f "
+                    f"./{file.parent.name}/{file.name} "
+                    f"-t {file.name.split('.')[1]}:latest ."
+                )
+                result = subprocess.run(
+                    command,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                    check=True,
+                )
+                click.echo(result.stdout)
+            except subprocess.CalledProcessError as e:
+                click.echo("Error during docker build:")
+                click.echo(e.stderr)
+                raise

silverback/_cli.py

@@ -1,7 +1,5 @@
 import asyncio
 import os
-import shlex
-import subprocess
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
@@ -19,6 +17,7 @@
 from ape.exceptions import Abort, ApeException
 from fief_client.integrations.cli import FiefAuth
 
+from silverback._build_utils import build_docker_images, generate_dockerfiles
 from silverback._click_ext import (
     SectionedHelpGroup,
     auth_required,
@@ -35,18 +34,6 @@
 from silverback.runner import PollingRunner, WebsocketRunner
 from silverback.worker import run_worker
 
-DOCKERFILE_CONTENT = """
-FROM ghcr.io/apeworx/silverback:stable
-USER root
-WORKDIR /app
-RUN chown harambe:harambe /app
-USER harambe
-COPY ape-config.yaml .
-COPY requirements.txt .
-RUN pip install --upgrade pip && pip install -r requirements.txt
-RUN ape plugins install -U .
-"""
-
 
 @click.group(cls=SectionedHelpGroup)
 def cli():
@@ -138,57 +125,25 @@ def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, bot):
 def build(generate, path):
     """Generate Dockerfiles and build bot images"""
     if generate:
-        if not (path := Path.cwd() / path).exists():
+        if (
+            not (path := Path.cwd() / path).exists()
+            and not (path := Path.cwd() / "bot").exists()
+            and not (path := Path.cwd() / "bot.py").exists()
+        ):
             raise FileNotFoundError(
-                f"The bots directory '{path}' does not exist. "
-                "You should have a `{path}/` folder in the root of your project."
+                f"The bots directory '{path}', 'bot/' and 'bot.py' does not exist in your path. "
+                f"You should have a '{path}/' or 'bot/' folder, or a 'bot.py' file in the root "
+                "of your project."
             )
-        files = {file for file in path.iterdir() if file.is_file()}
-        bots = []
-        for file in files:
-            if "__init__" in file.name:
-                bots = [file]
-                break
-            bots.append(file)
-        for bot in bots:
-            dockerfile_content = DOCKERFILE_CONTENT
-            if "__init__" in bot.name:
-                docker_filename = f"Dockerfile.{bot.parent.name}"
-                dockerfile_content += f"COPY {path.name}/ /app/bot"
-            else:
-                docker_filename = f"Dockerfile.{bot.name.replace('.py', '')}"
-                dockerfile_content += f"COPY {path.name}/{bot.name} /app/bot.py"
-            dockerfile_path = Path.cwd() / ".silverback-images" / docker_filename
-            dockerfile_path.parent.mkdir(exist_ok=True)
-            dockerfile_path.write_text(dockerfile_content.strip() + "\n")
-            click.echo(f"Generated {dockerfile_path}")
-        return
+        generate_dockerfiles(path)
 
     if not (path := Path.cwd() / ".silverback-images").exists():
         raise FileNotFoundError(
             f"The dockerfile directory '{path}' does not exist. "
             "You should have a `{path}/` folder in the root of your project."
         )
-    dockerfiles = {file for file in path.iterdir() if file.is_file()}
-    for file in dockerfiles:
-        try:
-            command = shlex.split(
-                "docker build -f "
-                f"./{file.parent.name}/{file.name} "
-                f"-t {file.name.split('.')[1]}:latest ."
-            )
-            result = subprocess.run(
-                command,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                text=True,
-                check=True,
-            )
-            click.echo(result.stdout)
-        except subprocess.CalledProcessError as e:
-            click.echo("Error during docker build:")
-            click.echo(e.stderr)
-            raise
+
+    build_docker_images(path)
 
 
 @cli.command(cls=ConnectedProviderCommand, section="Local Commands")