From c4aa54011c71e385d0815bc9b3bcfb5db4f03568 Mon Sep 17 00:00:00 2001 From: Juliya Smith Date: Tue, 29 Oct 2024 14:09:31 -0500 Subject: [PATCH 1/2] perf: make silverback --help faster --- setup.cfg | 1 + silverback/__init__.py | 24 ++++++- silverback/_cli.py | 136 +++++++++++++++++++++++---------------- silverback/_click_ext.py | 30 ++++++--- 4 files changed, 123 insertions(+), 68 deletions(-) diff --git a/setup.cfg b/setup.cfg index c362920f..fb8dd960 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [flake8] max-line-length = 100 +ignore = E704,W503,PYD002 exclude = venv* .eggs diff --git a/silverback/__init__.py b/silverback/__init__.py index b2ec0a03..1f55c662 100644 --- a/silverback/__init__.py +++ b/silverback/__init__.py @@ -1,6 +1,24 @@ -from .exceptions import CircuitBreaker, SilverbackException -from .main import SilverbackBot -from .state import StateSnapshot +def __getattr__(name: str): + if name == "CircuitBreaker": + from .exceptions import CircuitBreaker + + return CircuitBreaker + + elif name == "SilverbackException": + from .exceptions import SilverbackException + + return SilverbackException + + elif name == "SilverbackBot": + from .main import SilverbackBot + + return SilverbackBot + + elif name == "StateSnapshot": + from .state import StateSnapshot + + return StateSnapshot + __all__ = [ "StateSnapshot", diff --git a/silverback/_cli.py b/silverback/_cli.py index 28dbd0cc..601981b6 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -2,22 +2,21 @@ import os from datetime import datetime, timedelta, timezone from pathlib import Path +from typing import TYPE_CHECKING, Optional import click import yaml # type: ignore[import-untyped] -from ape.api import AccountAPI, NetworkAPI from ape.cli import ( AccountAliasPromptChoice, ConnectedProviderCommand, + LazyChoice, account_option, ape_cli_context, network_option, ) -from ape.contracts import ContractInstance from ape.exceptions import Abort, ApeException -from fief_client.integrations.cli import FiefAuth +from ape.logging import LogLevel -from silverback._build_utils import build_docker_images, generate_dockerfiles from silverback._click_ext import ( SectionedHelpGroup, auth_required, @@ -29,10 +28,14 @@ timedelta_callback, token_amount_callback, ) -from silverback.cluster.client import ClusterClient, PlatformClient -from silverback.cluster.types import ClusterTier, LogLevel, ResourceStatus -from silverback.runner import PollingRunner, WebsocketRunner -from silverback.worker import run_worker + +if TYPE_CHECKING: + from ape.api.accounts import AccountAPI + from ape.api.networks import NetworkAPI + from ape.contracts import ContractInstance + from fief_client.integrations.cli import FiefAuth + + from silverback.cluster.client import ClusterClient, PlatformClient @click.group(cls=SectionedHelpGroup) @@ -98,6 +101,7 @@ def _network_callback(ctx, param, val): @click.argument("bot", required=False, callback=bot_path_callback) def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, bot): """Run Silverback bot""" + from silverback.runner import PollingRunner, WebsocketRunner if not runner_class: # NOTE: Automatically select runner class @@ -124,6 +128,8 @@ def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, bot): @click.argument("path", required=False, type=str, default="bots") def build(generate, path): """Generate Dockerfiles and build bot images""" + from silverback._build_utils import build_docker_images, generate_dockerfiles + if generate: if ( not (path := Path.cwd() / path).exists() @@ -159,12 +165,14 @@ def build(generate, path): @click.argument("bot", required=False, callback=bot_path_callback) def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, bot): """Run Silverback task workers (advanced)""" + from silverback.worker import run_worker + asyncio.run(run_worker(bot.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) @cli.command(section="Cloud Commands (https://silverback.apeworx.io)") @auth_required -def login(auth: FiefAuth): +def login(auth: "FiefAuth"): """Login to ApeWorX Authorization Service (https://account.apeworx.io)""" auth.authorize() @@ -186,7 +194,7 @@ def workspaces(): @workspaces.command(name="list", section="Platform Commands (https://silverback.apeworx.io)") @platform_client -def list_workspaces(platform: PlatformClient): +def list_workspaces(platform: "PlatformClient"): """List available workspaces for your account""" if workspace_names := list(platform.workspaces): @@ -204,7 +212,7 @@ def list_workspaces(platform: PlatformClient): @workspaces.command(name="info", section="Platform Commands (https://silverback.apeworx.io)") @click.argument("workspace") @platform_client -def workspace_info(platform: PlatformClient, workspace: str): +def workspace_info(platform: "PlatformClient", workspace: str): """Get Configuration information about a WORKSPACE""" if not (workspace_info := platform.workspaces.get(workspace)): @@ -232,7 +240,7 @@ def workspace_info(platform: PlatformClient, workspace: str): ) @platform_client def new_workspace( - platform: PlatformClient, + platform: "PlatformClient", workspace_name: str, workspace_slug: str, ): @@ -273,7 +281,7 @@ def new_workspace( @click.argument("workspace") @platform_client def update_workspace( - platform: PlatformClient, + platform: "PlatformClient", workspace: str, name: str | None, slug: str | None, @@ -300,7 +308,7 @@ def update_workspace( @workspaces.command(name="delete", section="Platform Commands (https://silverback.apeworx.io)") @click.argument("workspace") @platform_client -def delete_workspace(platform: PlatformClient, workspace: str): +def delete_workspace(platform: "PlatformClient", workspace: str): """Delete an empty Workspace on the Silverback Platform""" if not (workspace_client := platform.workspaces.get(workspace)): @@ -316,7 +324,7 @@ def delete_workspace(platform: PlatformClient, workspace: str): @cluster.command(name="list", section="Platform Commands (https://silverback.apeworx.io)") @click.argument("workspace") @platform_client -def list_clusters(platform: PlatformClient, workspace: str): +def list_clusters(platform: "PlatformClient", workspace: str): """List available clusters in a WORKSPACE""" if not (workspace_client := platform.workspaces.get(workspace)): @@ -345,7 +353,7 @@ def list_clusters(platform: PlatformClient, workspace: str): @click.argument("workspace") @platform_client def new_cluster( - platform: PlatformClient, + platform: "PlatformClient", workspace: str, cluster_name: str | None, cluster_slug: str | None, @@ -365,6 +373,8 @@ def new_cluster( else: raise click.UsageError("Must provide a name or a slug/name combo") + from silverback.cluster.types import ResourceStatus + cluster = workspace_client.create_cluster( cluster_name=cluster_name, cluster_slug=cluster_slug, @@ -396,7 +406,7 @@ def new_cluster( @click.argument("cluster_path") @platform_client def update_cluster( - platform: PlatformClient, + platform: "PlatformClient", cluster_path: str, name: str | None, slug: str | None, @@ -435,21 +445,30 @@ def pay(): """Pay for CLUSTER with Crypto using ApePay streaming payments""" +def _default_tier(): + from silverback.cluster.types import ClusterTier + + return ClusterTier.STANDARD.name.capitalize() + + +def _tier_choices(): + from silverback.cluster.types import ClusterTier + + return [ + ClusterTier.STANDARD.name.capitalize(), + ClusterTier.PREMIUM.name.capitalize(), + ] + + @pay.command(name="create", cls=ConnectedProviderCommand) @account_option() @click.argument("cluster_path") @click.option( "-t", "--tier", - default=ClusterTier.STANDARD.name.capitalize(), + default=_default_tier, metavar="NAME", - type=click.Choice( - [ - ClusterTier.STANDARD.name.capitalize(), - ClusterTier.PREMIUM.name.capitalize(), - ], - case_sensitive=False, - ), + type=LazyChoice(_tier_choices, case_sensitive=False), help="Named set of options to use for cluster as a base (Defaults to Standard)", ) @click.option( @@ -480,13 +499,13 @@ def pay(): ) @platform_client def create_payment_stream( - platform: PlatformClient, - network: NetworkAPI, - account: AccountAPI, + platform: "PlatformClient", + network: "NetworkAPI", + account: "AccountAPI", cluster_path: str, tier: str, config_updates: list[tuple[str, str]], - token: ContractInstance | None, + token: Optional["ContractInstance"], token_amount: int | None, stream_time: timedelta | None, ): @@ -495,6 +514,7 @@ def create_payment_stream( NOTE: This action cannot be cancelled! Streams must exist for at least 1 hour before cancelling. """ + from silverback.cluster.types import ClusterTier, ResourceStatus if "/" not in cluster_path or len(cluster_path.split("/")) > 2: raise click.BadArgumentUsage(f"Invalid cluster path: '{cluster_path}'") @@ -611,9 +631,9 @@ def create_payment_stream( ) @platform_client def fund_payment_stream( - platform: PlatformClient, - network: NetworkAPI, - account: AccountAPI, + platform: "PlatformClient", + network: "NetworkAPI", + account: "AccountAPI", cluster_path: str, token_amount: int | None, stream_time: timedelta | None, @@ -623,6 +643,7 @@ def fund_payment_stream( NOTE: You can fund anyone else's Stream! """ + from silverback.cluster.types import ResourceStatus if "/" not in cluster_path or len(cluster_path.split("/")) > 2: raise click.BadArgumentUsage(f"Invalid cluster path: '{cluster_path}'") @@ -683,9 +704,9 @@ def fund_payment_stream( @click.argument("cluster_path", metavar="CLUSTER") @platform_client def cancel_payment_stream( - platform: PlatformClient, - network: NetworkAPI, - account: AccountAPI, + platform: "PlatformClient", + network: "NetworkAPI", + account: "AccountAPI", cluster_path: str, ): """ @@ -693,6 +714,7 @@ def cancel_payment_stream( NOTE: Only the Stream owner can perform this action! """ + from silverback.cluster.types import ResourceStatus if "/" not in cluster_path or len(cluster_path.split("/")) > 2: raise click.BadArgumentUsage(f"Invalid cluster path: '{cluster_path}'") @@ -722,7 +744,7 @@ def cancel_payment_stream( @cluster.command(name="info") @cluster_client -def cluster_info(cluster: ClusterClient): +def cluster_info(cluster: "ClusterClient"): """Get Configuration information about a CLUSTER""" # NOTE: This actually doesn't query the cluster's routes, which are protected @@ -737,7 +759,7 @@ def cluster_info(cluster: ClusterClient): @cluster.command(name="health") @cluster_client -def cluster_health(cluster: ClusterClient): +def cluster_health(cluster: "ClusterClient"): """Get Health information about a CLUSTER""" click.echo(yaml.safe_dump(cluster.health.model_dump())) @@ -755,7 +777,7 @@ def registry_auth(): @registry_auth.command(name="list") @cluster_client -def credentials_list(cluster: ClusterClient): +def credentials_list(cluster: "ClusterClient"): """List container registry credentials""" if creds := list(cluster.registry_credentials): @@ -768,7 +790,7 @@ def credentials_list(cluster: ClusterClient): @registry_auth.command(name="info") @click.argument("name") @cluster_client -def credentials_info(cluster: ClusterClient, name: str): +def credentials_info(cluster: "ClusterClient", name: str): """Show info about registry credentials""" if not (creds := cluster.registry_credentials.get(name)): @@ -781,7 +803,7 @@ def credentials_info(cluster: ClusterClient, name: str): @click.argument("name") @click.argument("registry") @cluster_client -def credentials_new(cluster: ClusterClient, name: str, registry: str): +def credentials_new(cluster: "ClusterClient", name: str, registry: str): """Add registry private registry credentials. This command will prompt you for a username and password. """ @@ -799,7 +821,7 @@ def credentials_new(cluster: ClusterClient, name: str, registry: str): @click.argument("name") @click.option("-r", "--registry") @cluster_client -def credentials_update(cluster: ClusterClient, name: str, registry: str | None = None): +def credentials_update(cluster: "ClusterClient", name: str, registry: str | None = None): """Update registry registry credentials""" if not (creds := cluster.registry_credentials.get(name)): raise click.UsageError(f"Unknown credentials '{name}'") @@ -814,7 +836,7 @@ def credentials_update(cluster: ClusterClient, name: str, registry: str | None = @registry_auth.command(name="remove") @click.argument("name") @cluster_client -def credentials_remove(cluster: ClusterClient, name: str): +def credentials_remove(cluster: "ClusterClient", name: str): """Remove a set of registry credentials""" if not (creds := cluster.registry_credentials.get(name)): raise click.UsageError(f"Unknown credentials '{name}'") @@ -851,7 +873,7 @@ def parse_envar(item: str): ) @click.argument("name") @cluster_client -def new_vargroup(cluster: ClusterClient, variables: dict, name: str): +def new_vargroup(cluster: "ClusterClient", variables: dict, name: str): """Create a new group of environment variables in a CLUSTER""" if len(variables) == 0: @@ -863,7 +885,7 @@ def new_vargroup(cluster: ClusterClient, variables: dict, name: str): @vars.command(name="list") @cluster_client -def list_vargroups(cluster: ClusterClient): +def list_vargroups(cluster: "ClusterClient"): """List latest revisions of all variable groups in a CLUSTER""" if group_names := list(cluster.variable_groups): @@ -876,7 +898,7 @@ def list_vargroups(cluster: ClusterClient): @vars.command(name="info") @click.argument("name") @cluster_client -def vargroup_info(cluster: ClusterClient, name: str): +def vargroup_info(cluster: "ClusterClient", name: str): """Show latest revision of a variable GROUP in a CLUSTER""" if not (vg := cluster.variable_groups.get(name)): @@ -909,7 +931,7 @@ def vargroup_info(cluster: ClusterClient, name: str): @click.argument("name") @cluster_client def update_vargroup( - cluster: ClusterClient, + cluster: "ClusterClient", name: str, new_name: str, updated_vars: dict[str, str], @@ -945,7 +967,7 @@ def update_vargroup( @vars.command(name="remove") @click.argument("name") @cluster_client -def remove_vargroup(cluster: ClusterClient, name: str): +def remove_vargroup(cluster: "ClusterClient", name: str): """ Remove a variable GROUP from a CLUSTER @@ -977,7 +999,7 @@ def bots(): @click.argument("name") @cluster_client def new_bot( - cluster: ClusterClient, + cluster: "ClusterClient", image: str, network: str, account: str | None, @@ -1025,7 +1047,7 @@ def new_bot( @bots.command(name="list", section="Configuration Commands") @cluster_client -def list_bots(cluster: ClusterClient): +def list_bots(cluster: "ClusterClient"): """List all bots in a CLUSTER (Regardless of status)""" if bot_names := list(cluster.bots): @@ -1038,7 +1060,7 @@ def list_bots(cluster: ClusterClient): @bots.command(name="info", section="Configuration Commands") @click.argument("bot_name", metavar="BOT") @cluster_client -def bot_info(cluster: ClusterClient, bot_name: str): +def bot_info(cluster: "ClusterClient", bot_name: str): """Get configuration information of a BOT in a CLUSTER""" if not (bot := cluster.bots.get(bot_name)): @@ -1080,7 +1102,7 @@ def bot_info(cluster: ClusterClient, bot_name: str): @click.argument("name", metavar="BOT") @cluster_client def update_bot( - cluster: ClusterClient, + cluster: "ClusterClient", new_name: str | None, image: str | None, network: str | None, @@ -1159,7 +1181,7 @@ def update_bot( @bots.command(name="remove", section="Configuration Commands") @click.argument("name", metavar="BOT") @cluster_client -def remove_bot(cluster: ClusterClient, name: str): +def remove_bot(cluster: "ClusterClient", name: str): """Remove BOT from CLUSTER (Shutdown if running)""" if not (bot := cluster.bots.get(name)): @@ -1175,7 +1197,7 @@ def remove_bot(cluster: ClusterClient, name: str): @bots.command(name="health", section="Bot Operation Commands") @click.argument("bot_name", metavar="BOT") @cluster_client -def bot_health(cluster: ClusterClient, bot_name: str): +def bot_health(cluster: "ClusterClient", bot_name: str): """Show current health of BOT in a CLUSTER""" if not (bot := cluster.bots.get(bot_name)): @@ -1187,7 +1209,7 @@ def bot_health(cluster: ClusterClient, bot_name: str): @bots.command(name="start", section="Bot Operation Commands") @click.argument("name", metavar="BOT") @cluster_client -def start_bot(cluster: ClusterClient, name: str): +def start_bot(cluster: "ClusterClient", name: str): """Start BOT running in CLUSTER (if stopped or terminated)""" if not (bot := cluster.bots.get(name)): @@ -1203,7 +1225,7 @@ def start_bot(cluster: ClusterClient, name: str): @bots.command(name="stop", section="Bot Operation Commands") @click.argument("name", metavar="BOT") @cluster_client -def stop_bot(cluster: ClusterClient, name: str): +def stop_bot(cluster: "ClusterClient", name: str): """Stop BOT from running in CLUSTER (if running)""" if not (bot := cluster.bots.get(name)): @@ -1233,7 +1255,7 @@ def stop_bot(cluster: ClusterClient, name: str): callback=timedelta_callback, ) @cluster_client -def show_bot_logs(cluster: ClusterClient, name: str, log_level: str, since: timedelta | None): +def show_bot_logs(cluster: "ClusterClient", name: str, log_level: str, since: timedelta | None): """Show runtime logs for BOT in CLUSTER""" start_time = None @@ -1255,7 +1277,7 @@ def show_bot_logs(cluster: ClusterClient, name: str, log_level: str, since: time @bots.command(name="errors", section="Bot Operation Commands") @click.argument("name", metavar="BOT") @cluster_client -def show_bot_errors(cluster: ClusterClient, name: str): +def show_bot_errors(cluster: "ClusterClient", name: str): """Show unacknowledged errors for BOT in CLUSTER""" if not (bot := cluster.bots.get(name)): diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index 0435bbcf..ebc6e2f7 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -1,16 +1,12 @@ from datetime import datetime, timedelta from functools import update_wrapper from pathlib import Path +from typing import TYPE_CHECKING import click -from ape import Contract, convert -from ape.contracts import ContractInstance -from ape.types import AddressType from fief_client import Fief from fief_client.integrations.cli import FiefAuth, FiefAuthNotAuthenticatedError -from silverback._importer import import_from_string -from silverback.cluster.client import ClusterClient, PlatformClient from silverback.cluster.settings import ( PROFILE_PATH, BaseProfile, @@ -18,14 +14,21 @@ PlatformProfile, ProfileSettings, ) +from silverback.exceptions import ImportFromStringError + +if TYPE_CHECKING: + from ape.contracts import ContractInstance + + from silverback.cluster.client import PlatformClient -from .exceptions import ImportFromStringError # NOTE: only load once settings = ProfileSettings.from_config_file() def cls_import_callback(ctx, param, cls_name): + from silverback._importer import import_from_string + if cls_name is None: return None # User explicitly provided None @@ -38,7 +41,10 @@ def cls_import_callback(ctx, param, cls_name): def contract_callback( ctx: click.Context, param: click.Parameter, contract_address: str -) -> ContractInstance: +) -> "ContractInstance": + from ape import Contract, convert + from ape.types import AddressType + return Contract(convert(contract_address, AddressType)) @@ -50,6 +56,8 @@ def token_amount_callback( if token_amount is None: return None + from ape import convert + return convert(token_amount, int) @@ -239,6 +247,8 @@ def get_platform_client(ctx: click.Context, *args, **kwargs): except FiefAuthNotAuthenticatedError as e: raise click.UsageError("Not authenticated, please use `silverback login` first.") from e + from silverback.cluster.client import PlatformClient + ctx.obj["platform"] = PlatformClient( base_url=profile.host, cookies=dict(session=auth.access_token_info()["access_token"]), @@ -305,13 +315,15 @@ def inject_cluster(ctx, param, value: str | None): def get_cluster_client(ctx: click.Context, *args, **kwargs): ctx.obj = ctx.obj or {} if isinstance(profile := ctx.obj.get("profile"), ClusterProfile): + from silverback.cluster.client import ClusterClient + kwargs["cluster"] = ClusterClient( base_url=profile.host, headers={"X-API-Key": profile.api_key}, ) elif isinstance(profile, PlatformProfile): - platform: PlatformClient = ctx.obj["platform"] + platform: "PlatformClient" = ctx.obj["platform"] kwargs["cluster"] = platform.get_cluster_client(*ctx.obj["cluster_path"]) else: @@ -328,6 +340,8 @@ def bot_path_callback(ctx: click.Context, param: click.Parameter, path: str | No elif ":" not in path: path += ":bot" + from silverback._importer import import_from_string + try: return import_from_string(path) except ImportFromStringError: From 33755f3df2bc15d4f9e4b5f862e1ee77c2fdef6b Mon Sep 17 00:00:00 2001 From: Juliya Smith Date: Mon, 4 Nov 2024 07:41:57 -0600 Subject: [PATCH 2/2] chore: flake8 --- silverback/_cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 60886504..a9cd138f 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -714,8 +714,6 @@ def cancel_payment_stream( NOTE: Only the Stream owner can perform this action! """ - from silverback.cluster.types import ResourceStatus - if "/" not in cluster_path or len(cluster_path.split("/")) > 2: raise click.BadArgumentUsage(f"Invalid cluster path: '{cluster_path}'")