Price api

.gitignore

@@ -1,5 +1,5 @@
 /*.db
-/config.py
+/src/config.py
 /__pycache__
 /oxend/*.sock
 */__pycache__

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+python_files = *_tests.py

run_fetcher.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+import src.app_fetcher

src/app_fetcher.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+from src import config
+from src.staking.fetcher import App
+
+app = App(config.backend.fetcher_name if config.backend.fetcher_name else __name__)
+app.run()

src/app_price.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+from src import config
+from src.price.app import create_app, PriceAppConfig
+
+price_config = PriceAppConfig(
+    name="price_api",
+    log_level=config.backend.log_level,
+    enable_perf=config.backend.performance_logging,
+    sqlite_db=config.backend.prices_sqlite_db,
+    sqlite_schema=config.backend.prices_sqlite_schema,
+    coingecko_api_key=config.backend.coingecko_api_key,
+    coingecko_api_url=config.backend.coingecko_api_url,
+    coingecko_api_token_ids=config.backend.coingecko_api_token_ids,
+    coingecko_api_currencies=config.backend.coingecko_api_currencies,
+    coingecko_api_rate_poll_rate_seconds=config.backend.prices_api_refetch_interval_seconds,
+    default_token=config.backend.prices_api_default_token,
+    default_currency=config.backend.prices_api_default_currency,
+)
+
+app = create_app(price_config)

src/app_registration.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+from src import config
+from src.registration.app import create_app, RegistrationAppConfig
+
+registration_config = RegistrationAppConfig(
+    name="registration_api",
+    log_level=config.backend.log_level,
+    enable_perf=config.backend.performance_logging,
+    sqlite_db=config.backend.registration_sqlite_db,
+    sqlite_schema=config.backend.registration_sqlite_schema,
+    api_rate_limit=config.backend.registration_api_rate_limit,
+    api_rate_limit_period=config.backend.registration_api_rate_limit_period,
+)
+
+app = create_app(registration_config)

src/app_snapshot.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from src import config
+from src.snapshot.app import create_app
+
+app = create_app(config)

src/app_staking.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+from src import config
+from src.staking.app import create_app, StakingAppConfig
+
+staking_app_config = StakingAppConfig(
+    name="staking_api",
+    log_level=config.backend.log_level,
+    enable_perf=config.backend.performance_logging,
+    sqlite_db=config.backend.sqlite_db,
+    sqlite_schema=config.backend.sqlite_schema,
+    sqlite_db_registrations=config.backend.registration_sqlite_db,
+    sqlite_schema_registrations=config.backend.registration_sqlite_schema,
+    rpc_api=config.backend.rpc_api,
+    rpc_api_cache=config.backend.rpc_api_cache,
+    rpc_shared=config.backend.rpc_shared,
+    rpc_shared_cache=config.backend.rpc_shared_cache,
+    rpc_api_usage_logging=config.backend.rpc_api_usage_logging,
+    log_level_generic=config.backend.log_level_generic,
+    cache_stale_time_seconds=config.backend.stale_time_seconds,
+    disable_db_file_rewrite=config.backend.disable_db_file_rewrite,
+    stale_time_seconds_contract_abis=config.backend.stale_time_seconds_contract_abis,
+    rpc_api_usage_logging_interval=config.backend.rpc_api_usage_logging_interval,
+)
+
+app = create_app(staking_app_config)

config.py → src/config.py

@@ -1,4 +1,4 @@
-from config_defaults import *
+from .config_defaults import *
 
 # Local settings.  Changes to this file are meant for a local installation (and should not be
 # committed to git).
@@ -12,3 +12,6 @@
 
 # backend              = stagenet_backend
 # backend.provider_url = 'tcp://127.0.0.1:6786'
+
+# backend.log_level = CUSTOM_LOG_LEVELS["SILLY"]
+# backend.log_level = logging.INFO

config_defaults.py → src/config_defaults.py

@@ -24,9 +24,11 @@ class Backend:
     log_level_generic                   = None  # Logs from other packages will use log_level if this is not set
     oxen_wallet_regex:              str = ""
     sqlite_db:                      str = "ssb.db"
-    sqlite_schema:                  str = "db/schema.sql"
+    sqlite_schema:                  str = "src/staking/schema.sql"
     rpc_shared:               list[str] = ""
     rpc_shared_cache:               int = 2
+    disable_db_file_rewrite:       bool = False
+
 
     """
     API CONFIG
@@ -35,23 +37,24 @@ class Backend:
     rpc_api:                        str = ""
     rpc_api_cache:                  int = 2
     rpc_api_usage_logging:         bool = False
-    rpc_api_usage_logging_interval: int = 600
+    rpc_api_usage_logging_interval: int = 300
+
     """
     REGISTRATION CONFIG
     """
     registration_api_name:              str = "registration_api"
     # NOTE: This can be the same DB as the main API, but you must manually run the registrations/schema.sql script in
     #   the main db so it can be populated with the required tables.
     registration_sqlite_db:             str = "ssb-registrations.db"
-    registration_sqlite_schema:         str = "registration/schema.sql"
+    registration_sqlite_schema:         str = "src/registration/schema.sql"
     # Creates a request per period limit by IP address (default is 100 requests per hour)
     registration_api_rate_limit:        int = 100
     registration_api_rate_limit_period: int = 3600
 
     """
     FETCHER CONFIG
     """
-    abi_dir = "web3client/abis"
+    abi_dir = "src/web3client/abis"
     # Arbitrum runs at ~4 blocks per second, and the rpc node has a limit of 30m, so scan for 120 blocks
     arbitrum_rescan_safety_blocks: int = 60
     arbitrum_scan_start_chunk_size: int = 20
@@ -82,6 +85,24 @@ class Backend:
     sqlite_snapshot_time_interval_seconds: int = 600
     snapshot_on_startup:                  bool = False
 
+    """
+    PRICE FETCHER CONFIG
+    """
+    prices_api_name:                    str = "prices_api"
+    enable_price_fetcher:               bool = False
+    coingecko_api_key:                  str = ""
+    coingecko_api_url:                  str = "https://api.coingecko.com/api"
+    coingecko_api_token_ids:      list[str] = ["ethereum", "chainflip"]
+    coingecko_api_currencies:     list[str] = ["usd", "aud"]
+
+    # Creates a request per period limit by IP address (default is 10 requests per 10 minutes)
+    prices_api_rate_limit:              int = 10
+    prices_api_rate_limit_period:       int = 600
+    prices_sqlite_db:                   str = "ssb-prices.db"
+    prices_sqlite_schema:               str = "price/schema.sql"
+    prices_api_default_currency:        str = "usd"
+    prices_api_default_token:           str = "ethereum"
+    prices_api_refetch_interval_seconds:int = 300
 
 # Session mainnet contracts
 mainnet_backend = Backend()

config_validate.py → src/config_validate.py

@@ -1,11 +1,11 @@
 import logging
 
-import config
-from log import Log
-from oxen.omq import omq_connection
-from oxen.rpc import OxenRPC
-from util import is_not_empty_string, valid_address_assertion
-from web3client.client import Web3Client
+from . import config
+from .log import Log
+from .oxen.rpc import OxenRPC
+from .util import is_not_empty_string, valid_address_assertion
+from .web3client.abi_manager import ABIManager
+from .web3client.client import Web3Client
 
 
 def validate_config(conf: config):

src/db/read.py

@@ -0,0 +1,20 @@
+import sqlite3
+
+from ..log import Log
+
+class DBReader:
+    def __init__(self, db_path: str, log_level: int, perf: bool = False, disable_db_file_rewrite: bool = False):
+        self.log = Log("db_reader", log_level, enable_perf=perf).logger
+
+        if not disable_db_file_rewrite:
+            if not db_path.startswith("file:"):
+                db_path = "file:" + db_path
+
+            if not db_path.endswith("?mode=ro"):
+                db_path = db_path + "?mode=ro"
+
+        self.log.info(f"Connecting to db at {db_path}")
+        self.db_path = db_path
+
+    def connect(self):
+        return sqlite3.connect(self.db_path, uri=True)

src/db/write.py

@@ -0,0 +1,12 @@
+import sqlite3
+
+from ..log import Log
+
+class DBWriter:
+    def __init__(self, db_path: str, log_level: int, perf: bool = False):
+        self.db_path = db_path
+        self.log = Log("db_writer", log_level, enable_perf=perf).logger
+        self.log.info(f"Connecting to db at {db_path}")
+
+    def connect(self):
+        return sqlite3.connect(self.db_path)

log/__init__.py → src/log/__init__.py

@@ -1,7 +1,7 @@
 import logging
 
-from log.perf import PerformanceLogger
-from log.util import add_logging_level
+from .perf import PerformanceLogger
+from .util import add_logging_level
 
 CUSTOM_LOG_LEVELS = {"SILLY": 1, "PERFORMANCE": 69}
 for label, lvl in CUSTOM_LOG_LEVELS.items():

log/time_keeper.py → src/log/time_keeper.py

@@ -2,8 +2,8 @@
 import statistics
 import time
 
-from log import PerformanceLogger
-from util import format_seconds, format_ms
+from ..log import PerformanceLogger
+from ..util import format_seconds, format_ms
 
 
 class TimeKeeper:

oxen/rpc.py → src/oxen/rpc.py

@@ -1,8 +1,9 @@
 import logging
 from typing import TypedDict
-from oxen.omq import FutureJSON, omq_connection, RPCUsageTracker
 from dataclasses import dataclass
 
+from ..oxen.omq import FutureJSON, omq_connection, RPCUsageTracker
+
 
 class ServiceNodeContributor(TypedDict):
     address: str
@@ -164,15 +165,15 @@ def get_network_info_from_network(self):
         self.log.silly("get_network_info_from_network info: {}".format(info))
 
         return NetworkInfo(
-            block_hash=info.get("top_block_hash"),
-            block_height=info.get("height"),
-            hard_fork=info.get("hard_fork"),
-            immutable_block_hash=info.get("immutable_block_hash"),
-            immutable_block_height=info.get("immutable_height"),
-            max_stakers=info.get("max_contributors"),
-            min_operator_contribution=info.get("min_operator_contribution"),
-            nettype=info.get("nettype"),
-            pulse_target_timestamp=info.get("pulse_target_timestamp"),
-            staking_requirement=info.get("staking_requirement"),
-            version=info.get("version"),
+            block_hash=info.get("top_block_hash", "NA"),
+            block_height=info.get("height", 0),
+            hard_fork=info.get("hard_fork", "NA"),
+            immutable_block_hash=info.get("immutable_block_hash", "NA"),
+            immutable_block_height=info.get("immutable_height", 0),
+            max_stakers=info.get("max_contributors", 0),
+            min_operator_contribution=info.get("min_operator_contribution", 0),
+            nettype=info.get("nettype", "NA"),
+            pulse_target_timestamp=info.get("pulse_target_timestamp", 0),
+            staking_requirement=info.get("staking_requirement", 0),
+            version=info.get("version", "NA"),
         )