record_pool_stats.py

import argparse
import decimal
import json
import logging
import os
import requests
import time
from datetime import datetime

import web3

from util import get_fleek_client

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()

STATS_FILE_PATH = os.environ["STATS_FILE_PATH"]
# IMPORTANT: Use checksummed addresses here
SWAP_CONTRACT_ADDRESS = os.environ["SWAP_CONTRACT_ADDRESS"]
DEPLOYMENT_BLOCK = int(os.environ["DEPLOYMENT_BLOCK"])
ADD_STATS_EVERY_N_BLOCK = int(os.environ["ADD_STATS_EVERY_N_BLOCK"])
HTTP_PROVIDER_URL = os.environ["HTTP_PROVIDER_URL"]
FLEEK_KEY_ID = os.environ["FLEEK_KEY_ID"]
FLEEK_KEY = os.environ["FLEEK_KEY"]
FLEEK_BUCKET = os.environ["FLEEK_BUCKET"]
INITIAL_VIRTUAL_PRICE = "1000000000000000000"

# TODO npm run build before this is accessible
SWAP_CONTRACT_ABI_PATH = "Swap.json"
FLEEK_ENDPOINT = "https://storageapi.fleek.co"

# Add timestamp, A, adminfee, swapfee in the future?
BLOCK_NUMBER_IND = 0
VIRTUAL_PRICE_IND = 1
BTC_PRICE_IND = 2


def get_btc_price_at_timestamp_date(ts):
    dt = datetime.fromtimestamp(ts)
    logger.info(f"Fetching price for timestamp: {ts} ({dt})")
    try:
        end_ts = ts
        start_ts = ts - 60 * 60 * 2
        url = (
            f"https://api.coingecko.com/api/v3/coins/bitcoin/market_chart/"
            f"range?vs_currency=usd&from={start_ts}&to={end_ts}"
        )
        price_info = requests.get(url).json()
        prices = price_info["prices"]
        price = int(decimal.Decimal(prices[-1][1]).quantize(decimal.Decimal("1")))
        return price
    except Exception as e:

        logger.error(f"Could not fetch btc price at {ts}: {e}")


def get_existing_stats_file_content(fleek_aws_client):
    try:
        obj = fleek_aws_client.get_object(Bucket=FLEEK_BUCKET, Key=STATS_FILE_PATH)
        return json.loads(obj["Body"].read().decode("utf-8"))
    except fleek_aws_client.exceptions.NoSuchKey:
        logger.info("No existing file has been found.")
        return []
    except Exception as e:
        logger.error(f"Error reading existing file: {e}")
        return None


def main(args):
    if args.dev:
        logger.info(
            "IMPORTANT: Remember to delete the existing file from"
            f"{FLEEK_BUCKET} bucket when starting to test a fresh env."
        )
        logger.info(
            "Running in dev mode, will generate stats as new blocks are" " mined."
        )
    try:
        f = open(SWAP_CONTRACT_ABI_PATH)
        swap_contract_artifact = json.loads(f.read())
        swap_contract_abi = swap_contract_artifact["abi"]
    except Exception as e:
        logger.error(f"Could not load swap contract ABI: {e}")

    w3 = web3.Web3(web3.Web3.HTTPProvider(HTTP_PROVIDER_URL))
    fleek_aws_client = get_fleek_client(FLEEK_KEY_ID, FLEEK_KEY)

    stats_content = get_existing_stats_file_content(fleek_aws_client)
    if stats_content is None:
        logger.error(
            "Error reading existing file. This can happen due to a"
            " Fleek outage. If it happens consistenly, delete the file "
            "from Fleek and wait for the script to regenerate the file "
            "from scratch."
        )
        return

    # Get the last block number
    if len(stats_content):
        last_block_num = stats_content[-1][BLOCK_NUMBER_IND]
    else:
        logger.info("Creating new file")
        last_block_num = DEPLOYMENT_BLOCK

    swap = w3.eth.contract(abi=swap_contract_abi, address=SWAP_CONTRACT_ADDRESS)

    next_block_num = last_block_num + ADD_STATS_EVERY_N_BLOCK
    logger.info(
        f"Next block number: {next_block_num}, current block: {w3.eth.blockNumber}"
    )

    while True:

        # w3.eth.blockNumber gets updated dynamically
        while w3.eth.blockNumber > next_block_num:
            logger.info(f"Fetching data for block: {next_block_num}")

            # Virtual price has more digits than Number.MAX_SAFE_INTEGER.
            # When the pool initializes, the virtual price is returned as 0 but it's
            # actually effectively 1
            try:
                virtual_price = str(
                    swap.functions.getVirtualPrice().call(
                        block_identifier=next_block_num
                    )
                    or INITIAL_VIRTUAL_PRICE
                )
            except web3.exceptions.BadFunctionCallOutput as e:
                logger.error(
                    f"Error calling fn on block {next_block_num}. This is "
                    f"likely due to the contract not having been deployed by"
                    f" this block, moving to the next block: {e}"
                )
                next_block_num += ADD_STATS_EVERY_N_BLOCK
                continue

            block_data = w3.eth.getBlock(next_block_num)
            btc_price = get_btc_price_at_timestamp_date(block_data.timestamp)
            if not btc_price:
                break

            stats_content.append([next_block_num, virtual_price, btc_price])

            # Rewrite the whole object every time, helps recover from where we left off,
            # if we're regenerating a lot of blocks and script stops due to provider
            # / rate limiting errors
            stats_bytes = json.dumps(stats_content, separators=(",", ":")).encode(
                "utf-8"
            )
            while True:
                try:
                    fleek_aws_client.put_object(
                        Bucket=FLEEK_BUCKET, Key=STATS_FILE_PATH, Body=stats_bytes
                    )
                    logger.info(
                        f"Uploaded cumulative stats to Fleek (latest block: {next_block_num})"
                    )
                    break
                except Exception as e:
                    logger.error(f"Error uploading stats file: {e}")

            next_block_num += ADD_STATS_EVERY_N_BLOCK

        if not args.dev:
            break

        time.sleep(1)

    logger.info("Done for now.")


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--dev", action="store_true")
    args = parser.parse_args()
    main(args)