progress in regtest migration

counterparty-core/counterpartycore/lib/api/apiserver.py

@@ -610,12 +610,15 @@ def __init__(self, stop_event):
     def start(self, args, log_stream):
         if self.process is not None:
             raise Exception("API Server is already running")
+        logger.warning("Starting API Server1...")
         self.process = Process(
             name="API",
             target=run_apiserver,
             args=(vars(args), self.server_ready_value, self.stop_event, os.getpid(), log_stream),
         )
+        logger.warning("Starting API Server2...")
         self.process.start()
+        logger.warning("Starting API Server3...")
         return self.process
 
     def is_ready(self):

counterparty-core/counterpartycore/lib/ledger/currentstate.py

@@ -74,6 +74,12 @@ def parsing_mempool(self):
     def block_parser_status(self):
         return self.state.get("BLOCK_PARSER_STATUS", "starting")
 
+    def set_stopping(self, stopping):
+        self.state["STOPPING"] = stopping
+
+    def stopping(self):
+        return self.state.get("STOPPING", False)
+
 
 class ConsensusHashBuilder(metaclass=helpers.SingletonMeta):
     def __init__(self):

counterparty-core/counterpartycore/lib/parser/follow.py

@@ -291,6 +291,8 @@ async def handle(self):
 
                 # Yield control to the event loop to allow other tasks to run
                 await asyncio.sleep(0)
+                if CurrentState().stopping():
+                    raise KeyboardInterrupt
             except asyncio.CancelledError:
                 logger.debug("BlockchainWatcher.handle() was cancelled.")
                 break  # Exit the loop if the task is cancelled

counterparty-core/counterpartycore/regtest/dreddhooks.py

@@ -1,13 +1,4 @@
 import dredd_hooks as hooks
-from counterpartycore.lib import config
-
-config.BACKEND_URL = "http://rpc:rpc@localhost:18443"
-config.BACKEND_SSL_NO_VERIFY = True
-config.REQUESTS_TIMEOUT = 20
-config.ADDRESSVERSION = config.ADDRESSVERSION_REGTEST
-config.NETWORK_NAME = "regtest"
-
-API_ROOT = "http://localhost:24000"
 
 
 @hooks.before_each

counterparty-core/counterpartycore/regtest/genapidoc.py

@@ -11,8 +11,8 @@
 from counterpartycore.lib.utils import database
 
 CURR_DIR = os.path.dirname(os.path.realpath(__file__))
-API_BLUEPRINT_FILE = os.path.join(CURR_DIR, "../../../../apiary.apib")
-DREDD_FILE = os.path.join(CURR_DIR, "../../../../dredd.yml")
+API_BLUEPRINT_FILE = os.path.join(CURR_DIR, "../../../apiary.apib")
+DREDD_FILE = os.path.join(CURR_DIR, "../../../dredd.yml")
 CACHE_FILE = os.path.join(CURR_DIR, "apidoc", "apicache.json")
 API_ROOT = "http://localhost:24000"
 
@@ -126,7 +126,7 @@
 DREDD_CONFIG = {
     "loglevel": "error",
     "language": "python",
-    "hookfiles": "./counterparty-core/counterpartycore/test/regtest/dreddhooks.py",
+    "hookfiles": "./counterparty-core/counterpartycore/regtest/dreddhooks.py",
     "path": [],
     "blueprint": "apiary.apib",
     "endpoint": "http://127.0.0.1:24000",
@@ -626,6 +626,9 @@ def generate_regtest_fixtures(db):
     print("Generating API documentation...")
     data_dir = sys.argv[1] if len(sys.argv) > 1 else "regtestnode"
     print(f"Using data directory: {data_dir}")
+    assert os.path.exists(
+        f"{data_dir}/counterparty.regtest.db"
+    ), f"Data directory {data_dir} does not exist."
     db = database.get_db_connection(
         f"{data_dir}/counterparty.regtest.db", read_only=True, check_wal=False
     )

counterparty-core/counterpartycore/regtest/regtestnode.py

@@ -19,6 +19,7 @@
 from bitcoinutils.transactions import Transaction, TxInput, TxOutput
 from counterpartycore.lib import config, exceptions
 from counterpartycore.lib.cli import main, server
+from counterpartycore.lib.ledger.currentstate import CurrentState
 from counterpartycore.lib.utils import database
 
 setup("regtest")
@@ -38,17 +39,38 @@ def __init__(self, datadir, log_stream):
                 "--gunicorn-workers=2",
                 "--no-telemetry",
                 "--electrs-url=http://localhost:3002",
-                "start",
                 "-vv",
+                "start",
             ]
         )
+        print("ARGS:", self.args)
         self.log_stream = log_stream
-        print(self.args)
         server.initialise_log_and_config(self.args, log_stream=self.log_stream)
 
     def run(self):
         server.start_all(self.args, self.log_stream)
 
+    def stop(self):
+        print("ASK TO STOP")
+        CurrentState().set_stopping(True)
+        while True:
+            if "Shutdown complete." in self.log_stream.getvalue():
+                print("Server stopped")
+                time.sleep(2)
+                return
+            print("Waiting for counterparty server to stop...")
+            time.sleep(1)
+
+    def command(self, action, arg=None):
+        if action == "reparse":
+            print("Reparse...")
+            server.reparse(block_index=arg)
+            print("Reparse done")
+        elif action == "rollback":
+            server.rollback(block_index=arg)
+        elif action == "check-db":
+            server.check_database()
+
 
 class RegtestNode:
     def __init__(
@@ -82,15 +104,15 @@ def __init__(
         self.tx_index = -1
         self.ready = False
         self.show_output = show_output
-        self.counterparty_server = sh.counterparty_server.bake(
-            "--regtest",
-            f"--data-dir={self.datadir}",
-            f"--wsgi-server={wsgi_server}",
-            "--gunicorn-workers=2",
-            "--no-telemetry",
-            "--electrs-url=http://localhost:3002",
-            "-vv",
-        )
+        # self.counterparty_server = sh.counterparty_server.bake(
+        #    "--regtest",
+        #    f"--data-dir={self.datadir}",
+        #    f"--wsgi-server={wsgi_server}",
+        #    "--gunicorn-workers=2",
+        #    "--no-telemetry",
+        #    "--electrs-url=http://localhost:3002",
+        #    "-vv",
+        # )
         self.burn_in_one_block = burn_in_one_block
         self.electrs_process = None
         self.electrs_process_2 = None
@@ -99,7 +121,8 @@ def __init__(
         self.second_node_started = False
 
     def start_counterparty_server(self):
-        self.counterparty_node = CounterpartyNode(self.datadir, self.server_out).start()
+        self.counterparty_node = CounterpartyNode(self.datadir, self.server_out)
+        self.counterparty_node.start()
 
     def api_call(self, url):
         result = sh.curl(f"http://localhost:24000/v2/{url}").strip()
@@ -308,7 +331,7 @@ def wait_for_counterparty_follower(self):
 
     def wait_for_counterparty_watcher(self):
         while True:
-            if "API.Watcher - Catch up completed." in self.server_out.getvalue():
+            if "Catch up completed." in self.server_out.getvalue():
                 print("Server ready")
                 return
             print("Waiting for counterparty server...")
@@ -522,10 +545,7 @@ def kill_gunicorn_workers(self):
 
     def stop_counterparty_server(self):
         try:
-            # self.counterparty_server_process.terminate()
-            # self.counterparty_server_process.wait()
-            # self.kill_gunicorn_workers()
-            pass
+            self.counterparty_node.stop()
         except Exception as e:
             print(e)
             pass
@@ -540,14 +560,14 @@ def stop_bitcoin_node(self, node=1):
             pass
 
     def stop(self):
+        print("Stopping counterparty-server...")
+        self.stop_counterparty_server()
         print("Stopping electrs...")
         self.stop_electrs()
         print("Stopping bitcoin node 1...")
         self.stop_bitcoin_node()
         print("Stopping bitcoin node 2...")
         self.stop_bitcoin_node(node=2)
-        print("Stopping counterparty-server...")
-        self.stop_counterparty_server()
 
     def get_node_state(self):
         try:
@@ -561,14 +581,11 @@ def get_node_state(self):
             return self.get_node_state()
 
     def check_node_state(self, command, previous_state):
+        print(f"Checking node state after {command}...")
+        print(self.server_out.getvalue())
+        print(("---------------------------------"))
         self.server_out = StringIO()
-        self.counterparty_server_process = self.counterparty_server(
-            "start",
-            _bg=True,
-            _out=self.server_out,
-            _err_to_out=True,
-            _bg_exc=False,
-        )
+        self.start_counterparty_server()
         self.wait_for_counterparty_follower()
         self.wait_for_counterparty_watcher()
         time.sleep(2)
@@ -583,19 +600,11 @@ def test_command(self, command):
         self.stop_counterparty_server()
         print(f"Running `{command}`...")
         if command == "check-db":
-            self.counterparty_server(
-                command,
-                _out=sys.stdout,
-                _err_to_out=True,
-                _bg_exc=False,
-            )
+            self.counterparty_node.command(command)
         else:
-            self.counterparty_server(
+            self.counterparty_node.command(
                 command,
                 150,  # avoid tx using `disable_protocol_changes` params (scenario_6_dispenser.py)
-                _out=sys.stdout,
-                _err_to_out=True,
-                _bg_exc=False,
             )
         self.check_node_state(command, state_before)
         print(f"`{command}` successful")
@@ -1245,16 +1254,19 @@ def test_rbf(self):
 
 
 class RegtestNodeThread(threading.Thread):
-    def __init__(self, wsgi_server="waitress", burn_in_one_block=True):
+    def __init__(self, wsgi_server="waitress", burn_in_one_block=True, datadir="regtestnode"):
         threading.Thread.__init__(self)
         self.wsgi_server = wsgi_server
         self.burn_in_one_block = burn_in_one_block
         self.daemon = True
         self.node = None
+        self.datadir = datadir
 
     def run(self):
         self.node = RegtestNode(
-            wsgi_server=self.wsgi_server, burn_in_one_block=self.burn_in_one_block
+            wsgi_server=self.wsgi_server,
+            burn_in_one_block=self.burn_in_one_block,
+            datadir=self.datadir,
         )
         self.node.start()
 

counterparty-core/counterpartycore/regtest/scenarios_test.py

@@ -70,9 +70,9 @@
 SCENARIOS += scenario_last_mempool.SCENARIO
 
 CURR_DIR = os.path.dirname(os.path.realpath(__file__))
-BASE_DIR = os.path.join(CURR_DIR, "../../../../")
+BASE_DIR = os.path.join(CURR_DIR, "../../../")
 
-# SCENARIOS = scenario_25_issuance.SCENARIO
+SCENARIOS = scenario_25_issuance.SCENARIO
 
 
 def compare_strings(string1, string2):
@@ -396,15 +396,15 @@ def check_api_v1(node):
 
 def run_scenarios(serve=False, wsgi_server="waitress"):
     try:
-        regtest_node_thread = RegtestNodeThread(wsgi_server=wsgi_server)
+        regtest_node_thread = RegtestNodeThread(wsgi_server=wsgi_server, datadir="regtest-node")
         regtest_node_thread.start()
 
         while not regtest_node_thread.ready():
             time.sleep(1)
 
         context = {}
 
-        # check_api_v1(regtest_node_thread.node)
+        check_api_v1(regtest_node_thread.node)
 
         # run all scenarios
         for item in SCENARIOS:
@@ -419,23 +419,28 @@ def run_scenarios(serve=False, wsgi_server="waitress"):
                 )
                 time.sleep(1)
         else:
+            regtest_node_thread.node.test_asset_conservation()
+            regtest_node_thread.node.reparse()
+
             print("Generating API documentation...")
             if os.path.exists(os.path.join(CURR_DIR, "apidoc/apicache.json")):
                 os.unlink(os.path.join(CURR_DIR, "apidoc/apicache.json"))
+            print("DATA DIR", os.path.abspath("regtest-node"))
             sh.python3(
                 os.path.join(CURR_DIR, "genapidoc.py"),
-                os.path.abspath("regtestnode"),
+                os.path.abspath("regtest-node"),
                 _out=sys.stdout,
                 _err_to_out=True,
                 _cwd=CURR_DIR,
             )
-            return
+
             print("Running Dredd...")
             sh.dredd(
                 _cwd=BASE_DIR,
                 _out=sys.stdout,
                 _err_to_out=True,
             )
+
             print("Testing invalid detach...")
             regtest_node_thread.node.test_invalid_detach()
             print("Testing transaction chaining...")

dredd.yml

@@ -1,6 +1,6 @@
 blueprint: apiary.apib
 endpoint: http://127.0.0.1:24000
-hookfiles: ./counterparty-core/counterpartycore/test/regtest/dreddhooks.py
+hookfiles: ./counterparty-core/counterpartycore/regtest/dreddhooks.py
 language: python
 loglevel: error
 only: