Merge pull request #753 from FlorentinD/aura-ci-cleanup-sigterm

aura ci cleanup sigterm

scripts/ci/aura_api_ci.py

@@ -81,8 +81,6 @@ def create_instance(self, name: str, memory: str, type: str) -> Dict[str, Any]:
             if should_retry:
                 logging.debug(f"Error code: {response.status_code} - Retrying in {wait_time} s")
 
-        response.raise_for_status()
-
         return response.json()["data"]  # type: ignore
 
     def check_running(self, db_id: str) -> None:
@@ -115,7 +113,7 @@ def check_running(self, db_id: str) -> None:
 
         response.raise_for_status()
 
-    def teardown_instance(self, db_id: str) -> None:
+    def teardown_instance(self, db_id: str) -> bool:
         TEARDOWN_MAX_WAIT_TIME = 10
 
         should_retry = True
@@ -138,8 +136,13 @@ def teardown_instance(self, db_id: str) -> None:
             if should_retry:
                 logging.debug(f"Status code: {response.status_code} - Retrying in {wait_time} s")
 
+        if response.status_code == 404:
+            return False
+
         response.raise_for_status()
 
+        return True
+
     def get_tenant_id(self) -> str:
         if self._tenant_id:
             return self._tenant_id

scripts/ci/run_targeting_aura_sessions.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import random as rd
+import signal
 import sys
 
 from aura_api_ci import AuraApiCI
@@ -23,6 +24,15 @@ def main() -> None:
     instance_id = create_result["id"]
     logging.info(f"Creation of database with id '{instance_id}'")
 
+    # Teardown instance on SIGNAL
+    def handle_signal(sig, frame):
+        logging.info("Received SIGNAL, tearing down instance")
+        aura_api.teardown_instance(instance_id)
+        sys.exit(1)
+
+    signal.signal(signal.SIGINT, handle_signal)
+    signal.signal(signal.SIGTERM, handle_signal)
+
     try:
         aura_api.check_running(instance_id)
         logging.info("Database up and running")

scripts/ci/run_targeting_aurads.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import random as rd
+import signal
 import sys
 
 from aura_api_ci import AuraApiCI
@@ -28,7 +29,7 @@ def run_notebooks(uri: str, username: str, password: str) -> None:
 def main() -> None:
     client_id = os.environ["AURA_API_CLIENT_ID"]
     client_secret = os.environ["AURA_API_CLIENT_SECRET"]
-    tenant_id = os.environ.get("TENANT_ID")
+    tenant_id = os.environ.get("AURA_API_TENANT_ID")
     aura_api = AuraApiCI(client_id=client_id, client_secret=client_secret, tenant_id=tenant_id)
 
     MAX_INT = 1000000
@@ -38,6 +39,14 @@ def main() -> None:
     instance_id = create_result["id"]
     logging.info("Creation of database accepted")
 
+    def handle_signal(sig, frame):
+        logging.info("Received SIGNAL, tearing down instance")
+        aura_api.teardown_instance(instance_id)
+        sys.exit(1)
+
+    signal.signal(signal.SIGINT, handle_signal)
+    signal.signal(signal.SIGTERM, handle_signal)
+
     try:
         aura_api.check_running(instance_id)
         logging.info("Database %s up and running", instance_id)

scripts/run_notebooks.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import signal
 import sys
 from datetime import datetime
 from pathlib import Path
@@ -29,6 +30,16 @@ def init_notebook(self, version_cell_index: int, tear_down_cells: List[IndexedCe
 
     # run the cell of a notebook
     def preprocess_cell(self, cell: Any, resources: Any, index: int) -> None:
+        if index == 0:
+
+            def handle_signal(sig, frame):
+                print("Received SIGNAL, running tear down cells")
+                self.teardown(resources)
+                sys.exit(1)
+
+            signal.signal(signal.SIGINT, handle_signal)
+            signal.signal(signal.SIGTERM, handle_signal)
+
         try:
             if not self._skip_rest:
                 super().preprocess_cell(cell, resources, index)  # type: ignore
@@ -40,13 +51,16 @@ def preprocess_cell(self, cell: Any, resources: Any, index: int) -> None:
 
             if self.tear_down_cells:
                 print(f"Running tear down cells due to error in notebook execution: {e}")
-                for td_cell, td_idx in self.tear_down_cells:
-                    try:
-                        super().preprocess_cell(td_cell, resources, td_idx)  # type: ignore
-                    except CellExecutionError as td_e:
-                        print(f"Error running tear down cell {td_idx}: {td_e}")
+                self.teardown(resources)
             raise e
 
+    def teardown(self, resources) -> None:
+        for td_cell, td_idx in self.tear_down_cells:
+            try:
+                super().preprocess_cell(td_cell, resources, td_idx)  # type: ignore
+            except CellExecutionError as td_e:
+                print(f"Error running tear down cell {td_idx}: {td_e}")
+
 
 class GdsTearDownCollector(ExecutePreprocessor):
     def __init__(self, **kw: Any):