ignore error in __del__, set status to complete

src/daisy/storage.py

@@ -606,6 +606,14 @@ def upload_metrics_tables(infiles: list,
         pool.join()
 
 
+def mark_upload_complete(url: str, run_id: int):
+    engine = create_engine(url)
+    session = sessionmaker(bind=engine)()
+    benchmark_run = session.query(BenchmarkRun).filter_by(id=run_id).first()
+    benchmark_run.status = "complete"
+    session.commit()
+
+
 def upload_result(infiles, outfile, *extras):
     """upload results into database.
 
@@ -729,9 +737,7 @@ def upload_result(infiles, outfile, *extras):
     #            schema=None,
     #            is_sqlite3=is_sqlite3)
 
-    # benchmark_run.status = "complete"
-    # session.commit()
-
+    mark_upload_complete(url, benchmark_run.id)
 
     logger.info("uploaded results under run_id {}".format(benchmark_run.id))
     touch(outfile)

src/daisy/table_cache.py

@@ -346,7 +346,12 @@ def close(self):
         if self.database_url is None:
             return
 
-        engine = create_engine(self.database_url)
+        try:
+            engine = create_engine(self.database_url)
+        except AttributeError:
+            # do not create indices if sqlalchemy already shut down
+            return
+
         if not self.have_created_indices:
             for index_name, info in self.indices.items():
                 table_name, fields = info

src/tests/test_storage.py

@@ -10,7 +10,7 @@
 def benchmark_layout(tmp_path):
 
     ntools = 2
-    nmetrics = 2000
+    nmetrics = 1000
 
     tools = [f"tool{x}" for x in range(ntools)]
     metrics = [f"metric{x}" for x in range(nmetrics)]
@@ -76,6 +76,8 @@ def test_upload(benchmark_layout, tmp_path, max_workers):
         run_df = pandas.read_sql("SELECT * FROM run", conn)
     assert len(run_df) == 1
 
+    assert run_df.status[0] == "complete"
+
     with db_engine.connect() as conn:
         ins_df = pandas.read_sql("SELECT * FROM instance", conn)
     assert list(ins_df.run_id.unique()) == [1]