diff --git a/AIPscan/Aggregator/tasks.py b/AIPscan/Aggregator/tasks.py index 2b1e2120..c1261211 100644 --- a/AIPscan/Aggregator/tasks.py +++ b/AIPscan/Aggregator/tasks.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import json import os +import shutil import requests from celery.utils.log import get_task_logger @@ -185,8 +186,10 @@ def workflow_coordinator( ) total_replicas = len([package for package in all_packages if package.is_replica()]) - summary = "aips: '{}'; sips: '{}'; dips: '{}'; deleted: '{}'; replicated: '{}'".format( - total_aips, total_sips, total_dips, total_deleted_aips, total_replicas + summary = ( + "aips: '{}'; sips: '{}'; dips: '{}'; deleted: '{}'; replicated: '{}'".format( + total_aips, total_sips, total_dips, total_deleted_aips, total_replicas + ) ) logger.info("%s", summary) @@ -272,6 +275,16 @@ def package_lists_request(self, apiUrl, timestamp, packages_directory): } +@celery.task() +def fetch_job_file_cleanup(fetch_job_id): + obj = FetchJob.query.filter_by(id=fetch_job_id).first() + + if os.path.isdir(obj.download_directory): + shutil.rmtree(obj.download_directory) + + logger.info("Cleaned up after fetch job {}".format(fetch_job_id)) + + @celery.task() def get_mets( package_uuid, diff --git a/AIPscan/Aggregator/tests/test_tasks.py b/AIPscan/Aggregator/tests/test_tasks.py index 1747250f..9f3c3f72 100644 --- a/AIPscan/Aggregator/tests/test_tasks.py +++ b/AIPscan/Aggregator/tests/test_tasks.py @@ -8,6 +8,7 @@ from AIPscan.Aggregator.tasks import ( TaskError, get_mets, + fetch_job_file_cleanup, make_request, parse_packages_and_load_mets, ) @@ -157,6 +158,22 @@ def mock_download_mets( delete_mets_file.call_count == 3 +def test_get_mets_task(app_instance, tmpdir, mocker): + test_downloads_dir = os.path.join(tmpdir, "downloads") + os.mkdir(test_downloads_dir) + + # Mock download directory should be created + assert os.path.isdir(test_downloads_dir) + + fetch_job1 = test_helpers.create_test_fetch_job( + download_directory=test_downloads_dir + ) + fetch_job_file_cleanup(fetch_job1.id) + + # Mock download directory should be removed + assert not os.path.isdir(test_downloads_dir) + + @pytest.mark.parametrize( "response, raises_task_error", [ diff --git a/AIPscan/Aggregator/views.py b/AIPscan/Aggregator/views.py index 4f45d3ba..2abf56d3 100644 --- a/AIPscan/Aggregator/views.py +++ b/AIPscan/Aggregator/views.py @@ -315,6 +315,9 @@ def get_mets_task_status(coordinatorid): downloadStart = _format_date(start) obj.download_end = downloadEnd db.session.commit() + + tasks.fetch_job_file_cleanup.delay(fetchJobId) + response = {"state": "COMPLETED"} flash("Fetch Job {} completed".format(downloadStart)) return jsonify(response)