work

AIPscan/Aggregator/database_helpers.py

@@ -25,6 +25,7 @@
     FileType,
     Pipeline,
     StorageLocation,
+    StorageService,
 )
 
 logger = get_task_logger(__name__)
@@ -203,19 +204,20 @@ def create_storage_location_object(current_location, description, storage_servic
     return storage_location
 
 
-def create_or_update_storage_location(current_location, api_url, storage_service_id):
+def create_or_update_storage_location(current_location, storage_service):
     """Create or update Storage Location and return it."""
     storage_location = StorageLocation.query.filter_by(
         current_location=current_location
     ).first()
+
     request_url, request_url_without_api_key = get_storage_service_api_url(
-        api_url, current_location
+        storage_service, current_location
     )
     response = tasks.make_request(request_url, request_url_without_api_key)
     description = response.get("description")
     if not storage_location:
         return create_storage_location_object(
-            current_location, description, storage_service_id
+            current_location, description, storage_service.id
         )
 
     if storage_location.description != description:
@@ -233,11 +235,13 @@ def create_pipeline_object(origin_pipeline, dashboard_url):
     return pipeline
 
 
-def create_or_update_pipeline(origin_pipeline, api_url):
+def create_or_update_pipeline(origin_pipeline, storage_service_id):
     """Create or update Storage Location and return it."""
     pipeline = Pipeline.query.filter_by(origin_pipeline=origin_pipeline).first()
+
+    storage_service = StorageService.query.get(storage_service_id)
     request_url, request_url_without_api_key = get_storage_service_api_url(
-        api_url, origin_pipeline
+        storage_service, origin_pipeline
     )
     response = tasks.make_request(request_url, request_url_without_api_key)
     dashboard_url = response.get("remote_name")

AIPscan/Aggregator/task_helpers.py

@@ -95,32 +95,27 @@ def _tz_neutral_date(date):
     return date
 
 
-def get_mets_url(api_url, package_uuid, path_to_mets):
+def get_mets_url(storage_service, package_uuid, path_to_mets):
     """Construct a URL from which we can download the METS files that
     we are interested in.
     """
-    am_url = "baseUrl"
-    user_name = "userName"
-    api_key = "apiKey"
-
     mets_url = "{}/api/v2/file/{}/extract_file/?relative_path_to_file={}&username={}&api_key={}".format(
-        api_url[am_url].rstrip("/"),
+        storage_service.url.rstrip("/"),
         package_uuid,
         path_to_mets,
-        api_url[user_name],
-        api_url[api_key],
+        storage_service.user_name,
+        storage_service.api_key,
     )
     return mets_url
 
 
-def get_storage_service_api_url(api_url, api_path):
+def get_storage_service_api_url(storage_service, api_path):
     """Return URL to fetch location infofrom Storage Service."""
-    base_url = api_url.get("baseUrl", "").rstrip("/")
+    base_url = storage_service.url.rstrip("/")
     request_url_without_api_key = "{}{}".format(base_url, api_path).rstrip("/")
-    user_name = api_url.get("userName")
-    api_key = api_url.get("apiKey", "")
+
     request_url = "{}?username={}&api_key={}".format(
-        request_url_without_api_key, user_name, api_key
+        request_url_without_api_key, storage_service.user_name, storage_service.api_key
     )
     return request_url, request_url_without_api_key
 

AIPscan/Aggregator/tasks.py

@@ -61,18 +61,20 @@ def start_mets_task(
     """Initiate a get_mets task worker and record the event in the
     celery database.
     """
+    storage_service = StorageService.query.get(storage_service_id)
     storage_location = database_helpers.create_or_update_storage_location(
-        current_location, api_url, storage_service_id
+        current_location, storage_service
     )
 
-    pipeline = database_helpers.create_or_update_pipeline(origin_pipeline, api_url)
+    pipeline = database_helpers.create_or_update_pipeline(
+        origin_pipeline, storage_service_id
+    )
 
     # Call worker to download and parse METS File.
     get_mets_task = get_mets.delay(
         package_uuid,
         aip_size,
         relative_path_to_mets,
-        api_url,
         timestamp_str,
         package_list_no,
         storage_service_id,
@@ -151,7 +153,6 @@ def delete_aip(uuid):
 def workflow_coordinator(
     self, api_url, timestamp, storage_service_id, fetch_job_id, packages_directory
 ):
-
     logger.info("Packages directory is: %s", packages_directory)
 
     # Send package list request to a worker.
@@ -284,7 +285,6 @@ def get_mets(
     package_uuid,
     aip_size,
     relative_path_to_mets,
-    api_url,
     timestamp_str,
     package_list_no,
     storage_service_id,
@@ -313,8 +313,13 @@ def get_mets(
         tasklogger = customlogger
 
     # Download METS file
+    storage_service = StorageService.query.get(storage_service_id)
     download_file = download_mets(
-        api_url, package_uuid, relative_path_to_mets, timestamp_str, package_list_no
+        storage_service,
+        package_uuid,
+        relative_path_to_mets,
+        timestamp_str,
+        package_list_no,
     )
     mets_name = os.path.basename(download_file)
     mets_hash = file_sha256_hash(download_file)

AIPscan/Aggregator/templates/storage_service.html

@@ -24,8 +24,8 @@
   <tr><td width=20%><strong>Download offset</strong></td><td>{{ storage_service.download_offset}}</td></tr>
   <tr><td width=20%><strong>Default service</strong></td><td>{{ storage_service.default}}</td></tr>
 </table>
-<a href="{{ url_for('aggregator.edit_storage_service', id=storage_service.id) }}"><button type="button" class="btn btn-info">Edit</button></a>
-<a href="{{ url_for('aggregator.delete_storage_service', id=storage_service.id) }}"><button type="button" class="btn btn-danger">Delete</button></a>
+<a href="{{ url_for('aggregator.edit_storage_service', storage_service_id=storage_service.id) }}"><button type="button" class="btn btn-info">Edit</button></a>
+<a href="{{ url_for('aggregator.delete_storage_service', storage_service_id=storage_service.id) }}"><button type="button" class="btn btn-danger">Delete</button></a>
 {% if mets_fetch_jobs %}
 <a href="{{ url_for('reporter.view_aips', storage_service_id=storage_service.id) }}"><button type="button" class="btn btn-success">View AIPs</button></a>
 {% endif %}
@@ -73,7 +73,7 @@
         </td>
         <td>{{ mets_fetch_job.aips|length }}</td>
         <td>
-          <a href="{{ url_for('aggregator.delete_fetch_job', id= mets_fetch_job.id) }}"><button type="button" class="btn btn-danger">Delete</button></a>
+          <a href="{{ url_for('aggregator.delete_fetch_job', fetch_job_id=mets_fetch_job.id) }}"><button type="button" class="btn btn-danger">Delete</button></a>
         </td>
       </tr>
     {% endfor %}

AIPscan/Aggregator/tests/test_database_helpers.py

@@ -16,6 +16,7 @@
     FileType,
     Pipeline,
     StorageLocation,
+    StorageService,
 )
 
 FIXTURES_DIR = "fixtures"
@@ -289,14 +290,25 @@ def test_create_or_update_storage_location(
     """Test that Storage Locations are created or updated as expected."""
     make_request = mocker.patch("AIPscan.Aggregator.tasks.make_request")
     make_request.return_value = {"description": new_description}
+
     create_storage_location_object = mocker.patch(
         "AIPscan.Aggregator.database_helpers.create_storage_location_object"
     )
 
+    # FIX
+    storage_service = StorageService(
+        name="a",
+        url="b",
+        user_name="c",
+        api_key="12345",
+        download_limit=0,
+        download_offset=0,
+        default="",
+    )
+    storage_service.id = 1
+
     storage_location = database_helpers.create_or_update_storage_location(
-        current_location=current_location,
-        api_url={},
-        storage_service_id=storage_service_id,
+        current_location=current_location, storage_service=storage_service
     )
 
     if location_created:
@@ -320,12 +332,18 @@ def test_create_or_update_pipeline(
     """Test that Storage Locations are created or updated as expected."""
     make_request = mocker.patch("AIPscan.Aggregator.tasks.make_request")
     make_request.return_value = {"remote_name": new_url}
+
     create_pipeline_object = mocker.patch(
         "AIPscan.Aggregator.database_helpers.create_pipeline_object"
     )
 
+    get_storage_service_api_url = mocker.patch(
+        "AIPscan.Aggregator.database_helpers.get_storage_service_api_url"
+    )
+    get_storage_service_api_url.return_value = (None, None)
+
     pipeline = database_helpers.create_or_update_pipeline(
-        origin_pipeline=origin_pipeline, api_url={}
+        origin_pipeline=origin_pipeline, storage_service_id=None
     )
 
     if pipeline_created:

AIPscan/Aggregator/tests/test_task_helpers.py

@@ -7,6 +7,7 @@
 
 import pytest
 
+from AIPscan import models
 from AIPscan.Aggregator import task_helpers
 from AIPscan.Aggregator.types import StorageServicePackage
 
@@ -76,46 +77,49 @@ def test_format_api_url(url_api_dict, base_url, url_without_api_key, url_with_ap
 
 
 @pytest.mark.parametrize(
-    "api_url, package_uuid, path_to_mets, result",
+    "ss_args, package_uuid, path_to_mets, result",
     [
         (
-            {"baseUrl": "http://example.com", "userName": "1234", "apiKey": "1234"},
+            ["Test", "http://example.com", "1234", "1234", 0, 0, None],
             "1234",
             "1234",
             "http://example.com/api/v2/file/1234/extract_file/?relative_path_to_file=1234&username=1234&api_key=1234",
         ),
         (
-            {"baseUrl": "http://example.com/", "userName": "1234", "apiKey": "1234"},
+            ["Test", "http://example.com", "1234", "1234", 0, 0, None],
             "1234",
             "1234",
             "http://example.com/api/v2/file/1234/extract_file/?relative_path_to_file=1234&username=1234&api_key=1234",
         ),
     ],
 )
-def test_get_mets_url(api_url, package_uuid, path_to_mets, result):
+def test_get_mets_url(ss_args, package_uuid, path_to_mets, result):
     """Ensure that the URL for retrieving METS is constructed properly.
     """
-    mets_url = task_helpers.get_mets_url(api_url, package_uuid, path_to_mets)
+    ss = models.StorageService(*ss_args)
+    mets_url = task_helpers.get_mets_url(ss, package_uuid, path_to_mets)
     assert mets_url == result
 
 
 @pytest.mark.parametrize(
-    "api_url, current_location, expected_url, expected_url_without_api_key",
+    "ss_args, current_location, expected_url, expected_url_without_api_key",
     [
         (
-            {"baseUrl": "http://example.com", "userName": "1234", "apiKey": "12345"},
+            ["Test", "http://example.com", "1234", "12345", 0, 0, None],
             "/api/v2/location/{}".format(LOCATION_UUID),
             "http://example.com/api/v2/location/1b60c346-85a0-4a3c-a88b-0c1b3255e2ec?username=1234&api_key=12345",
             "http://example.com/api/v2/location/1b60c346-85a0-4a3c-a88b-0c1b3255e2ec",
         )
     ],
 )
 def test_get_storage_service_api_url(
-    api_url, current_location, expected_url, expected_url_without_api_key
+    ss_args, current_location, expected_url, expected_url_without_api_key
 ):
     """Ensure construction of URL to fetch Resource information."""
+    storage_service = models.StorageService(*ss_args)
+
     url, url_without_secrets = task_helpers.get_storage_service_api_url(
-        api_url, current_location
+        storage_service, current_location
     )
     assert url == expected_url
     assert url_without_secrets == expected_url_without_api_key

AIPscan/Aggregator/tests/test_tasks.py

@@ -78,8 +78,6 @@ def mock_download_mets(
     aips = _get_aips(storage_service.id)
     assert not aips
 
-    api_url = {"baseUrl": "http://test-url", "userName": "test", "apiKey": "test"}
-
     # Set up custom logger and add handler to capture output
     customlogger = logging.getLogger(__name__)
     customlogger.setLevel(logging.DEBUG)
@@ -96,7 +94,6 @@ def mock_download_mets(
         package_uuid=package_uuid,
         aip_size=1000,
         relative_path_to_mets="test",
-        api_url=api_url,
         timestamp_str=datetime.now()
         .replace(microsecond=0)
         .strftime("%Y-%m-%d-%H-%M-%S"),
@@ -121,7 +118,6 @@ def mock_download_mets(
         package_uuid=package_uuid,
         aip_size=1000,
         relative_path_to_mets="test",
-        api_url=api_url,
         timestamp_str=datetime.now()
         .replace(microsecond=0)
         .strftime("%Y-%m-%d-%H-%M-%S"),
@@ -147,7 +143,6 @@ def mock_download_mets(
         package_uuid=package_uuid,
         aip_size=1000,
         relative_path_to_mets="test",
-        api_url=api_url,
         timestamp_str=datetime.now()
         .replace(microsecond=0)
         .strftime("%Y-%m-%d-%H-%M-%S"),

AIPscan/Reporter/views.py

@@ -329,15 +329,9 @@ def download_mets(aip_id):
     aip = AIP.query.get(aip_id)
     storage_service = StorageService.query.get(aip.storage_service_id)
 
-    api_url = {
-        "baseUrl": storage_service.url,
-        "userName": storage_service.user_name,
-        "apiKey": storage_service.api_key,
-    }
-
     mets_response = requests.get(
         get_mets_url(
-            api_url,
+            storage_service,
             aip.uuid,
             f"{aip.transfer_name}-{aip.uuid}/data/METS.{aip.uuid}.xml",
         )

tools/helpers/fetch.py

@@ -8,6 +8,7 @@
     process_package_object,
 )
 from AIPscan.Aggregator.tasks import delete_aip, get_mets, make_request
+from AIPscan.models import StorageService
 
 
 def assemble_api_url_dict(storage_service, offset=0, limit=1_000_000):
@@ -95,12 +96,13 @@ def import_packages(
             if not package.is_aip():
                 continue
 
+            storage_service = StorageService.query.get(storage_service_id)
             storage_location = database_helpers.create_or_update_storage_location(
-                package.current_location, api_url, storage_service_id
+                package.current_location, storage_service
             )
 
             pipeline = database_helpers.create_or_update_pipeline(
-                package.origin_pipeline, api_url
+                package.origin_pipeline, storage_service_id
             )
 
             args = [