transfers: Add SS API key auth

Storage service now requires auth for the API. Add flags for storage service
user and API key and use them when calling the API.

README.md

@@ -47,17 +47,19 @@ When running, automated transfers stores its working state in a sqlite database.
 
 The `transfers.py` script can be modified to adjust how automated transfers work.  The full set of parameters that can be changed are:
 
-* `-u USERNAME, --user USERNAME` [REQUIRED]: Username of the dashboard user to authenticate as.
-* `-k KEY, --api-key KEY` [REQUIRED]: API key of the dashboard user.
+* `-u USERNAME, --user USERNAME` [REQUIRED]: Username of the Archivematica dashboard user to authenticate as.
+* `-k KEY, --api-key KEY` [REQUIRED]: API key of the Archivematica dashboard user.
+* `--ss-user USERNAME, --user USERNAME` [REQUIRED]: Username of the Storage Service user to authenticate as. Storage Service 0.8 and up requires this; earlier versions will ignore any value provided.
+* `--ss-api-key KEY` [REQUIRED]: API key of the Storage Service user. Storage Service 0.8 and up requires this; earlier versions will ignore any value provided.
 * `-t UUID, --transfer-source UUID`: [REQUIRED] Transfer Source Location UUID to fetch transfers from. Check the next section for more details on this field.
-* `-c FILE, --config-file FILE`: config file containing file paths for log/database/PID files. Default: log/database/PID files stored in the same directory as the script (not recommended for production)
 * `--transfer-path PATH`: Relative path within the Transfer Source. Default: ""
 * `--depth DEPTH, -d DEPTH`: Depth to create the transfers from relative to the transfer source location and path. Default of 1 creates transfers from the children of transfer-path.
 * `--am-url URL, -a URL`:Archivematica URL. Default: http://127.0.0.1
 * `--ss-url URL, -s URL`: Storage Service URL. Default: http://127.0.0.1:8000
 * `--transfer-type TYPE`: Type of transfer to start. One of: 'standard' (default), 'unzipped bag', 'zipped bag', 'dspace'.
 * `--files`: If set, start transfers from files as well as folders.
 * `--hide`: If set, hides the Transfer and SIP once completed.
+* `-c FILE, --config-file FILE`: config file containing file paths for log/database/PID files. Default: log/database/PID files stored in the same directory as the script (not recommended for production)
 
 ### Configuration - Getting Correct UUIDs and Setting Processing Rules
 

fixtures/vcr_cassettes/get_next_transfer_all_complete.yaml

@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.5.3 CPython/3.4.0 Linux/3.13.0-43-generic]
     method: GET
-    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz
+    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz&api_key=7016762e174c940df304e8343c659af5005b4d6b&username=test
   response:
     body: {string: '{"directories": ["QmFnVHJhbnNmZXI=", "Q1NWbWV0YWRhdGE=", "RGlnaXRpemF0aW9uT3V0cHV0",
         "RFNwYWNlRXhwb3J0", "SW1hZ2Vz", "SVNPRGlza0ltYWdl", "TXVsdGltZWRpYQ==", "T0NSSW1hZ2U=",

fixtures/vcr_cassettes/get_next_transfer_bad_source.yaml

@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.5.3 CPython/3.4.0 Linux/3.13.0-43-generic]
     method: GET
-    uri: http://127.0.0.1:8000/api/v2/location/badd8d39-9cee-495e-b7ee-5e6292549bad/browse/?path=U2FtcGxlVHJhbnNmZXJz
+    uri: http://127.0.0.1:8000/api/v2/location/badd8d39-9cee-495e-b7ee-5e6292549bad/browse/?path=U2FtcGxlVHJhbnNmZXJz&api_key=7016762e174c940df304e8343c659af5005b4d6b&username=test
   response:
     body: {string: Resource with UUID badd8d39-9cee-495e-b7ee-5e6292549bad does not
         exist}

fixtures/vcr_cassettes/get_next_transfer_depth.yaml

@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.5.3 CPython/3.4.0 Linux/3.13.0-43-generic]
     method: GET
-    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz
+    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz&api_key=7016762e174c940df304e8343c659af5005b4d6b&username=test
   response:
     body: {string: '{"directories": ["QmFnVHJhbnNmZXI=", "Q1NWbWV0YWRhdGE=", "RGlnaXRpemF0aW9uT3V0cHV0",
         "RFNwYWNlRXhwb3J0", "SW1hZ2Vz", "SVNPRGlza0ltYWdl", "TXVsdGltZWRpYQ==", "T0NSSW1hZ2U=",
@@ -32,7 +32,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.5.3 CPython/3.4.0 Linux/3.13.0-43-generic]
     method: GET
-    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJzL0JhZ1RyYW5zZmVy
+    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJzL0JhZ1RyYW5zZmVy&api_key=7016762e174c940df304e8343c659af5005b4d6b&username=test
   response:
     body: {string: '{"directories": ["ZGF0YQ=="], "entries": ["YmFnLWluZm8udHh0",
         "YmFnaXQudHh0", "ZGF0YQ==", "bWFuaWZlc3Qtc2hhNTEyLnR4dA=="]}'}

fixtures/vcr_cassettes/get_next_transfer_existing_set.yaml

@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.5.3 CPython/3.4.0 Linux/3.13.0-43-generic]
     method: GET
-    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz
+    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz&api_key=7016762e174c940df304e8343c659af5005b4d6b&username=test
   response:
     body: {string: '{"directories": ["QmFnVHJhbnNmZXI=", "Q1NWbWV0YWRhdGE=", "RGlnaXRpemF0aW9uT3V0cHV0",
         "RFNwYWNlRXhwb3J0", "SW1hZ2Vz", "SVNPRGlza0ltYWdl", "TXVsdGltZWRpYQ==", "T0NSSW1hZ2U=",

fixtures/vcr_cassettes/get_next_transfer_failed_auth.yaml

@@ -0,0 +1,21 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [python-requests/2.8.1]
+    method: GET
+    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?username=demo&path=U2FtcGxlVHJhbnNmZXJz&api_key=dne
+  response:
+    body: {string: ''}
+    headers:
+      Content-Type: [text/html; charset=utf-8]
+      Date: ['Fri, 27 May 2016 00:24:26 GMT']
+      Server: [WSGIServer/0.1 Python/2.7.6]
+      Vary: [Cookie]
+      WWW-Authenticate: [Basic Realm="django-tastypie"]
+      X-Frame-Options: [SAMEORIGIN]
+    status: {code: 401, message: UNAUTHORIZED}
+version: 1

fixtures/vcr_cassettes/get_next_transfer_files.yaml

@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.5.3 CPython/3.4.0 Linux/3.13.0-43-generic]
     method: GET
-    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz
+    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz&api_key=7016762e174c940df304e8343c659af5005b4d6b&username=test
   response:
     body: {string: '{"directories": ["QmFnVHJhbnNmZXI=", "Q1NWbWV0YWRhdGE=", "RGlnaXRpemF0aW9uT3V0cHV0",
         "RFNwYWNlRXhwb3J0", "SW1hZ2Vz", "SVNPRGlza0ltYWdl", "TXVsdGltZWRpYQ==", "T0NSSW1hZ2U=",

fixtures/vcr_cassettes/get_next_transfer_first_run.yaml

@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.5.3 CPython/3.4.0 Linux/3.13.0-43-generic]
     method: GET
-    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz
+    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?path=U2FtcGxlVHJhbnNmZXJz&api_key=7016762e174c940df304e8343c659af5005b4d6b&username=test
   response:
     body: {string: '{"directories": ["QmFnVHJhbnNmZXI=", "Q1NWbWV0YWRhdGE=", "RGlnaXRpemF0aW9uT3V0cHV0",
         "RFNwYWNlRXhwb3J0", "SW1hZ2Vz", "SVNPRGlza0ltYWdl", "TXVsdGltZWRpYQ==", "T0NSSW1hZ2U=",

fixtures/vcr_cassettes/get_next_transfer_no_prefix.yaml

@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.5.3 CPython/3.4.0 Linux/3.13.0-43-generic]
     method: GET
-    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/
+    uri: http://127.0.0.1:8000/api/v2/location/2a3d8d39-9cee-495e-b7ee-5e629254934d/browse/?api_key=7016762e174c940df304e8343c659af5005b4d6b&username=test
   response:
     body: {string: '{"directories": ["T1BGIGZvcm1hdC1jb3JwdXM=", "U2FtcGxlVHJhbnNmZXJz",
         "VGVzdFRyYW5zZmVycw=="], "entries": ["T1BGIGZvcm1hdC1jb3JwdXM=", "UkVBRE1FLm1k",

tests/test_transfers.py

@@ -13,6 +13,8 @@
 SS_URL = 'http://127.0.0.1:8000'
 USER = 'demo'
 API_KEY = '1c34274c0df0bca7edf9831dd838b4a6345ac2ef'
+SS_USER = 'test'
+SS_KEY = '7016762e174c940df304e8343c659af5005b4d6b'
 
 TS_LOCATION_UUID = '2a3d8d39-9cee-495e-b7ee-5e629254934d'
 PATH_PREFIX = b'SampleTransfers'
@@ -95,7 +97,7 @@ def test_get_accession_id_no_script(self):
     def test_get_next_transfer_first_run(self):
         # All default values
         # Test
-        path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, COMPLETED, FILES)
+        path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, COMPLETED, FILES)
         # Verify
         assert path == b'SampleTransfers/BagTransfer'
 
@@ -104,7 +106,7 @@ def test_get_next_transfer_existing_set(self):
         # Set completed set
         completed = {b'SampleTransfers/BagTransfer'}
         # Test
-        path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, completed, FILES)
+        path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, completed, FILES)
         # Verify
         assert path == b'SampleTransfers/CSVmetadata'
 
@@ -113,7 +115,7 @@ def test_get_next_transfer_depth(self):
         # Set depth
         depth = 2
         # Test
-        path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, depth, COMPLETED, FILES)
+        path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY, TS_LOCATION_UUID, PATH_PREFIX, depth, COMPLETED, FILES)
         # Verify
         assert path == b'SampleTransfers/BagTransfer/data'
 
@@ -122,7 +124,7 @@ def test_get_next_transfer_no_prefix(self):
         # Set no prefix
         path_prefix = b''
         # Test
-        path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, path_prefix, DEPTH, COMPLETED, FILES)
+        path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY, TS_LOCATION_UUID, path_prefix, DEPTH, COMPLETED, FILES)
         # Verify
         assert path == b'OPF format-corpus'
 
@@ -131,7 +133,7 @@ def test_get_next_transfer_all_complete(self):
         # Set completed set to be all elements
         completed = {b'SampleTransfers/BagTransfer', b'SampleTransfers/CSVmetadata', b'SampleTransfers/DigitizationOutput', b'SampleTransfers/DSpaceExport', b'SampleTransfers/Images', b'SampleTransfers/ISODiskImage', b'SampleTransfers/Multimedia', b'SampleTransfers/OCRImage', b'SampleTransfers/OfficeDocs', b'SampleTransfers/RawCameraImages', b'SampleTransfers/structMapSample'}
         # Test
-        path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, completed, FILES)
+        path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, completed, FILES)
         # Verify
         assert path is None
 
@@ -140,7 +142,7 @@ def test_get_next_transfer_bad_source(self):
         # Set bad TS Location UUID
         ts_location_uuid = 'badd8d39-9cee-495e-b7ee-5e6292549bad'
         # Test
-        path = transfer.get_next_transfer(SS_URL, ts_location_uuid, PATH_PREFIX, DEPTH, COMPLETED, FILES)
+        path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY, ts_location_uuid, PATH_PREFIX, DEPTH, COMPLETED, FILES)
         # Verify
         assert path is None
 
@@ -150,6 +152,16 @@ def test_get_next_transfer_files(self):
         files = True
         completed = {b'SampleTransfers/BagTransfer'}
         # Test
-        path = transfer.get_next_transfer(SS_URL, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, completed, files)
+        path = transfer.get_next_transfer(SS_URL, SS_USER, SS_KEY, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, completed, files)
         # Verify
         assert path == b'SampleTransfers/BagTransfer.zip'
+
+    @vcr.use_cassette('fixtures/vcr_cassettes/get_next_transfer_failed_auth.yaml')
+    def test_get_next_transfer_failed_auth(self):
+        # All default values
+        ss_user = 'demo'
+        ss_key = 'dne'
+        # Test
+        path = transfer.get_next_transfer(SS_URL, ss_user, ss_key, TS_LOCATION_UUID, PATH_PREFIX, DEPTH, COMPLETED, FILES)
+        # Verify
+        assert path is None