diff --git a/CHANGES.txt b/CHANGES.txt index 7b84457..8edb9c6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,8 @@ +2.8.0 (09/04/2024) +================== + +- Add replace file in archive and fetch file from archive (#41) + 2.7.2 (16/11/2023) ================== diff --git a/setup.py b/setup.py index 9bd901c..b4dcc16 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ ] setup(name='storageprovider-client', - version='2.7.2', + version='2.8.0', description='storageprovider client', long_description=README + '\n\n' + CHANGES, classifiers=[ diff --git a/storageprovider/client.py b/storageprovider/client.py index 533aee8..2cf4957 100644 --- a/storageprovider/client.py +++ b/storageprovider/client.py @@ -1,4 +1,11 @@ +import logging +from typing import Callable + import requests +from requests import RequestException +from requests import Response + +LOG = logging.getLogger(__name__) class StorageProviderClient: @@ -11,6 +18,42 @@ def __init__(self, base_url, collection): def get_auth_header(system_token): return {"Authorization": f"Bearer {system_token}"} + def _execute_requests_method( + self, + requests_method: Callable, + system_token: str, + url: str, + response_code: int = 200, + headers: dict = None, + **requests_kwargs, + ) -> Response: + """ + Send a request with the given params. + + This is a simple utility method to handle authorization headers, + basic accept, content-type headers and catch request exceptions. + + :param requests_method: a requests method to call. eg. requests.get, requests.post + :param system_token: oauth system token + :param url: url to post to. + :param response_code: expected response code + :param headers: extra headers to add to the request + :param requests_kwargs: extra kwargs which will be added to the requests call. + :return: The response + """ + headers = headers or {} + if system_token: + headers.update(self.get_auth_header(system_token)) + try: + response = requests_method(url, headers=headers, **requests_kwargs) + except RequestException: + LOG.exception(f"{requests_method} {url} failed.") + raise + if response.status_code != response_code: + raise InvalidStateException(response.status_code, response.text) + + return response + def delete_object(self, container_key, object_key, system_token=None): """ delete an object from the data store @@ -20,27 +63,12 @@ def delete_object(self, container_key, object_key, system_token=None): :param system_token: oauth system token :raises InvalidStateException: if the response is in an invalid state """ - headers = {} - if system_token: - headers = self.get_auth_header(system_token) - res = requests.delete( - self.base_url + "/containers/" + container_key + "/" + object_key, - headers=headers, - ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) - - def _get_object_res(self, container_key, object_key, system_token): - headers = {} - if system_token: - headers = self.get_auth_header(system_token) - res = requests.get( - self.base_url + "/containers/" + container_key + "/" + object_key, - headers=headers, + response = self._execute_requests_method( + requests.delete, + system_token, + f"{self.base_url}/containers/{container_key}/{object_key}", ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) - return res + return response def get_object_streaming(self, container_key, object_key, system_token=None): """ @@ -51,17 +79,14 @@ def get_object_streaming(self, container_key, object_key, system_token=None): :return content of the object as a stream :raises InvalidStateException: if the response is in an invalid state """ - headers = {} - if system_token: - headers = self.get_auth_header(system_token) - res = requests.get( - self.base_url + "/containers/" + container_key + "/" + object_key, - headers=headers, + response = self._execute_requests_method( + requests.get, + system_token, + f"{self.base_url}/containers/{container_key}/{object_key}", stream=True, ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) - return res.iter_content(1024 * 1024) + + return response.iter_content(1024 * 1024) def get_object(self, container_key, object_key, system_token=None): """ @@ -73,8 +98,12 @@ def get_object(self, container_key, object_key, system_token=None): :return content of the object :raises InvalidStateException: if the response is in an invalid state """ - res = self._get_object_res(container_key, object_key, system_token) - return res.content + response = self._execute_requests_method( + requests.get, + system_token, + f"{self.base_url}/containers/{container_key}/{object_key}", + ) + return response.content def get_object_and_metadata(self, container_key, object_key, system_token=None): """ @@ -86,11 +115,15 @@ def get_object_and_metadata(self, container_key, object_key, system_token=None): :return content of the object :raises InvalidStateException: if the response is in an invalid state """ - res = self._get_object_res(container_key, object_key, system_token) - metadata = res.headers + response = self._execute_requests_method( + requests.get, + system_token, + f"{self.base_url}/containers/{container_key}/{object_key}", + ) + metadata = response.headers metadata["mime"] = metadata["Content-Type"] metadata["size"] = metadata["Content-Length"] - return {"object": res.content, "metadata": metadata} + return {"object": response.content, "metadata": metadata} def get_object_metadata(self, container_key, object_key, system_token=None): """ @@ -102,16 +135,12 @@ def get_object_metadata(self, container_key, object_key, system_token=None): :return headers of the object :raises InvalidStateException: if the response is in an invalid state """ - headers = {} - if system_token: - headers = self.get_auth_header(system_token) - res = requests.get( + response = self._execute_requests_method( + requests.get, + system_token, f"{self.base_url}/containers/{container_key}/{object_key}/meta", - headers=headers, ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) - result = res.json() + result = response.json() result["Content-Type"] = result["mime"] # backwards compatibility result["Content-Length"] = result["size"] # backwards compatibility return result @@ -133,22 +162,21 @@ def copy_object_and_create_key( :raises InvalidStateException: if the response is in an invalid state """ headers = {"content-type": "application/json"} - if system_token: - headers.update(self.get_auth_header(system_token)) object_data = { "host_url": self.host_url, "collection_key": self.collection, "container_key": source_container_key, "object_key": source_object_key, } - res = requests.post( - self.base_url + "/containers/" + output_container_key, - json=object_data, + response = self._execute_requests_method( + requests.post, + system_token, + f"{self.base_url}/containers/{output_container_key}", + response_code=201, headers=headers, + json=object_data, ) - if res.status_code != 201: - raise InvalidStateException(res.status_code, res.text) - object_key = res.json()["object_key"] + object_key = response.json()["object_key"] if isinstance(object_key, str): object_key = str(object_key) return object_key @@ -172,25 +200,19 @@ def copy_object( :raises InvalidStateException: if the response is in an invalid state """ headers = {"content-type": "application/json"} - if system_token: - headers.update(self.get_auth_header(system_token)) object_data = { "host_url": self.host_url, "collection_key": self.collection, "container_key": source_container_key, "object_key": source_object_key, } - res = requests.put( - self.base_url - + "/containers/" - + output_container_key - + "/" - + output_object_key, - json=object_data, + self._execute_requests_method( + requests.put, + system_token, + f"{self.base_url}/containers/{output_container_key}/{output_object_key}", headers=headers, + json=object_data, ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) def update_object_and_key(self, container_key, object_data, system_token=None): """ @@ -202,16 +224,15 @@ def update_object_and_key(self, container_key, object_data, system_token=None): :raises InvalidStateException: if the response is in an invalid state """ headers = {"content-type": "application/octet-stream"} - if system_token: - headers.update(self.get_auth_header(system_token)) - res = requests.post( - self.base_url + "/containers/" + container_key, - data=object_data, + response = self._execute_requests_method( + requests.post, + system_token, + f"{self.base_url}/containers/{container_key}", + response_code=201, headers=headers, + data=object_data, ) - if res.status_code != 201: - raise InvalidStateException(res.status_code, res.text) - object_key = res.json()["object_key"] + object_key = response.json()["object_key"] if isinstance(object_key, str): object_key = str(object_key) return object_key @@ -227,15 +248,13 @@ def update_object(self, container_key, object_key, object_data, system_token=Non :raises InvalidStateException: if the response is in an invalid state """ headers = {"content-type": "application/octet-stream"} - if system_token: - headers.update(self.get_auth_header(system_token)) - res = requests.put( - self.base_url + "/containers/" + container_key + "/" + object_key, - data=object_data, + return self._execute_requests_method( + requests.put, + system_token, + f"{self.base_url}/containers/{container_key}/{object_key}", headers=headers, + data=object_data, ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) def list_object_keys_for_container(self, container_key, system_token=None): """ @@ -247,14 +266,13 @@ def list_object_keys_for_container(self, container_key, system_token=None): :raises InvalidStateException: if the response is in an invalid state """ headers = {"Accept": "application/json"} - if system_token: - headers.update(self.get_auth_header(system_token)) - res = requests.get( - self.base_url + "/containers/" + container_key, headers=headers + response = self._execute_requests_method( + requests.get, + system_token, + f"{self.base_url}/containers/{container_key}", + headers=headers, ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) - return res.content + return response.content def get_container_data_streaming( self, container_key, system_token=None, translations=None @@ -269,17 +287,15 @@ def get_container_data_streaming( """ translations = translations or {} headers = {"Accept": "application/zip"} - if system_token: - headers.update(self.get_auth_header(system_token)) - res = requests.get( - self.base_url + "/containers/" + container_key, - params=translations, + response = self._execute_requests_method( + requests.get, + system_token, + f"{self.base_url}/containers/{container_key}", headers=headers, stream=True, + params=translations, ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) - return res.iter_content(1024 * 1024) + return response.iter_content(1024 * 1024) def get_container_data(self, container_key, system_token=None, translations=None): """ @@ -293,16 +309,14 @@ def get_container_data(self, container_key, system_token=None, translations=None """ translations = translations or {} headers = {"Accept": "application/zip"} - if system_token: - headers.update(self.get_auth_header(system_token)) - res = requests.get( - self.base_url + "/containers/" + container_key, - params=translations, + response = self._execute_requests_method( + requests.get, + system_token, + f"{self.base_url}/containers/{container_key}", headers=headers, + params=translations, ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) - return res.content + return response.content def create_container(self, container_key, system_token=None): """ @@ -312,14 +326,11 @@ def create_container(self, container_key, system_token=None): :param system_token: oauth system token :raises InvalidStateException: if the response is in an invalid state """ - headers = {} - if system_token: - headers = self.get_auth_header(system_token) - res = requests.put( - self.base_url + "/containers/" + container_key, headers=headers + return self._execute_requests_method( + requests.put, + system_token, + f"{self.base_url}/containers/{container_key}", ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) def create_container_and_key(self, system_token=None): """ @@ -329,13 +340,15 @@ def create_container_and_key(self, system_token=None): :return the key generated for the container :raises InvalidStateException: if the response is in an invalid state """ - headers = {} - if system_token: - headers = self.get_auth_header(system_token) - res = requests.post(self.base_url + "/containers", headers=headers) - if res.status_code != 201: - raise InvalidStateException(res.status_code, res.text) - container_key = res.json()["container_key"] + + response = self._execute_requests_method( + requests.post, + system_token, + f"{self.base_url}/containers", + response_code=201, + ) + + container_key = response.json()["container_key"] if isinstance(container_key, str): container_key = str(container_key) return container_key @@ -348,14 +361,87 @@ def delete_container(self, container_key, system_token=None): :param system_token: oauth system token :raises InvalidStateException: if the response is in an invalid state """ - headers = {} - if system_token: - headers = self.get_auth_header(system_token) - res = requests.delete( - self.base_url + "/containers/" + container_key, headers=headers + return self._execute_requests_method( + requests.delete, + system_token, + f"{self.base_url}/containers/{container_key}", + ) + + def get_object_from_archive( + self, + container_key, + object_key, + file_name, + system_token=None + ): + """ + retrieve an object from an archive in the data store + :param container_key: key of the container in the data store + :param object_key: specific object key for the object in the container + :param file_name: name of the file to get from the zip + :param system_token: oauth system token + :return content of the object as a stream + :raises InvalidStateException: if the response is in an invalid state + """ + response = self._execute_requests_method( + requests.get, + system_token, + f"{self.base_url}/containers/{container_key}/{object_key}/{file_name}", + ) + + return response.content + + def get_object_from_archive_streaming( + self, + container_key, + object_key, + file_name, + system_token=None + ): + """ + retrieve an object from an archive in the data storeas a stream + :param container_key: key of the container in the data store + :param object_key: specific object key for the object in the container + :param system_token: oauth system token + :return content of the object as a stream + :raises InvalidStateException: if the response is in an invalid state + """ + response = self._execute_requests_method( + requests.get, + system_token, + f"{self.base_url}/containers/{container_key}/{object_key}/{file_name}", + stream=True, + ) + + return response.iter_content(1024 * 1024) + + def replace_file_in_zip_object( + self, + container_key, + object_key, + file_to_replace, + new_file_content, + new_file_name, + system_token=None, + ): + """ + replace a file in a zip in the data store + :param container_key: key of the container in the data store + :param object_key: specific object key for the object in the container + :param file_to_replace: name of the file to replace in the zip + :param new_file_content: content of the new file + :param new_file_name: name of the new file + :param system_token: oauth system token + :return content of the updated zip file + """ + response = self._execute_requests_method( + requests.put, + system_token, + f"{self.base_url}/containers/{container_key}/{object_key}/{file_to_replace}", + data=new_file_content, + params={"new_file_name": new_file_name}, ) - if res.status_code != 200: - raise InvalidStateException(res.status_code, res.text) + return response.json() class InvalidStateException(Exception): diff --git a/tests/test_client.py b/tests/test_client.py index 562a63d..32b68f3 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,3 +1,4 @@ +import io import os import unittest from unittest.mock import Mock @@ -9,6 +10,7 @@ test_collection_key = "test_collection" test_container_key = "test_container_key" test_object_key = "test_object_key" +test_file_name = "test.pdf" test_base_url = "http://localhost:6543" test_check_url = test_base_url + "/collections/" + test_collection_key @@ -606,3 +608,47 @@ def test_delete_container_KO(self, mock_requests): self.assertTrue(error_thrown) self.assertEqual(400, error.status_code) self.assertEqual("test error, http status code: 400", str(error)) + + @patch("storageprovider.client.requests") + def test_get_object_from_archive_streaming(self, mock_requests): + mock_requests.get.return_value.status_code = 200 + self.storageproviderclient.get_object_from_archive_streaming( + test_container_key, test_object_key, test_file_name + ) + mock_requests.get.assert_called_with( + f"{test_check_url}/containers/{test_container_key}" + f"/{test_object_key}/{test_file_name}", + headers={}, + stream=True, + ) + + @patch("storageprovider.client.requests") + def test_get_object_from_archive(self, mock_requests): + mock_requests.get.return_value.status_code = 200 + self.storageproviderclient.get_object_from_archive( + test_container_key, test_object_key, test_file_name + ) + mock_requests.get.assert_called_with( + f"{test_check_url}/containers/{test_container_key}" + f"/{test_object_key}/{test_file_name}", + headers={}, + ) + + @patch("storageprovider.client.requests") + def test_replace_file_in_zip_object(self, mock_requests): + mock_requests.put.return_value.status_code = 200 + new_file_name = "new_file.pdf" + new_file_content = io.BytesIO(b"test") + self.storageproviderclient.replace_file_in_zip_object( + test_container_key, + test_object_key, + test_file_name, + new_file_content, + new_file_name, + ) + mock_requests.put.assert_called_with( + f"{test_check_url}/containers/{test_container_key}/{test_object_key}/{test_file_name}", + headers={}, + data=new_file_content, + params={"new_file_name": "new_file.pdf"} + )