diff --git a/CHANGES.md b/CHANGES.md index 5529690..9c9a423 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,23 @@ # Jira one change log +**Release 0.8.4** - 2024-05-20 + +Thanks to [@huyz](https://github.com/huyz) for the below fixes and improvements to v0.8.4 + +Fixes: +- 🐛 `get_attachments_on_projects`: Overwrite attachment file by default +- 🐛 `json_field_builder`: check if `sprint_custom_id` is None +- 🐛 `path_builder`: handle multi-dir base_dir +- 🐛 `download_attachments`: avoid conflicts/overwrites by isolating attachments (helps with https://github.com/princenyeche/jiraone/issues/112) + +Improvements: +- ✨ `download_attachments`: make defaults and behaviour match `get_attachments_on_projects` + +Features: +- ✨ `download_attachments`: support `create_html_directors` option (resolves https://github.com/princenyeche/jiraone/issues/112) +- ✨ `download_attachments`: support `overwrite=False` option (to speed up incremental backups) + + **Release 0.8.3** - 2024-04-01 * Fixing the CSV change file type and merge_files to run without JQL * Added a new endpoint to `runbackup` diff --git a/SECURITY.md b/SECURITY.md index 574b65f..d8a5941 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,17 +2,18 @@ ## Supported Versions -Below shows the list of supported versions for the jiraone library +Below is the list of supported versions for the jiraone library | Version | Supported | |---------|--------------------| +| 0.8.4 | :white_check_mark: | | 0.8.3 | :white_check_mark: | | 0.8.2 | :white_check_mark: | | 0.8.1 | :white_check_mark: | | 0.7.9 | :white_check_mark: | | 0.7.8 | :white_check_mark: | -| 0.7.7 | :white_check_mark: | -| 0.7.6 | :white_check_mark: | +| 0.7.7 | :x: | +| 0.7.6 | :x: | | 0.7.5 | :x: | | 0.7.4 | :x: | | 0.7.3 | :x: | @@ -31,4 +32,4 @@ Below shows the list of supported versions for the jiraone library ## Reporting a Vulnerability -Please for any security related issue, email to support@elfapp.website +For any security-related issue, please email support@elfapp.website diff --git a/pyproject.toml b/pyproject.toml index f7dcc67..58cb63f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "jiraone" -version = "v0.8.3" +version = "v0.8.4" authors = [ { name="Prince Nyeche", email="support@elfapp.website" }, ] diff --git a/src/jiraone/__init__.py b/src/jiraone/__init__.py index d305eac..a8d68f4 100644 --- a/src/jiraone/__init__.py +++ b/src/jiraone/__init__.py @@ -37,7 +37,7 @@ from jiraone.management import manage __author__ = "Prince Nyeche" -__version__ = "0.8.3" +__version__ = "0.8.4" __all__ = [ "LOGIN", "endpoint", diff --git a/src/jiraone/reporting.py b/src/jiraone/reporting.py index 5428db5..d02b3b3 100644 --- a/src/jiraone/reporting.py +++ b/src/jiraone/reporting.py @@ -518,21 +518,30 @@ def get_attachments_on_projects( self, attachment_folder: str = "Attachment", attachment_file_name: str = "attachment_file.csv", + mode: str = 'w', **kwargs, ) -> None: - """Return all attachments of a Project or Projects + """Fetch the list of all the attachments of a Project or Projects + and write it out to an attachment list CSV file named ``attachment_file_name`` + located in ``attachment_folder``. - Get the size of attachments on an Issue, count those attachments - collectively and return the total number on all Projects searched. - JQL is used as a means to search for the project. + Also, get the size of the attachment for each Issue, sum up the size of all + attachments, and output the total for all Projects as the last + row of the output attachment list CSV file. - :param attachment_folder: A temp folder + JQL is used to search for the attachments. - :param attachment_file_name: A filename for the attachment + :param attachment_folder: Target directory where the attachment list CSV + file will be written. - :param kwargs: Addition argument to supply. + :param attachment_file_name: Filename of the attachment list CSV to be written. - :return: None + :param mode: Write mode for attachment list CVS to be written. By default it + is 'w', which means that any existing file will be overwritten. + For example, set to 'a' if you want to append to instead of truncating any + existing file. + + :param kwargs: Additional arguments to specify. """ attach_list = deque() count_start_at = 0 @@ -551,6 +560,7 @@ def get_attachments_on_projects( folder=attachment_folder, file_name=attachment_file_name, data=headers, + mode=mode, **kwargs, ) @@ -955,45 +965,85 @@ def move_attachments_across_instances( @staticmethod def download_attachments( - file_folder: str = None, - file_name: str = None, + file_folder: str = 'Attachment', + file_name: str = 'attachment_file.csv', download_path: str = "Downloads", attach: int = 8, + skip_csv_header: bool = True, + overwrite: bool = True, + create_html_redirectors: bool = False, **kwargs, ) -> None: - """Download the attachments to your local device read from a csv file. + """Go through the attachment list CSV file named ``file_name`` and located in the + ``file_folder``; for each row, download the attachment indicated to your local device. - we assume you're getting this from ``def get_attachments_on_project()`` - method. + Calling this method with default arguments assumes that you've + previously called the ``def get_attachments_on_project()`` method + with default arguments. + + To avoid conflicts whenever attachments have the same filename (e.g. ``screenshot-1.png``), + each downloaded attachment will be placed in a separate directory which corresponds to the + content ID that Jira gives the attachment. - :param attach: integers to specify the index of the columns + :param download_path: the directory where the downloaded attachments are to be stored - :param file_folder: a folder or directory where the file extract exist + :param file_folder: the directory where the attachment list CSV file can be found + (Default corresponds to the output of ``def get_attachments_on_project()`` + when called with default arguments) - :param download_path: a directory where files are stored + :param file_name: file name of the attachment list CSV file + (Default corresponds to the output of ``def get_attachments_on_project()`` + when called with default arguments) - :param file_name: a file name to a file + :param attach: index of the column that corresponds to 'Attachment URL' in the attachment + list CSV file. + (Default is 8, which corresponds to the output of ``def get_attachments_on_project()``) - e.g - * attach=6, - * file=8 + :param skip_csv_header: when set to True, skips the first line of the attachment list CSV file; i.e. + assumes that the first line represents a header row. + (Default is True, which corresponds to the output of ``def get_attachments_on_project()``) + + :param overwrite: when True, any attachments will be overwritten. When False, downloading + of the attachment will be skipped. Setting this to False can significantly speed up + incremental backups by only downloading attachments that have not yet been downloaded. + + :param create_html_redirectors: is used when you want to use the downloaded attachments + as part of a website to mirror and serve the attachments separately from the + Jira website. When set to True, an ``index.html`` will be created + for each attachment so that the original Jira Attachment URL, e.g. + https://yourorganization.atlassian.net/rest/api/3/attachment/content/112736 , + can be more easily rewritten to something like + https://yourmirrorsite.com/jiraone/MYPROJ/attachment/content/112736 . + The ``index.html`` will take care of the HTTP redirect that will point to the + attachment with the original filename. :param kwargs: Additional keyword argument **Acceptable options** - * file: A row to the index of the column - - the above example corresponds with the index if using the - ``def get_attachments_on_project()`` otherwise, specify your value - in each keyword args when calling the method. + * file: index of the column 'Name of file' in the attachment list CSV file. + (Default is 6, which corresponds to the output of + ``def get_attachments_on_project()``) :return: None """ + HTML_REDIRECTOR_TEMPLATE = """<!DOCTYPE html> +<html> +<head> + <title>Download File</title> + <meta http-equiv="refresh" content="0; url={path}"> +</head> +<body> + <p>If you are not redirected automatically, please click <a href="{path}">here</a> to download the file.</p> +</body> +</html> +""" + file: int = kwargs.get("file", 6) read = file_reader( folder=file_folder, file_name=file_name, + skip=skip_csv_header, **kwargs, ) add_log( @@ -1008,22 +1058,41 @@ def download_attachments( count += 1 attachment = r[attach] _file_name = r[file] - fetch = LOGIN.get(attachment) - file_writer( - download_path, - file_name=_file_name, - mode="wb", - content=fetch.content, - mark="file", - ) - print( - "Attachment downloaded to {}".format(download_path), - "Status code: {}".format(fetch.status_code), - ) - add_log( - "Attachment downloaded to {}".format(download_path), - "info", - ) + if attachment == '' or _file_name == '': + # For example the last line of the attachment list CSV may have: ,,,,Total Size: + # 0.09 MB,,,, + continue + content_id = attachment.split('/')[-1] + individual_download_path = os.path.join(download_path, content_id) + if not os.path.exists(individual_download_path): + os.makedirs(individual_download_path) + if overwrite or not os.path.exists(os.path.join(individual_download_path, _file_name)): + fetch = LOGIN.get(attachment) + file_writer( + folder=individual_download_path, + file_name=_file_name, + mode="wb", + content=fetch.content, + mark="file", + ) + print( + "Attachment downloaded to {}".format(individual_download_path), + "Status code: {}".format(fetch.status_code), + ) + add_log( + "Attachment downloaded to {}".format(individual_download_path), + "info", + ) + if create_html_redirectors: + # Create HTML file with a template that + html_content = HTML_REDIRECTOR_TEMPLATE.format(path=_file_name) + # Write the content to file + with open(os.path.join(individual_download_path, 'index.html'), 'w') as html_file: + html_file.write(html_content) + add_log( + "Attachment HTML redirector created in {}".format(individual_download_path), + "info", + ) if last_cell is True: if count >= (length - 1): break @@ -5830,7 +5899,7 @@ def json_field_builder() -> None: {f"{sprint_item[sub_sprint]}": []} ) - if "customType" in sprint_custom_id: + if sprint_custom_id is not None and "customType" in sprint_custom_id: if sprint_custom_id["customType"].endswith("gh-sprint"): extract = sprint_custom_id["id"].split("_") config["sprint_cf"] = "cf[{}]".format(extract[1]) @@ -7745,7 +7814,7 @@ def path_builder( path, ) if not os.path.exists(base_dir): - os.mkdir(base_dir) + os.makedirs(base_dir) add_log( f"Building Path {path}", "info", diff --git a/test.py b/test.py index db6d20a..387aaa8 100644 --- a/test.py +++ b/test.py @@ -1,6 +1,7 @@ """ Run test for different features of the library """ + import os import json import unittest @@ -17,6 +18,7 @@ USER, file_writer, field, + __version__, ) from jiraone.module import time_in_status, bulk_change_email @@ -30,10 +32,7 @@ def setUp(self): """Configure test case""" user = os.environ.get("JIRAONEUSERNAME") or "email" password = os.environ.get("JIRAONEUSERPASS") or "token" - link = ( - os.environ.get("JIRAONEUSERURL") - or "https://yourinstance.atlassian.net" - ) + link = os.environ.get("JIRAONEUSERURL") or "https://yourinstance.atlassian.net" LOGIN(user=user, password=password, url=link) self.payload = { "fields": { @@ -62,14 +61,12 @@ def test_context_login_session(self): LOGIN.session.headers = LOGIN.headers LOGIN.session.auth = LOGIN.auth_request session = LOGIN.session.get(endpoint.myself()) - self.assertTrue(session.status_code < 300, - "Session context failed") + self.assertTrue(session.status_code < 300, "Session context failed") def test_endpoints(self): """Test endpoint constant extraction""" load = LOGIN.get(endpoint.myself()) - self.assertTrue(load.status_code < 300, - "Unable to load self endpoint") + self.assertTrue(load.status_code < 300, "Unable to load self endpoint") def test_data_extraction(self): """Test response data from an endpoint""" @@ -81,8 +78,7 @@ def test_data_extraction(self): def test_issue_export_csv(self): """Test CSV issue export""" jql = self.jql - path = path_builder("TEST", - "test_import_CSV_sample.csv") + path = path_builder("TEST", "test_import_CSV_sample.csv") # testing parameters issue_export( jql=jql, @@ -91,15 +87,13 @@ def test_issue_export_csv(self): final_file="test_import_CSV_sample.csv", ) self.assertTrue( - os.path.isfile(path), - "Unable to detect CSV file for issue export" + os.path.isfile(path), "Unable to detect CSV file for issue export" ) def test_issue_export_json(self): """Test JSON issue export""" jql = self.jql - path = path_builder("TEST", - "test_import_json_sample.json") + path = path_builder("TEST", "test_import_json_sample.json") LOGIN.api = True issue_export( jql=jql, @@ -108,17 +102,14 @@ def test_issue_export_json(self): final_file="test_import_json_sample.json", ) self.assertTrue( - os.path.isfile(path), - "Unable to detect JSON file for issue export" + os.path.isfile(path), "Unable to detect JSON file for issue export" ) def test_time_in_status(self): """Test for time in status for CSV or JSON""" key = self.issue_key - json_file = path_builder("TEST", - "test_time_in_status.json") - csv_file = path_builder("TEST", - "test_time_in_status.csv") + json_file = path_builder("TEST", "test_time_in_status.json") + csv_file = path_builder("TEST", "test_time_in_status.csv") time_in_status( PROJECT, key, @@ -181,8 +172,7 @@ def test_delete_attachment(self): delete_attachments(search=issue_key) # check for file existence image_url = LOGIN.get(upload.json()[0].get("content")) - self.assertFalse(image_url.status_code < 300, - "Attachment still exist") + self.assertFalse(image_url.status_code < 300, "Attachment still exist") def test_search_user(self): """Test for user search""" @@ -228,17 +218,12 @@ def rearrange_users() -> list: values[1][0], values[1][3], ) - file_writer( - file_name="sampleTest.csv", data=headers, mark="single", mode="w+" - ) - file_writer( - file_name="sampleTest.csv", data=values, mark="many", mode="a+" - ) + file_writer(file_name="sampleTest.csv", data=headers, mark="single", mode="w+") + file_writer(file_name="sampleTest.csv", data=values, mark="many", mode="a+") bulk_change_email("sampleTest.csv", self.token) # check for those users verify_user = manage.manage_profile(account_id_two) - self.assertTrue(verify_user.status_code < 300, - "Verifying user failed") + self.assertTrue(verify_user.status_code < 300, "Verifying user failed") self.assertNotEqual( right_email_two, verify_user.json().get("account").get("email"), @@ -269,15 +254,13 @@ def test_organization_access(self): disable = manage.manage_user( i.get("account_id"), json=payload, disable=True ) - self.assertTrue(disable.status_code < 300, - "Disabling user failed") + self.assertTrue(disable.status_code < 300, "Disabling user failed") # enable user for i in user_list: enable = manage.manage_user( i.get("account_id"), json=payload, disable=False ) - self.assertTrue(enable.status_code < 300, - "Enabling user failed") + self.assertTrue(enable.status_code < 300, "Enabling user failed") def test_multiprocessing_history_extraction(self): """Test for multiprocessing on history extraction""" @@ -285,12 +268,8 @@ def test_multiprocessing_history_extraction(self): LOGIN.api = True if hasattr(PROJECT, "async_change_log"): project_attr = getattr(PROJECT, "async_change_log") - project_attr( - self.jql, folder="TEST", file="sampleAsyncFile.csv", flush=10 - ) - self.assertTrue( - os.path.isfile(path), "Unable to find change log file." - ) + project_attr(self.jql, folder="TEST", file="sampleAsyncFile.csv", flush=10) + self.assertTrue(os.path.isfile(path), "Unable to find change log file.") def test_field_extraction(self): """Test for field extraction""" @@ -302,16 +281,14 @@ def test_http_request(self): to Jira resource""" # get get = LOGIN.get(endpoint.myself()) - self.assertTrue(get.status_code < 300, - "GET request is not working") + self.assertTrue(get.status_code < 300, "GET request is not working") # post LOGIN.api = False post = LOGIN.post( endpoint.issues(), payload=self.payload, ) - self.assertTrue(post.status_code < 300, - "POST request is not working") + self.assertTrue(post.status_code < 300, "POST request is not working") if post.status_code < 300: issue_key = post.json().get("key") # put @@ -319,13 +296,79 @@ def test_http_request(self): endpoint.issues(issue_key), payload={"fields": {"priority": {"name": "Highest"}}}, ) - self.assertTrue(put.status_code < 300, - "PUT request is not working") + self.assertTrue(put.status_code < 300, "PUT request is not working") # delete delete = LOGIN.delete(endpoint.issues(issue_key)) - self.assertTrue( - delete.status_code < 300, "DELETE request is not working" + self.assertTrue(delete.status_code < 300, "DELETE request is not working") + + def test_get_attachments(self): + """Test for get attachments on projects""" + # upload the same attachment trice + upload = self.uploader() + self.assertTrue(upload is True, "Cannot add attachment") + if __version__ >= "0.8.4": + PROJECT.get_attachments_on_projects(query=self.jql) + path = path_builder("Attachment", "attachment_file.csv") + self.assertTrue(os.path.isfile(path), "Unable to find attachment file.") + + def test_download_attachments(self): + """Test for download attachment files""" + # upload some attachments to an issue or multiple issues + upload = self.uploader() + self.assertTrue(upload is True, "Cannot add attachment") + # then download those attachments + if __version__ >= "0.8.4": + PROJECT.get_attachments_on_projects(query=self.jql) + PROJECT.download_attachments( + file_folder="Attachment", + file_name="attachment_file.csv", + download_path="Downloads", + skip_csv_header=True, + overwrite=False, + create_html_redirectors=True, ) + # verify download + is_download = [] + path = path_builder("Attachment", "attachment_file.csv") + read_attachment = file_reader(file_name=path, skip=True) + for attach_id in read_attachment: + uri_attachment = attach_id[8].split("/")[-1] + file_name = attach_id[6] + new_path = path_builder(f"Downloads/{uri_attachment}", f"{file_name}") + if os.path.isfile(new_path): + is_download.append(True) + else: + is_download.append(False) + self.assertTrue(all(is_download) is True, "Attachment download failed") + + + def uploader(self) -> bool: + """uploader function""" + count = 0 + check_upload = [] + for image in [self.issue_keys, self.issue_keys, self.issue_keys]: + issue_key = image + # upload a public file for the test + file_name = "test-attachment-{}".format(count) + attach_file = self.attachment # public accessible file + fetch = LOGIN.custom_method("GET", attach_file) + payload = {"file": (file_name, fetch.content)} + new_headers = { + "Accept": "application/json", + "X-Atlassian-Token": "no-check", + } + LOGIN.headers = new_headers + upload = LOGIN.post( + endpoint.issue_attachments(issue_key, query="attachments"), + files=payload, + ) + count += 1 + if upload.status_code < 300: + check_upload.append(True) + else: + check_upload.append(False) + + return all(check_upload) def tearDown(self): """Closes the test operations"""