From 9f1db974b3a98fca1b952f61f09b37eb1bdd5367 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 31 Oct 2022 17:37:25 -0700 Subject: [PATCH] feat: parse new USPS informed delivery digest emails (#739) * feat: parse new USPS informed delivery digest emails * linting --- .../mail_and_packages/helpers.py | 83 +- .../mail_and_packages/manifest.json | 2 +- requirements.txt | 3 +- tests/conftest.py | 26 + tests/test_emails/new_informed_delivery.eml | 11076 ++++++++++++++++ tests/test_helpers.py | 23 + 6 files changed, 11188 insertions(+), 25 deletions(-) create mode 100644 tests/test_emails/new_informed_delivery.eml diff --git a/custom_components/mail_and_packages/helpers.py b/custom_components/mail_and_packages/helpers.py index c0ab97d1..ceee4c17 100644 --- a/custom_components/mail_and_packages/helpers.py +++ b/custom_components/mail_and_packages/helpers.py @@ -1,5 +1,6 @@ """Helper functions for Mail and Packages.""" +import base64 import datetime import email import hashlib @@ -17,6 +18,7 @@ from typing import Any, List, Optional, Type, Union import aiohttp +from bs4 import BeautifulSoup from homeassistant.config_entries import ConfigEntry from homeassistant.const import ( CONF_HOST, @@ -632,31 +634,61 @@ def get_mails( if server_response == "OK": _LOGGER.debug("Informed Delivery email found processing...") for num in data[0].split(): - msg = email.message_from_string( - email_fetch(account, num, "(RFC822)")[1][0][1].decode("utf-8") - ) - - # walking through the email parts to find images - for part in msg.walk(): - if part.get_content_maintype() == "multipart": - continue - if part.get("Content-Disposition") is None: - continue + msg = email_fetch(account, num, "(RFC822)")[1] + for response_part in msg: + if isinstance(response_part, tuple): + msg = email.message_from_bytes(response_part[1]) + _LOGGER.debug("msg: %s", msg) + + # walking through the email parts to find images + for part in msg.walk(): + if part.get_content_type() == "text/html": + _LOGGER.debug("Found html email processing...") + part = part.get_payload(decode=True) + part = part.decode("utf-8", "ignore") + soup = BeautifulSoup(part, "html.parser") + found_images = soup.find_all(id="mailpiece-image-src-id") + if not found_images: + continue + _LOGGER.debug("Found images: %s", bool(found_images)) - _LOGGER.debug("Extracting image from email") + # Convert all the images to binary data + for image in found_images: + filename = random_filename() + data = str(image["src"]).split(",")[1] + _LOGGER.debug("Data: %s", data) + try: + with open( + image_output_path + filename, "wb" + ) as the_file: + the_file.write(base64.b64decode(data)) + images.append(image_output_path + filename) + image_count = image_count + 1 + except Exception as err: + _LOGGER.critical( + "Error opening filepath: %s", str(err) + ) + return image_count + + # Log error message if we are unable to open the filepath for + # some reason + elif part.get_content_type() == "image/jpeg": + _LOGGER.debug("Extracting image from email") + try: + with open( + image_output_path + part.get_filename(), "wb" + ) as the_file: + the_file.write(part.get_payload(decode=True)) + images.append( + image_output_path + part.get_filename() + ) + image_count = image_count + 1 + except Exception as err: + _LOGGER.critical("Error opening filepath: %s", str(err)) + return image_count - # Log error message if we are unable to open the filepath for - # some reason - try: - with open( - image_output_path + part.get_filename(), "wb" - ) as the_file: - the_file.write(part.get_payload(decode=True)) - images.append(image_output_path + part.get_filename()) - image_count = image_count + 1 - except Exception as err: - _LOGGER.critical("Error opening filepath: %s", str(err)) - return image_count + elif part.get_content_type() == "multipart": + continue # Remove duplicate images _LOGGER.debug("Removing duplicate images.") @@ -738,6 +770,11 @@ def get_mails( return image_count +def random_filename(ext: str = ".jpg") -> str: + """Generate random filename.""" + return f"{str(uuid.uuid4())}{ext}" + + def _generate_mp4(path: str, image_file: str) -> None: """Generate mp4 from gif. diff --git a/custom_components/mail_and_packages/manifest.json b/custom_components/mail_and_packages/manifest.json index 711596aa..7675280d 100644 --- a/custom_components/mail_and_packages/manifest.json +++ b/custom_components/mail_and_packages/manifest.json @@ -9,7 +9,7 @@ "@firstof9" ], "config_flow": true, - "requirements": ["Pillow>=9.0"], + "requirements": ["beautifulsoup4","Pillow>=9.0"], "iot_class": "cloud_polling", "version": "0.0.0-dev" } \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 626182af..8e6177bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -Pillow>=9.0 \ No newline at end of file +Pillow>=9.0 +beautifulsoup4 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index f1f1d7f8..c04ce503 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -265,6 +265,32 @@ def mock_imap_usps_informed_digest(): yield mock_conn +@pytest.fixture() +def mock_imap_usps_new_informed_digest(): + """Mock imap class values.""" + with patch( + "custom_components.mail_and_packages.helpers.imaplib" + ) as mock_imap_usps_new_informed_digest: + mock_conn = mock.Mock(spec=imaplib.IMAP4_SSL) + mock_imap_usps_new_informed_digest.IMAP4_SSL.return_value = mock_conn + + mock_conn.login.return_value = ( + "OK", + [b"user@fake.email authenticated (Success)"], + ) + mock_conn.list.return_value = ( + "OK", + [b'(\\HasNoChildren) "/" "INBOX"'], + ) + mock_conn.search.return_value = ("OK", [b"1"]) + mock_conn.uid.return_value = ("OK", [b"1"]) + f = open("tests/test_emails/new_informed_delivery.eml", "r") + email_file = f.read() + mock_conn.fetch.return_value = ("OK", [(b"", email_file.encode("utf-8"))]) + mock_conn.select.return_value = ("OK", []) + yield mock_conn + + @pytest.fixture() def mock_imap_usps_informed_digest_missing(): """Mock imap class values.""" diff --git a/tests/test_emails/new_informed_delivery.eml b/tests/test_emails/new_informed_delivery.eml new file mode 100644 index 00000000..48c7494a --- /dev/null +++ b/tests/test_emails/new_informed_delivery.eml @@ -0,0 +1,11076 @@ +Date: Mon, 31 Oct 2022 15:02:19 +0000 +Subject: Your Daily Digest for Mon, Oct 31 +From: USPSInformeddelivery@email.informeddelivery.usps.com +To: testuser@fake.email +Content-Transfer-Encoding: quoted-printable +Content-Type: text/html; charset="utf-8" + + +=20 + =20 + = +=20 + =20 + Informed Delivery=20 + =20 + =20 + =20 + =20 +
=20 + =20 + + =20 + =20 + =20 + +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
You= + have mail and packages arriving soon. 10/31/2022
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"USPS.com= COMING TO YOUR MAILBOX SOON.
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
View all mail on dashboard
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
FromBest Egg
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D""
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Ride
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Learn Learn More
3D"Set Set a Reminder
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
FromVote AGAINST Mark Kelly and Katie Hobbs
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D""
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Ride
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Learn Learn More
3D"Set Set a Reminder
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
FromFeeding America
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D""
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Ride
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Learn Learn More
3D"Set Set a Reminder
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
FromFeeding America
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + = +=20 + =20 + =20 +
3D""
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Ride
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Learn Learn More
3D"Set Set a Reminder
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
FromVote AGAINST Mark Kelly and Katie Hobbs
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D""
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Ride
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Learn Learn More
3D"Set Set a Reminder
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
FromBest Egg
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D""
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Ride
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D"Learn Learn More
3D"Set Set a Reminder
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
= +=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D""= +
 
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
3D"SetSet a R= +eminder
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
= +=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D""
 
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
3D"SetSet a R= +eminder
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
= +=20 + =20 + =20 + =20 + = +=20 + =20 + =20 +
3D""
 
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
3D"SetSet a R= +eminder
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
= +=20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D""
 
=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 +
+ Do more with your mail=20 +
3D"SetSet a R= +eminder
=20 +
=20 +


= +=20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 +
3D==20 +
View and manage all packages on dashboard
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + + =20 + =20 + =20 + =20 + =20 + =20 + =20 + +
+
Arriving Today=20 +
+
Monday, Oct 31=20 +

+
=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +

CNE -LAX

92= +14490276543059184146

=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + + =20 + =20 + =20 + =20 + =20 + =20 + =20 + +
+
Arriving Soon=20 +
+

+
=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +

CNE -LAX

92= +12490276543059243005

Esti= +mated Delivery on: Wednesday, Nov 02

=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + =20 + +
+
Outbound =20 +
+

+

No packages available to display.

=20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +
=20 + =20 + =20 + =20 + =20 + =20 + =20 +

*Th= +ese images represent mail pieces that are sorted on USPS=C2=AE a= +utomated equipment. Some of your mail may not be shown here.

You subscribed to this service with USPS=C2=AE Innova= +tive Business Technology, PO Box 23972, Washington DC 20026-3972.

If you no longer wish to receive daily email notifications,= + unsubscribe here.

If you need supp= +ort, please visit user support for Informed Delivery=C2=AE.

For more information about this service, please visit general informat= +ion about Informed Delivery.

Copyright =C2=A9 = +2022 United States Postal Service=C2=AE. All Rights Reserved. Th= +e Eagle Logo and the trade dress of USPS=C2=AE Packaging are amo= +ng the many trademarks of the U.S. Postal Service=C2=AE.

This is an automated email, please do not reply to this me= +ssage. This message is for the designated recipient only and may contain pr= +ivileged, proprietary, or otherwise private information. If you have receiv= +ed it in error, please delete. Any other use of the email by you is prohibi= +ted.

=20 +


3D"" 3D""

=20 +
=20 + + diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 56f14767..30d0b49e 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -515,6 +515,29 @@ async def test_informed_delivery_emails( assert "USPS Informed Delivery" in caplog.text +async def test_new_informed_delivery_emails( + mock_imap_usps_new_informed_digest, + mock_osremove, + mock_osmakedir, + mock_listdir, + mock_os_path_splitext, + mock_image, + mock_resizeimage, + mock_copyfile, + caplog, +): + m_open = mock_open() + with patch("builtins.open", m_open, create=True): + result = get_mails( + mock_imap_usps_new_informed_digest, "./", "5", "mail_today.gif", False + ) + assert result == 4 + assert "USPSInformedDelivery@usps.gov" in caplog.text + assert "USPSInformeddelivery@informeddelivery.usps.com" in caplog.text + assert "USPSInformeddelivery@email.informeddelivery.usps.com" in caplog.text + assert "USPS Informed Delivery" in caplog.text + + # async def test_get_mails_image_error( # mock_imap_usps_informed_digest, # mock_osremove,