diff --git a/.gitignore b/.gitignore
index eff8c02..3bc59df 100644
--- a/.gitignore
+++ b/.gitignore
@@ -156,3 +156,6 @@ cython_debug/
# PyCharm
.idea/
+
+# VSCode
+.vscode
diff --git a/Pipfile b/Pipfile
index 43516c1..aa9f89c 100644
--- a/Pipfile
+++ b/Pipfile
@@ -4,6 +4,7 @@ verify_ssl = true
name = "pypi"
[packages]
+attrs = "*"
boto3 = "*"
duckdb = "*"
pandas = "*"
@@ -14,15 +15,14 @@ black = "*"
boto3-stubs = {version = "*", extras = ["s3"]}
coveralls = "*"
ipython = "*"
+moto = "*"
mypy = "*"
+pandas-stubs = "*"
pre-commit = "*"
+pytest-mock = "*"
pyarrow-stubs = "*"
pytest = "*"
ruff = "*"
setuptools = "*"
-pandas-stubs = "*"
-moto = "*"
-pytest-mock = "*"
-
[requires]
python_version = "3.12"
diff --git a/Pipfile.lock b/Pipfile.lock
index 7b97bef..2074530 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "3f9d0481f2dc996f511acff686489d2a2bb88469cb7ec87e769b77e6c53b2549"
+ "sha256": "d9e081653de47fb0c08e6c6f8e86f46e91d9f55868a240d088603e4af2dd91e3"
},
"pipfile-spec": 6,
"requires": {
@@ -16,22 +16,31 @@
]
},
"default": {
+ "attrs": {
+ "hashes": [
+ "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346",
+ "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"
+ ],
+ "index": "pypi",
+ "markers": "python_version >= '3.7'",
+ "version": "==24.2.0"
+ },
"boto3": {
"hashes": [
- "sha256:473438feafe77d29fbea532a91a65de0d8751a4fa5822127218710a205e28e7a",
- "sha256:ccb1a365d3084de53b58f8dfc056462f49b16931c139f4c8ac5f0bca8cb8fe81"
+ "sha256:5ef7166fe5060637b92af8dc152cd7acecf96b3fc9c5456706a886cadb534391",
+ "sha256:fc8001519c8842e766ad3793bde3fbd0bb39e821a582fc12cf67876b8f3cf7f1"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
- "version": "==1.35.73"
+ "version": "==1.35.78"
},
"botocore": {
"hashes": [
- "sha256:8a6a0f5ad119e38d850571df8c625dbad66aec1b20c15f84cdcb95258f9f1edb",
- "sha256:b2e3ecdd1769f011f72c4c0d0094570ba125f4ca327f24269e4d68eb5d9878b9"
+ "sha256:41c37bd7c0326f25122f33ec84fb80fc0a14d7fcc9961431b0e57568e88c9cb5",
+ "sha256:6905036c25449ae8dba5e950e4b908e4b8a6fe6b516bf61e007ecb62fa21f323"
],
"markers": "python_version >= '3.8'",
- "version": "==1.35.73"
+ "version": "==1.35.78"
},
"duckdb": {
"hashes": [
@@ -102,64 +111,64 @@
},
"numpy": {
"hashes": [
- "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe",
- "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0",
- "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48",
- "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a",
- "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564",
- "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958",
- "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17",
- "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0",
- "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee",
- "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b",
- "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4",
- "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4",
- "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6",
- "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4",
- "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d",
- "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f",
- "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f",
- "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f",
- "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56",
- "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9",
- "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd",
- "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23",
- "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed",
- "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a",
- "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098",
- "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1",
- "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512",
- "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f",
- "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09",
- "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f",
- "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc",
- "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8",
- "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0",
- "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761",
- "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef",
- "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5",
- "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e",
- "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b",
- "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d",
- "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43",
- "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c",
- "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41",
- "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff",
- "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408",
- "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2",
- "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9",
- "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57",
- "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb",
- "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9",
- "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3",
- "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a",
- "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0",
- "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e",
- "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598",
- "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4"
+ "sha256:0557eebc699c1c34cccdd8c3778c9294e8196df27d713706895edc6f57d29608",
+ "sha256:0798b138c291d792f8ea40fe3768610f3c7dd2574389e37c3f26573757c8f7ef",
+ "sha256:0da8495970f6b101ddd0c38ace92edea30e7e12b9a926b57f5fabb1ecc25bb90",
+ "sha256:0f0986e917aca18f7a567b812ef7ca9391288e2acb7a4308aa9d265bd724bdae",
+ "sha256:122fd2fcfafdefc889c64ad99c228d5a1f9692c3a83f56c292618a59aa60ae83",
+ "sha256:140dd80ff8981a583a60980be1a655068f8adebf7a45a06a6858c873fcdcd4a0",
+ "sha256:16757cf28621e43e252c560d25b15f18a2f11da94fea344bf26c599b9cf54b73",
+ "sha256:18142b497d70a34b01642b9feabb70156311b326fdddd875a9981f34a369b671",
+ "sha256:1c92113619f7b272838b8d6702a7f8ebe5edea0df48166c47929611d0b4dea69",
+ "sha256:1e25507d85da11ff5066269d0bd25d06e0a0f2e908415534f3e603d2a78e4ffa",
+ "sha256:30bf971c12e4365153afb31fc73f441d4da157153f3400b82db32d04de1e4066",
+ "sha256:3579eaeb5e07f3ded59298ce22b65f877a86ba8e9fe701f5576c99bb17c283da",
+ "sha256:36b2b43146f646642b425dd2027730f99bac962618ec2052932157e213a040e9",
+ "sha256:3905a5fffcc23e597ee4d9fb3fcd209bd658c352657548db7316e810ca80458e",
+ "sha256:3a4199f519e57d517ebd48cb76b36c82da0360781c6a0353e64c0cac30ecaad3",
+ "sha256:3f2f5cddeaa4424a0a118924b988746db6ffa8565e5829b1841a8a3bd73eb59a",
+ "sha256:40deb10198bbaa531509aad0cd2f9fadb26c8b94070831e2208e7df543562b74",
+ "sha256:440cfb3db4c5029775803794f8638fbdbf71ec702caf32735f53b008e1eaece3",
+ "sha256:4723a50e1523e1de4fccd1b9a6dcea750c2102461e9a02b2ac55ffeae09a4410",
+ "sha256:4bddbaa30d78c86329b26bd6aaaea06b1e47444da99eddac7bf1e2fab717bd72",
+ "sha256:4e58666988605e251d42c2818c7d3d8991555381be26399303053b58a5bbf30d",
+ "sha256:54dc1d6d66f8d37843ed281773c7174f03bf7ad826523f73435deb88ba60d2d4",
+ "sha256:57fcc997ffc0bef234b8875a54d4058afa92b0b0c4223fc1f62f24b3b5e86038",
+ "sha256:58b92a5828bd4d9aa0952492b7de803135038de47343b2aa3cc23f3b71a3dc4e",
+ "sha256:5a145e956b374e72ad1dff82779177d4a3c62bc8248f41b80cb5122e68f22d13",
+ "sha256:6ab153263a7c5ccaf6dfe7e53447b74f77789f28ecb278c3b5d49db7ece10d6d",
+ "sha256:7832f9e8eb00be32f15fdfb9a981d6955ea9adc8574c521d48710171b6c55e95",
+ "sha256:7fe4bb0695fe986a9e4deec3b6857003b4cfe5c5e4aac0b95f6a658c14635e31",
+ "sha256:7fe8f3583e0607ad4e43a954e35c1748b553bfe9fdac8635c02058023277d1b3",
+ "sha256:85ad7d11b309bd132d74397fcf2920933c9d1dc865487128f5c03d580f2c3d03",
+ "sha256:9874bc2ff574c40ab7a5cbb7464bf9b045d617e36754a7bc93f933d52bd9ffc6",
+ "sha256:a184288538e6ad699cbe6b24859206e38ce5fba28f3bcfa51c90d0502c1582b2",
+ "sha256:a222d764352c773aa5ebde02dd84dba3279c81c6db2e482d62a3fa54e5ece69b",
+ "sha256:a50aeff71d0f97b6450d33940c7181b08be1441c6c193e678211bff11aa725e7",
+ "sha256:a55dc7a7f0b6198b07ec0cd445fbb98b05234e8b00c5ac4874a63372ba98d4ab",
+ "sha256:a62eb442011776e4036af5c8b1a00b706c5bc02dc15eb5344b0c750428c94219",
+ "sha256:a7d41d1612c1a82b64697e894b75db6758d4f21c3ec069d841e60ebe54b5b571",
+ "sha256:a98f6f20465e7618c83252c02041517bd2f7ea29be5378f09667a8f654a5918d",
+ "sha256:afe8fb968743d40435c3827632fd36c5fbde633b0423da7692e426529b1759b1",
+ "sha256:b0b227dcff8cdc3efbce66d4e50891f04d0a387cce282fe1e66199146a6a8fca",
+ "sha256:b30042fe92dbd79f1ba7f6898fada10bdaad1847c44f2dff9a16147e00a93661",
+ "sha256:b606b1aaf802e6468c2608c65ff7ece53eae1a6874b3765f69b8ceb20c5fa78e",
+ "sha256:b6207dc8fb3c8cb5668e885cef9ec7f70189bec4e276f0ff70d5aa078d32c88e",
+ "sha256:c2aed8fcf8abc3020d6a9ccb31dbc9e7d7819c56a348cc88fd44be269b37427e",
+ "sha256:cb24cca1968b21355cc6f3da1a20cd1cebd8a023e3c5b09b432444617949085a",
+ "sha256:cff210198bb4cae3f3c100444c5eaa573a823f05c253e7188e1362a5555235b3",
+ "sha256:d35717333b39d1b6bb8433fa758a55f1081543de527171543a2b710551d40881",
+ "sha256:df12a1f99b99f569a7c2ae59aa2d31724e8d835fc7f33e14f4792e3071d11221",
+ "sha256:e09d40edfdb4e260cb1567d8ae770ccf3b8b7e9f0d9b5c2a9992696b30ce2742",
+ "sha256:e12c6c1ce84628c52d6367863773f7c8c8241be554e8b79686e91a43f1733773",
+ "sha256:e2b8cd48a9942ed3f85b95ca4105c45758438c7ed28fff1e4ce3e57c3b589d8e",
+ "sha256:e500aba968a48e9019e42c0c199b7ec0696a97fa69037bea163b55398e390529",
+ "sha256:ebe5e59545401fbb1b24da76f006ab19734ae71e703cdb4a8b347e84a0cece67",
+ "sha256:f0dd071b95bbca244f4cb7f70b77d2ff3aaaba7fa16dc41f58d14854a6204e6c",
+ "sha256:f8c8b141ef9699ae777c6278b52c706b653bf15d135d302754f6b2e90eb30367"
],
"markers": "python_version >= '3.12'",
- "version": "==2.1.3"
+ "version": "==2.2.0"
},
"pandas": {
"hashes": [
@@ -284,11 +293,11 @@
},
"six": {
"hashes": [
- "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
- "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+ "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274",
+ "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==1.16.0"
+ "version": "==1.17.0"
},
"tzdata": {
"hashes": [
@@ -347,39 +356,39 @@
},
"boto3": {
"hashes": [
- "sha256:473438feafe77d29fbea532a91a65de0d8751a4fa5822127218710a205e28e7a",
- "sha256:ccb1a365d3084de53b58f8dfc056462f49b16931c139f4c8ac5f0bca8cb8fe81"
+ "sha256:5ef7166fe5060637b92af8dc152cd7acecf96b3fc9c5456706a886cadb534391",
+ "sha256:fc8001519c8842e766ad3793bde3fbd0bb39e821a582fc12cf67876b8f3cf7f1"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
- "version": "==1.35.73"
+ "version": "==1.35.78"
},
"boto3-stubs": {
"extras": [
"s3"
],
"hashes": [
- "sha256:b935f0b62be1e18445f63cd9f5bbb4fe9a792d99efa9eb7f37b641ed4a6e70e0",
- "sha256:d1c072dfa59fbe0d91ba8e8966e844d9eb79ccc5f59e49914f796f29cd96a14d"
+ "sha256:5d023cf1fcc723dfdba29653e0ad9b9933985c813a25bc21807e21eab81e21b4",
+ "sha256:bb7824c09cbf868940b8d500e7f4ca99cb7e074c0f86771f832ce15c33a313bd"
],
"markers": "python_version >= '3.8'",
- "version": "==1.35.73"
+ "version": "==1.35.78"
},
"botocore": {
"hashes": [
- "sha256:8a6a0f5ad119e38d850571df8c625dbad66aec1b20c15f84cdcb95258f9f1edb",
- "sha256:b2e3ecdd1769f011f72c4c0d0094570ba125f4ca327f24269e4d68eb5d9878b9"
+ "sha256:41c37bd7c0326f25122f33ec84fb80fc0a14d7fcc9961431b0e57568e88c9cb5",
+ "sha256:6905036c25449ae8dba5e950e4b908e4b8a6fe6b516bf61e007ecb62fa21f323"
],
"markers": "python_version >= '3.8'",
- "version": "==1.35.73"
+ "version": "==1.35.78"
},
"botocore-stubs": {
"hashes": [
- "sha256:54f7bcc325382050ae6aa839163f93f5c4e777db9c0fd2da3ad0744720895fbe",
- "sha256:e9a20b0a29621674b46225fdb88bf00a0bca5216413d717895b75ba2dd63c6cc"
+ "sha256:4cb5c1fca33048a2afca2002719a8d696f7051ab4f0ef5f5ee96df7aaf76a055",
+ "sha256:86d11b64a72c25766d551a2fedcc93e374d3c9d27aea11a7516af1d357e09637"
],
"markers": "python_version >= '3.8'",
- "version": "==1.35.73"
+ "version": "==1.35.78"
},
"certifi": {
"hashes": [
@@ -594,71 +603,71 @@
"toml"
],
"hashes": [
- "sha256:093896e530c38c8e9c996901858ac63f3d4171268db2c9c8b373a228f459bbc5",
- "sha256:09b9f848b28081e7b975a3626e9081574a7b9196cde26604540582da60235fdf",
- "sha256:0b0c69f4f724c64dfbfe79f5dfb503b42fe6127b8d479b2677f2b227478db2eb",
- "sha256:13618bed0c38acc418896005732e565b317aa9e98d855a0e9f211a7ffc2d6638",
- "sha256:13690e923a3932e4fad4c0ebfb9cb5988e03d9dcb4c5150b5fcbf58fd8bddfc4",
- "sha256:177f01eeaa3aee4a5ffb0d1439c5952b53d5010f86e9d2667963e632e30082cc",
- "sha256:193e3bffca48ad74b8c764fb4492dd875038a2f9925530cb094db92bb5e47bed",
- "sha256:1defe91d41ce1bd44b40fabf071e6a01a5aa14de4a31b986aa9dfd1b3e3e414a",
- "sha256:1f188a2402f8359cf0c4b1fe89eea40dc13b52e7b4fd4812450da9fcd210181d",
- "sha256:202a2d645c5a46b84992f55b0a3affe4f0ba6b4c611abec32ee88358db4bb649",
- "sha256:24eda3a24a38157eee639ca9afe45eefa8d2420d49468819ac5f88b10de84f4c",
- "sha256:2e4e0f60cb4bd7396108823548e82fdab72d4d8a65e58e2c19bbbc2f1e2bfa4b",
- "sha256:379c111d3558272a2cae3d8e57e6b6e6f4fe652905692d54bad5ea0ca37c5ad4",
- "sha256:37cda8712145917105e07aab96388ae76e787270ec04bcb9d5cc786d7cbb8443",
- "sha256:38c51297b35b3ed91670e1e4efb702b790002e3245a28c76e627478aa3c10d83",
- "sha256:3985b9be361d8fb6b2d1adc9924d01dec575a1d7453a14cccd73225cb79243ee",
- "sha256:3988665ee376abce49613701336544041f2117de7b7fbfe91b93d8ff8b151c8e",
- "sha256:3ac47fa29d8d41059ea3df65bd3ade92f97ee4910ed638e87075b8e8ce69599e",
- "sha256:3b4b4299dd0d2c67caaaf286d58aef5e75b125b95615dda4542561a5a566a1e3",
- "sha256:3ea8bb1ab9558374c0ab591783808511d135a833c3ca64a18ec927f20c4030f0",
- "sha256:3fe47da3e4fda5f1abb5709c156eca207eacf8007304ce3019eb001e7a7204cb",
- "sha256:428ac484592f780e8cd7b6b14eb568f7c85460c92e2a37cb0c0e5186e1a0d076",
- "sha256:44e6c85bbdc809383b509d732b06419fb4544dca29ebe18480379633623baafb",
- "sha256:4674f0daa1823c295845b6a740d98a840d7a1c11df00d1fd62614545c1583787",
- "sha256:4be32da0c3827ac9132bb488d331cb32e8d9638dd41a0557c5569d57cf22c9c1",
- "sha256:4db3ed6a907b555e57cc2e6f14dc3a4c2458cdad8919e40b5357ab9b6db6c43e",
- "sha256:5c52a036535d12590c32c49209e79cabaad9f9ad8aa4cbd875b68c4d67a9cbce",
- "sha256:629a1ba2115dce8bf75a5cce9f2486ae483cb89c0145795603d6554bdc83e801",
- "sha256:62a66ff235e4c2e37ed3b6104d8b478d767ff73838d1222132a7a026aa548764",
- "sha256:63068a11171e4276f6ece913bde059e77c713b48c3a848814a6537f35afb8365",
- "sha256:63c19702db10ad79151a059d2d6336fe0c470f2e18d0d4d1a57f7f9713875dcf",
- "sha256:644ec81edec0f4ad17d51c838a7d01e42811054543b76d4ba2c5d6af741ce2a6",
- "sha256:6535d996f6537ecb298b4e287a855f37deaf64ff007162ec0afb9ab8ba3b8b71",
- "sha256:6f4548c5ead23ad13fb7a2c8ea541357474ec13c2b736feb02e19a3085fac002",
- "sha256:716a78a342679cd1177bc8c2fe957e0ab91405bd43a17094324845200b2fddf4",
- "sha256:74610105ebd6f33d7c10f8907afed696e79c59e3043c5f20eaa3a46fddf33b4c",
- "sha256:768939f7c4353c0fac2f7c37897e10b1414b571fd85dd9fc49e6a87e37a2e0d8",
- "sha256:86cffe9c6dfcfe22e28027069725c7f57f4b868a3f86e81d1c62462764dc46d4",
- "sha256:8aae5aea53cbfe024919715eca696b1a3201886ce83790537d1c3668459c7146",
- "sha256:8b2b8503edb06822c86d82fa64a4a5cb0760bb8f31f26e138ec743f422f37cfc",
- "sha256:912e95017ff51dc3d7b6e2be158dedc889d9a5cc3382445589ce554f1a34c0ea",
- "sha256:9a7b8ac36fd688c8361cbc7bf1cb5866977ece6e0b17c34aa0df58bda4fa18a4",
- "sha256:9e89d5c8509fbd6c03d0dd1972925b22f50db0792ce06324ba069f10787429ad",
- "sha256:ae270e79f7e169ccfe23284ff5ea2d52a6f401dc01b337efb54b3783e2ce3f28",
- "sha256:b07c25d52b1c16ce5de088046cd2432b30f9ad5e224ff17c8f496d9cb7d1d451",
- "sha256:b39e6011cd06822eb964d038d5dff5da5d98652b81f5ecd439277b32361a3a50",
- "sha256:bd55f8fc8fa494958772a2a7302b0354ab16e0b9272b3c3d83cdb5bec5bd1779",
- "sha256:c15b32a7aca8038ed7644f854bf17b663bc38e1671b5d6f43f9a2b2bd0c46f63",
- "sha256:c1b4474beee02ede1eef86c25ad4600a424fe36cff01a6103cb4533c6bf0169e",
- "sha256:c79c0685f142ca53256722a384540832420dff4ab15fec1863d7e5bc8691bdcc",
- "sha256:c9ebfb2507751f7196995142f057d1324afdab56db1d9743aab7f50289abd022",
- "sha256:d7ad66e8e50225ebf4236368cc43c37f59d5e6728f15f6e258c8639fa0dd8e6d",
- "sha256:d82ab6816c3277dc962cfcdc85b1efa0e5f50fb2c449432deaf2398a2928ab94",
- "sha256:d9fd2547e6decdbf985d579cf3fc78e4c1d662b9b0ff7cc7862baaab71c9cc5b",
- "sha256:de38add67a0af869b0d79c525d3e4588ac1ffa92f39116dbe0ed9753f26eba7d",
- "sha256:e19122296822deafce89a0c5e8685704c067ae65d45e79718c92df7b3ec3d331",
- "sha256:e44961e36cb13c495806d4cac67640ac2866cb99044e210895b506c26ee63d3a",
- "sha256:e4c81ed2820b9023a9a90717020315e63b17b18c274a332e3b6437d7ff70abe0",
- "sha256:e683e6ecc587643f8cde8f5da6768e9d165cd31edf39ee90ed7034f9ca0eefee",
- "sha256:f39e2f3530ed1626c66e7493be7a8423b023ca852aacdc91fb30162c350d2a92",
- "sha256:f56f49b2553d7dd85fd86e029515a221e5c1f8cb3d9c38b470bc38bde7b8445a",
- "sha256:fb9fc32399dca861584d96eccd6c980b69bbcd7c228d06fb74fe53e007aa8ef9"
+ "sha256:0824a28ec542a0be22f60c6ac36d679e0e262e5353203bea81d44ee81fe9c6d4",
+ "sha256:085161be5f3b30fd9b3e7b9a8c301f935c8313dcf928a07b116324abea2c1c2c",
+ "sha256:0ae1387db4aecb1f485fb70a6c0148c6cdaebb6038f1d40089b1fc84a5db556f",
+ "sha256:0d59fd927b1f04de57a2ba0137166d31c1a6dd9e764ad4af552912d70428c92b",
+ "sha256:0f957943bc718b87144ecaee70762bc2bc3f1a7a53c7b861103546d3a403f0a6",
+ "sha256:13a9e2d3ee855db3dd6ea1ba5203316a1b1fd8eaeffc37c5b54987e61e4194ae",
+ "sha256:1a330812d9cc7ac2182586f6d41b4d0fadf9be9049f350e0efb275c8ee8eb692",
+ "sha256:22be16571504c9ccea919fcedb459d5ab20d41172056206eb2994e2ff06118a4",
+ "sha256:2d10e07aa2b91835d6abec555ec8b2733347956991901eea6ffac295f83a30e4",
+ "sha256:35371f8438028fdccfaf3570b31d98e8d9eda8bb1d6ab9473f5a390969e98717",
+ "sha256:3c026eb44f744acaa2bda7493dad903aa5bf5fc4f2554293a798d5606710055d",
+ "sha256:41ff7b0da5af71a51b53f501a3bac65fb0ec311ebed1632e58fc6107f03b9198",
+ "sha256:4401ae5fc52ad8d26d2a5d8a7428b0f0c72431683f8e63e42e70606374c311a1",
+ "sha256:44349150f6811b44b25574839b39ae35291f6496eb795b7366fef3bd3cf112d3",
+ "sha256:447af20e25fdbe16f26e84eb714ba21d98868705cb138252d28bc400381f6ffb",
+ "sha256:4a8d8977b0c6ef5aeadcb644da9e69ae0dcfe66ec7f368c89c72e058bd71164d",
+ "sha256:4e12ae8cc979cf83d258acb5e1f1cf2f3f83524d1564a49d20b8bec14b637f08",
+ "sha256:592ac539812e9b46046620341498caf09ca21023c41c893e1eb9dbda00a70cbf",
+ "sha256:5e6b86b5847a016d0fbd31ffe1001b63355ed309651851295315031ea7eb5a9b",
+ "sha256:608a7fd78c67bee8936378299a6cb9f5149bb80238c7a566fc3e6717a4e68710",
+ "sha256:61f70dc68bd36810972e55bbbe83674ea073dd1dcc121040a08cdf3416c5349c",
+ "sha256:65dad5a248823a4996724a88eb51d4b31587aa7aa428562dbe459c684e5787ae",
+ "sha256:777abfab476cf83b5177b84d7486497e034eb9eaea0d746ce0c1268c71652077",
+ "sha256:7e216d8044a356fc0337c7a2a0536d6de07888d7bcda76febcb8adc50bdbbd00",
+ "sha256:85d9636f72e8991a1706b2b55b06c27545448baf9f6dbf51c4004609aacd7dcb",
+ "sha256:899b8cd4781c400454f2f64f7776a5d87bbd7b3e7f7bda0cb18f857bb1334664",
+ "sha256:8a289d23d4c46f1a82d5db4abeb40b9b5be91731ee19a379d15790e53031c014",
+ "sha256:8d2dfa71665a29b153a9681edb1c8d9c1ea50dfc2375fb4dac99ea7e21a0bcd9",
+ "sha256:8e3c3e38930cfb729cb8137d7f055e5a473ddaf1217966aa6238c88bd9fd50e6",
+ "sha256:8f8770dfc6e2c6a2d4569f411015c8d751c980d17a14b0530da2d7f27ffdd88e",
+ "sha256:932fc826442132dde42ee52cf66d941f581c685a6313feebed358411238f60f9",
+ "sha256:96d636c77af18b5cb664ddf12dab9b15a0cfe9c0bde715da38698c8cea748bfa",
+ "sha256:97ddc94d46088304772d21b060041c97fc16bdda13c6c7f9d8fcd8d5ae0d8611",
+ "sha256:98caba4476a6c8d59ec1eb00c7dd862ba9beca34085642d46ed503cc2d440d4b",
+ "sha256:9901d36492009a0a9b94b20e52ebfc8453bf49bb2b27bca2c9706f8b4f5a554a",
+ "sha256:99e266ae0b5d15f1ca8d278a668df6f51cc4b854513daab5cae695ed7b721cf8",
+ "sha256:9c38bf15a40ccf5619fa2fe8f26106c7e8e080d7760aeccb3722664c8656b030",
+ "sha256:a27801adef24cc30871da98a105f77995e13a25a505a0161911f6aafbd66e678",
+ "sha256:abd3e72dd5b97e3af4246cdada7738ef0e608168de952b837b8dd7e90341f015",
+ "sha256:adb697c0bd35100dc690de83154627fbab1f4f3c0386df266dded865fc50a902",
+ "sha256:b12c6b18269ca471eedd41c1b6a1065b2f7827508edb9a7ed5555e9a56dcfc97",
+ "sha256:b9389a429e0e5142e69d5bf4a435dd688c14478a19bb901735cdf75e57b13845",
+ "sha256:ba9e7484d286cd5a43744e5f47b0b3fb457865baf07bafc6bee91896364e1419",
+ "sha256:bb5555cff66c4d3d6213a296b360f9e1a8e323e74e0426b6c10ed7f4d021e464",
+ "sha256:be57b6d56e49c2739cdf776839a92330e933dd5e5d929966fbbd380c77f060be",
+ "sha256:c69e42c892c018cd3c8d90da61d845f50a8243062b19d228189b0224150018a9",
+ "sha256:ccc660a77e1c2bf24ddbce969af9447a9474790160cfb23de6be4fa88e3951c7",
+ "sha256:d5275455b3e4627c8e7154feaf7ee0743c2e7af82f6e3b561967b1cca755a0be",
+ "sha256:d75cded8a3cff93da9edc31446872d2997e327921d8eed86641efafd350e1df1",
+ "sha256:d872ec5aeb086cbea771c573600d47944eea2dcba8be5f3ee649bfe3cb8dc9ba",
+ "sha256:d891c136b5b310d0e702e186d70cd16d1119ea8927347045124cb286b29297e5",
+ "sha256:db1dab894cc139f67822a92910466531de5ea6034ddfd2b11c0d4c6257168073",
+ "sha256:e28bf44afa2b187cc9f41749138a64435bf340adfcacb5b2290c070ce99839d4",
+ "sha256:e5ea1cf0872ee455c03e5674b5bca5e3e68e159379c1af0903e89f5eba9ccc3a",
+ "sha256:e77363e8425325384f9d49272c54045bbed2f478e9dd698dbc65dbc37860eb0a",
+ "sha256:ee5defd1733fd6ec08b168bd4f5387d5b322f45ca9e0e6c817ea6c4cd36313e3",
+ "sha256:f1592791f8204ae9166de22ba7e6705fa4ebd02936c09436a1bb85aabca3e599",
+ "sha256:f2d1ec60d6d256bdf298cb86b78dd715980828f50c46701abc3b0a2b3f8a0dc0",
+ "sha256:f3ca78518bc6bc92828cd11867b121891d75cae4ea9e908d72030609b996db1b",
+ "sha256:f7b15f589593110ae767ce997775d645b47e5cbbf54fd322f8ebea6277466cec",
+ "sha256:fd1213c86e48dfdc5a0cc676551db467495a95a662d2396ecd58e719191446e1",
+ "sha256:ff74026a461eb0660366fb01c650c1d00f833a086b336bdad7ab00cc952072b3"
],
"markers": "python_version >= '3.9'",
- "version": "==7.6.8"
+ "version": "==7.6.9"
},
"coveralls": {
"hashes": [
@@ -676,7 +685,6 @@
"sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b",
"sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc",
"sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543",
- "sha256:60eb32934076fa07e4316b7b2742fa52cbb190b42c2df2863dbc4230a0a9b385",
"sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c",
"sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591",
"sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede",
@@ -684,7 +692,6 @@
"sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f",
"sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123",
"sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c",
- "sha256:9abcc2e083cbe8dde89124a47e5e53ec38751f0d7dfd36801008f316a127d7ba",
"sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c",
"sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285",
"sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd",
@@ -923,10 +930,10 @@
},
"mypy-boto3-s3": {
"hashes": [
- "sha256:571e659c1d355499d5e5070f33e613a1e251e6f5d2a57d535c5eaef52ebb6a86",
- "sha256:b2a18ca57079659eb602dcfc4abb56425c793ccb1939826e401d4f2ddf9128b0"
+ "sha256:34ac4cacf8acdafa6e71a2810116b2546376f241761f9eec6ac5a9887309372b",
+ "sha256:fd4a8734c3bb5a2da52e22258b1836a14aa3460816df25c831790e464334021f"
],
- "version": "==1.35.72"
+ "version": "==1.35.76.post1"
},
"mypy-extensions": {
"hashes": [
@@ -946,64 +953,64 @@
},
"numpy": {
"hashes": [
- "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe",
- "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0",
- "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48",
- "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a",
- "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564",
- "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958",
- "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17",
- "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0",
- "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee",
- "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b",
- "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4",
- "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4",
- "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6",
- "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4",
- "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d",
- "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f",
- "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f",
- "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f",
- "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56",
- "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9",
- "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd",
- "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23",
- "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed",
- "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a",
- "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098",
- "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1",
- "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512",
- "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f",
- "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09",
- "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f",
- "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc",
- "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8",
- "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0",
- "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761",
- "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef",
- "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5",
- "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e",
- "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b",
- "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d",
- "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43",
- "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c",
- "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41",
- "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff",
- "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408",
- "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2",
- "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9",
- "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57",
- "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb",
- "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9",
- "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3",
- "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a",
- "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0",
- "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e",
- "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598",
- "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4"
+ "sha256:0557eebc699c1c34cccdd8c3778c9294e8196df27d713706895edc6f57d29608",
+ "sha256:0798b138c291d792f8ea40fe3768610f3c7dd2574389e37c3f26573757c8f7ef",
+ "sha256:0da8495970f6b101ddd0c38ace92edea30e7e12b9a926b57f5fabb1ecc25bb90",
+ "sha256:0f0986e917aca18f7a567b812ef7ca9391288e2acb7a4308aa9d265bd724bdae",
+ "sha256:122fd2fcfafdefc889c64ad99c228d5a1f9692c3a83f56c292618a59aa60ae83",
+ "sha256:140dd80ff8981a583a60980be1a655068f8adebf7a45a06a6858c873fcdcd4a0",
+ "sha256:16757cf28621e43e252c560d25b15f18a2f11da94fea344bf26c599b9cf54b73",
+ "sha256:18142b497d70a34b01642b9feabb70156311b326fdddd875a9981f34a369b671",
+ "sha256:1c92113619f7b272838b8d6702a7f8ebe5edea0df48166c47929611d0b4dea69",
+ "sha256:1e25507d85da11ff5066269d0bd25d06e0a0f2e908415534f3e603d2a78e4ffa",
+ "sha256:30bf971c12e4365153afb31fc73f441d4da157153f3400b82db32d04de1e4066",
+ "sha256:3579eaeb5e07f3ded59298ce22b65f877a86ba8e9fe701f5576c99bb17c283da",
+ "sha256:36b2b43146f646642b425dd2027730f99bac962618ec2052932157e213a040e9",
+ "sha256:3905a5fffcc23e597ee4d9fb3fcd209bd658c352657548db7316e810ca80458e",
+ "sha256:3a4199f519e57d517ebd48cb76b36c82da0360781c6a0353e64c0cac30ecaad3",
+ "sha256:3f2f5cddeaa4424a0a118924b988746db6ffa8565e5829b1841a8a3bd73eb59a",
+ "sha256:40deb10198bbaa531509aad0cd2f9fadb26c8b94070831e2208e7df543562b74",
+ "sha256:440cfb3db4c5029775803794f8638fbdbf71ec702caf32735f53b008e1eaece3",
+ "sha256:4723a50e1523e1de4fccd1b9a6dcea750c2102461e9a02b2ac55ffeae09a4410",
+ "sha256:4bddbaa30d78c86329b26bd6aaaea06b1e47444da99eddac7bf1e2fab717bd72",
+ "sha256:4e58666988605e251d42c2818c7d3d8991555381be26399303053b58a5bbf30d",
+ "sha256:54dc1d6d66f8d37843ed281773c7174f03bf7ad826523f73435deb88ba60d2d4",
+ "sha256:57fcc997ffc0bef234b8875a54d4058afa92b0b0c4223fc1f62f24b3b5e86038",
+ "sha256:58b92a5828bd4d9aa0952492b7de803135038de47343b2aa3cc23f3b71a3dc4e",
+ "sha256:5a145e956b374e72ad1dff82779177d4a3c62bc8248f41b80cb5122e68f22d13",
+ "sha256:6ab153263a7c5ccaf6dfe7e53447b74f77789f28ecb278c3b5d49db7ece10d6d",
+ "sha256:7832f9e8eb00be32f15fdfb9a981d6955ea9adc8574c521d48710171b6c55e95",
+ "sha256:7fe4bb0695fe986a9e4deec3b6857003b4cfe5c5e4aac0b95f6a658c14635e31",
+ "sha256:7fe8f3583e0607ad4e43a954e35c1748b553bfe9fdac8635c02058023277d1b3",
+ "sha256:85ad7d11b309bd132d74397fcf2920933c9d1dc865487128f5c03d580f2c3d03",
+ "sha256:9874bc2ff574c40ab7a5cbb7464bf9b045d617e36754a7bc93f933d52bd9ffc6",
+ "sha256:a184288538e6ad699cbe6b24859206e38ce5fba28f3bcfa51c90d0502c1582b2",
+ "sha256:a222d764352c773aa5ebde02dd84dba3279c81c6db2e482d62a3fa54e5ece69b",
+ "sha256:a50aeff71d0f97b6450d33940c7181b08be1441c6c193e678211bff11aa725e7",
+ "sha256:a55dc7a7f0b6198b07ec0cd445fbb98b05234e8b00c5ac4874a63372ba98d4ab",
+ "sha256:a62eb442011776e4036af5c8b1a00b706c5bc02dc15eb5344b0c750428c94219",
+ "sha256:a7d41d1612c1a82b64697e894b75db6758d4f21c3ec069d841e60ebe54b5b571",
+ "sha256:a98f6f20465e7618c83252c02041517bd2f7ea29be5378f09667a8f654a5918d",
+ "sha256:afe8fb968743d40435c3827632fd36c5fbde633b0423da7692e426529b1759b1",
+ "sha256:b0b227dcff8cdc3efbce66d4e50891f04d0a387cce282fe1e66199146a6a8fca",
+ "sha256:b30042fe92dbd79f1ba7f6898fada10bdaad1847c44f2dff9a16147e00a93661",
+ "sha256:b606b1aaf802e6468c2608c65ff7ece53eae1a6874b3765f69b8ceb20c5fa78e",
+ "sha256:b6207dc8fb3c8cb5668e885cef9ec7f70189bec4e276f0ff70d5aa078d32c88e",
+ "sha256:c2aed8fcf8abc3020d6a9ccb31dbc9e7d7819c56a348cc88fd44be269b37427e",
+ "sha256:cb24cca1968b21355cc6f3da1a20cd1cebd8a023e3c5b09b432444617949085a",
+ "sha256:cff210198bb4cae3f3c100444c5eaa573a823f05c253e7188e1362a5555235b3",
+ "sha256:d35717333b39d1b6bb8433fa758a55f1081543de527171543a2b710551d40881",
+ "sha256:df12a1f99b99f569a7c2ae59aa2d31724e8d835fc7f33e14f4792e3071d11221",
+ "sha256:e09d40edfdb4e260cb1567d8ae770ccf3b8b7e9f0d9b5c2a9992696b30ce2742",
+ "sha256:e12c6c1ce84628c52d6367863773f7c8c8241be554e8b79686e91a43f1733773",
+ "sha256:e2b8cd48a9942ed3f85b95ca4105c45758438c7ed28fff1e4ce3e57c3b589d8e",
+ "sha256:e500aba968a48e9019e42c0c199b7ec0696a97fa69037bea163b55398e390529",
+ "sha256:ebe5e59545401fbb1b24da76f006ab19734ae71e703cdb4a8b347e84a0cece67",
+ "sha256:f0dd071b95bbca244f4cb7f70b77d2ff3aaaba7fa16dc41f58d14854a6204e6c",
+ "sha256:f8c8b141ef9699ae777c6278b52c706b653bf15d135d302754f6b2e90eb30367"
],
"markers": "python_version >= '3.12'",
- "version": "==2.1.3"
+ "version": "==2.2.0"
},
"packaging": {
"hashes": [
@@ -1144,12 +1151,12 @@
},
"pyarrow-stubs": {
"hashes": [
- "sha256:6f4a32a14dd5526851830cfb9e91223019d46c192c15ae8d951f762114b73c1e",
- "sha256:d58d85d299dc2b1f16994c1fcf2e0f3ab28ec211876233756b9f6534e43a2b39"
+ "sha256:624b8f47ed207075faeb7a475d87178145adbd709d8feae93d24574b97d9cb42",
+ "sha256:af9433bf1daa27df8e5a6364d5fe7db9feeb203394686c0f01821d52bc571ee0"
],
"index": "pypi",
"markers": "python_version >= '3.8' and python_version < '4'",
- "version": "==17.12"
+ "version": "==17.13"
},
"pycparser": {
"hashes": [
@@ -1270,28 +1277,28 @@
},
"ruff": {
"hashes": [
- "sha256:2029b8c22da147c50ae577e621a5bfbc5d1fed75d86af53643d7a7aee1d23871",
- "sha256:2666520828dee7dfc7e47ee4ea0d928f40de72056d929a7c5292d95071d881d1",
- "sha256:288326162804f34088ac007139488dcb43de590a5ccfec3166396530b58fb89d",
- "sha256:2954cdbe8dfd8ab359d4a30cd971b589d335a44d444b6ca2cb3d1da21b75e4b6",
- "sha256:333c57013ef8c97a53892aa56042831c372e0bb1785ab7026187b7abd0135ad5",
- "sha256:3583db9a6450364ed5ca3f3b4225958b24f78178908d5c4bc0f46251ccca898f",
- "sha256:364e6674450cbac8e998f7b30639040c99d81dfb5bbc6dfad69bc7a8f916b3d1",
- "sha256:55873cc1a473e5ac129d15eccb3c008c096b94809d693fc7053f588b67822737",
- "sha256:93335cd7c0eaedb44882d75a7acb7df4b77cd7cd0d2255c93b28791716e81790",
- "sha256:a885d68342a231b5ba4d30b8c6e1b1ee3a65cf37e3d29b3c74069cdf1ee1e3c9",
- "sha256:adf314fc458374c25c5c4a4a9270c3e8a6a807b1bec018cfa2813d6546215540",
- "sha256:b12c39b9448632284561cbf4191aa1b005882acbc81900ffa9f9f471c8ff7e26",
- "sha256:b22346f845fec132aa39cd29acb94451d030c10874408dbf776af3aaeb53284c",
- "sha256:b2f2f7a7e7648a2bfe6ead4e0a16745db956da0e3a231ad443d2a66a105c04fa",
- "sha256:b8a4f7385c2285c30f34b200ca5511fcc865f17578383db154e098150ce0a087",
- "sha256:cd054486da0c53e41e0086e1730eb77d1f698154f910e0cd9e0d64274979a209",
- "sha256:d2c16e3508c8cc73e96aa5127d0df8913d2290098f776416a4b157657bee44c5",
- "sha256:fae0805bd514066f20309f6742f6ee7904a773eb9e6c17c45d6b1600ca65c9b5"
+ "sha256:1ca4e3a87496dc07d2427b7dd7ffa88a1e597c28dad65ae6433ecb9f2e4f022f",
+ "sha256:2aae99ec70abf43372612a838d97bfe77d45146254568d94926e8ed5bbb409ea",
+ "sha256:32096b41aaf7a5cc095fa45b4167b890e4c8d3fd217603f3634c92a541de7248",
+ "sha256:5fe716592ae8a376c2673fdfc1f5c0c193a6d0411f90a496863c99cd9e2ae25d",
+ "sha256:60f578c11feb1d3d257b2fb043ddb47501ab4816e7e221fbb0077f0d5d4e7b6f",
+ "sha256:705832cd7d85605cb7858d8a13d75993c8f3ef1397b0831289109e953d833d29",
+ "sha256:729850feed82ef2440aa27946ab39c18cb4a8889c1128a6d589ffa028ddcfc22",
+ "sha256:81c148825277e737493242b44c5388a300584d73d5774defa9245aaef55448b0",
+ "sha256:ac42caaa0411d6a7d9594363294416e0e48fc1279e1b0e948391695db2b3d5b1",
+ "sha256:b402ddee3d777683de60ff76da801fa7e5e8a71038f57ee53e903afbcefdaa58",
+ "sha256:b84f4f414dda8ac7f75075c1fa0b905ac0ff25361f42e6d5da681a465e0f78e5",
+ "sha256:c49ab4da37e7c457105aadfd2725e24305ff9bc908487a9bf8d548c6dad8bb3d",
+ "sha256:cbd5cf9b0ae8f30eebc7b360171bd50f59ab29d39f06a670b3e4501a36ba5897",
+ "sha256:d261d7850c8367704874847d95febc698a950bf061c9475d4a8b7689adc4f7fa",
+ "sha256:e769083da9439508833cfc7c23e351e1809e67f47c50248250ce1ac52c21fb93",
+ "sha256:ec016beb69ac16be416c435828be702ee694c0d722505f9c1f35e1b9c0cc1bf5",
+ "sha256:f05cdf8d050b30e2ba55c9b09330b51f9f97d36d4673213679b965d25a785f3c",
+ "sha256:fb88e2a506b70cfbc2de6fae6681c4f944f7dd5f2fe87233a7233d888bad73e8"
],
"index": "pypi",
"markers": "python_version >= '3.7'",
- "version": "==0.8.1"
+ "version": "==0.8.2"
},
"s3transfer": {
"hashes": [
@@ -1312,11 +1319,11 @@
},
"six": {
"hashes": [
- "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
- "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+ "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274",
+ "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==1.16.0"
+ "version": "==1.17.0"
},
"stack-data": {
"hashes": [
@@ -1335,11 +1342,20 @@
},
"types-awscrt": {
"hashes": [
- "sha256:0d362a5d62d68ca4216f458172f41c1123ec04791d68364de8ee8b61b528b262",
- "sha256:a20b425dabb258bc3d07a5e7de503fd9558dd1542d72de796e74e402c6d493b2"
+ "sha256:b1b9bb10f337e3fe8f5f508860eb354d9fe093f02e1485955a9e0bdd4e250074",
+ "sha256:eeb4bd596100927704c8b9f964ec8a246be4943d546f3fd2a8efdddebea422ea"
],
"markers": "python_version >= '3.8'",
- "version": "==0.23.1"
+ "version": "==0.23.4"
+ },
+ "types-python-dateutil": {
+ "hashes": [
+ "sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb",
+ "sha256:e248a4bc70a486d3e3ec84d0dc30eec3a5f979d6e7ee4123ae043eedbb987f53"
+ ],
+ "index": "pypi",
+ "markers": "python_version >= '3.8'",
+ "version": "==2.9.0.20241206"
},
"types-pytz": {
"hashes": [
diff --git a/tests/conftest.py b/tests/conftest.py
index 84dbfc0..e2da0a3 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -51,12 +51,7 @@ def sample_records_iter_without_partitions():
def _records_iter(num_records):
return generate_sample_records(
- num_records,
- source=None,
- run_date=None,
- run_type=None,
- action=None,
- run_id=None,
+ num_records, run_date="invalid run-date", year=None, month=None, day=None
)
return _records_iter
diff --git a/tests/test_dataset_write.py b/tests/test_dataset_write.py
index 3852544..f8c1e2d 100644
--- a/tests/test_dataset_write.py
+++ b/tests/test_dataset_write.py
@@ -1,23 +1,24 @@
# ruff: noqa: S105, S106, SLF001, PLR2004, PD901, D209, D205
-
-import datetime
import math
import os
+import re
+from datetime import UTC, datetime
+from unittest.mock import patch
import pyarrow.dataset as ds
import pytest
+from tests.utils import generate_sample_records
from timdex_dataset_api.dataset import (
MAX_ROWS_PER_FILE,
TIMDEX_DATASET_SCHEMA,
DatasetNotLoadedError,
TIMDEXDataset,
)
-from timdex_dataset_api.exceptions import InvalidDatasetRecordError
from timdex_dataset_api.record import DatasetRecord
-def test_dataset_record_serialization():
+def test_dataset_record_init():
values = {
"timdex_record_id": "alma:123",
"source_record": b"Hello World.",
@@ -26,38 +27,35 @@ def test_dataset_record_serialization():
"run_date": "2024-12-01",
"run_type": "full",
"action": "index",
- "run_id": "abc123",
+ "run_id": "000-111-aaa-bbb",
}
- dataset_record = DatasetRecord(**values)
- assert dataset_record.to_dict() == values
+ record = DatasetRecord(**values)
+ assert record
+ assert (record.year, record.month, record.day) == (
+ "2024",
+ "12",
+ "01",
+ )
-def test_dataset_record_serialization_with_partition_values_provided():
- dataset_record = DatasetRecord(
- timdex_record_id="alma:123",
- source_record=b"Hello World.",
- transformed_record=b"""{"title":["Hello World."]}""",
- )
- partition_values = {
- "source": "alma",
- "run_date": "2024-12-01",
- "run_type": "daily",
- "action": "index",
- "run_id": "000-111-aaa-bbb",
- }
- assert dataset_record.to_dict(partition_values=partition_values) == {
+def test_dataset_record_init_with_invalid_run_date_raise_error():
+ values = {
"timdex_record_id": "alma:123",
"source_record": b"Hello World.",
"transformed_record": b"""{"title":["Hello World."]}""",
- "source": "alma",
- "run_date": "2024-12-01",
- "run_type": "daily",
+ "source": "libguides",
+ "run_date": "-12-01",
+ "run_type": "full",
"action": "index",
"run_id": "000-111-aaa-bbb",
}
+ with pytest.raises(
+ ValueError, match=re.escape("time data '-12-01' does not match format '%Y-%m-%d'")
+ ):
+ DatasetRecord(**values)
-def test_dataset_record_serialization_missing_partition_raise_error():
+def test_dataset_record_serialization():
values = {
"timdex_record_id": "alma:123",
"source_record": b"Hello World.",
@@ -66,14 +64,22 @@ def test_dataset_record_serialization_missing_partition_raise_error():
"run_date": "2024-12-01",
"run_type": "full",
"action": "index",
- "run_id": None, # <------ missing partition here
+ "run_id": "abc123",
}
dataset_record = DatasetRecord(**values)
- with pytest.raises(
- InvalidDatasetRecordError,
- match="Partition values are missing: run_id",
- ):
- assert dataset_record.to_dict() == values
+ assert dataset_record.to_dict() == {
+ "timdex_record_id": "alma:123",
+ "source_record": b"Hello World.",
+ "transformed_record": b"""{"title":["Hello World."]}""",
+ "source": "libguides",
+ "run_date": datetime(2024, 12, 1).astimezone(UTC),
+ "run_type": "full",
+ "action": "index",
+ "run_id": "abc123",
+ "year": "2024",
+ "month": "12",
+ "day": "01",
+ }
def test_dataset_write_records_to_new_dataset(new_dataset, sample_records_iter):
@@ -134,52 +140,6 @@ def test_dataset_write_to_multiple_locations_raise_error(sample_records_iter):
timdex_dataset.write(sample_records_iter(10))
-def test_dataset_write_mixin_partition_values_used(
- new_dataset, sample_records_iter_without_partitions
-):
- partition_values = {
- "source": "alma",
- "run_date": "2024-12-01",
- "run_type": "daily",
- "action": "index",
- "run_id": "000-111-aaa-bbb",
- }
- _written_files = new_dataset.write(
- sample_records_iter_without_partitions(10),
- partition_values=partition_values,
- )
- new_dataset.reload()
-
- # load as pandas dataframe and assert column values
- df = new_dataset.dataset.to_table().to_pandas()
- row = df.iloc[0]
- assert row.source == partition_values["source"]
- assert row.run_date == datetime.date(2024, 12, 1)
- assert row.run_type == partition_values["run_type"]
- assert row.action == partition_values["action"]
- assert row.action == partition_values["action"]
-
-
-def test_dataset_write_schema_partitions_correctly_ordered(
- new_dataset, sample_records_iter
-):
- written_files = new_dataset.write(
- sample_records_iter(10),
- partition_values={
- "source": "alma",
- "run_date": "2024-12-01",
- "run_type": "daily",
- "run_id": "000-111-aaa-bbb",
- "action": "index",
- },
- )
- file = written_files[0]
- assert (
- "/source=alma/run_date=2024-12-01/run_type=daily"
- "/run_id=000-111-aaa-bbb/action=index/" in file.path
- )
-
-
def test_dataset_write_schema_applied_to_dataset(new_dataset, sample_records_iter):
new_dataset.write(sample_records_iter(10))
@@ -194,67 +154,63 @@ def test_dataset_write_schema_applied_to_dataset(new_dataset, sample_records_ite
assert set(dataset.schema.names) == set(TIMDEX_DATASET_SCHEMA.names)
-def test_dataset_write_partition_deleted_when_written_to_again(
- new_dataset, sample_records_iter
-):
- """This tests the existing_data_behavior="delete_matching" configuration when writing
- to a dataset."""
- partition_values = {
- "source": "alma",
- "run_date": "2024-12-01",
- "run_type": "daily",
- "action": "index",
- "run_id": "000-111-aaa-bbb",
- }
+def test_dataset_write_partition_for_single_source(new_dataset, sample_records_iter):
+ written_files = new_dataset.write(sample_records_iter(10))
+ assert len(written_files) == 1
+ assert os.path.exists(new_dataset.location)
+ assert "year=2024/month=12/day=01" in written_files[0].path
- # perform FIRST write to run_date="2024-12-01"
- written_files_1 = new_dataset.write(
- sample_records_iter(10),
- partition_values=partition_values,
- )
- # assert that files from first write are present at this time
- assert os.path.exists(written_files_1[0].path)
+def test_dataset_write_partition_for_multiple_sources(new_dataset, sample_records_iter):
+ # perform write for source="alma" and run_date="2024-12-01"
+ written_files_source_a = new_dataset.write(sample_records_iter(10))
+ new_dataset.reload()
- # perform unrelated write with new run_date to confirm this is untouched during delete
- new_partition_values = partition_values.copy()
- new_partition_values["run_date"] = "2024-12-15"
- new_partition_values["run_id"] = "222-333-ccc-ddd"
- written_files_x = new_dataset.write(
- sample_records_iter(7),
- partition_values=new_partition_values,
- )
+ assert os.path.exists(written_files_source_a[0].path)
+ assert new_dataset.row_count == 10
- # perform SECOND write to run_date="2024-12-01", expecting this to delete everything
- # under this combination of partitions (i.e. the first write)
- written_files_2 = new_dataset.write(
- sample_records_iter(10),
- partition_values=partition_values,
+ # perform write for source="libguides" and run_date="2024-12-01"
+ written_files_source_b = new_dataset.write(
+ generate_sample_records(
+ num_records=7, timdex_record_id_prefix="libguides", source="libguides"
+ )
)
-
new_dataset.reload()
- # assert 17 rows: second write for run_date="2024-12-01" @ 10 rows +
- # run_date="2024-12-15" @ 5 rows
+ assert os.path.exists(written_files_source_b[0].path)
+ assert os.path.exists(written_files_source_a[0].path)
assert new_dataset.row_count == 17
- # assert that files from first run_date="2024-12-01" are gone, second exist
- # and files from run_date="2024-12-15" also exist
- assert not os.path.exists(written_files_1[0].path)
- assert os.path.exists(written_files_2[0].path)
- assert os.path.exists(written_files_x[0].path)
+def test_dataset_write_partition_ignore_existing_data(new_dataset, sample_records_iter):
+ # perform two (2) writes for source="alma" and run_date="2024-12-01"
+ written_files_source_a0 = new_dataset.write(sample_records_iter(10))
+ written_files_source_a1 = new_dataset.write(sample_records_iter(10))
+ new_dataset.reload()
-def test_dataset_write_missing_partitions_raise_error(new_dataset, sample_records_iter):
- missing_partition_values = {
- "source": "libguides",
- "run_date": None,
- "run_type": None,
- "action": None,
- "run_id": None,
- }
- with pytest.raises(InvalidDatasetRecordError, match="Partition values are missing"):
- _ = new_dataset.write(
- sample_records_iter(10),
- partition_values=missing_partition_values,
- )
+ # assert that both files exist and no overwriting occurs
+ assert os.path.exists(written_files_source_a0[0].path)
+ assert os.path.exists(written_files_source_a1[0].path)
+ assert new_dataset.row_count == 20
+
+
+@patch("timdex_dataset_api.dataset.uuid.uuid4")
+def test_dataset_write_partition_overwrite_files_with_same_name(
+ mock_uuid, new_dataset, sample_records_iter
+):
+ """This test is to demonstrate existing_data_behavior="overwrite_or_ignore".
+
+ It is extremely unlikely for the uuid.uuid4 method to generate duplicate values,
+ so for testing purposes, this method is patched to return the same value
+ and therefore generate similarly named files.
+ """
+ mock_uuid.return_value = "abc"
+
+ # perform two (2) writes for source="alma" and run_date="2024-12-01"
+ _ = new_dataset.write(sample_records_iter(10))
+ written_files_source_a1 = new_dataset.write(sample_records_iter(7))
+ new_dataset.reload()
+
+ # assert that only the second file exists and overwriting occurs
+ assert os.path.exists(written_files_source_a1[0].path)
+ assert new_dataset.row_count == 7
diff --git a/timdex_dataset_api/__init__.py b/timdex_dataset_api/__init__.py
index a8d3cad..059018f 100644
--- a/timdex_dataset_api/__init__.py
+++ b/timdex_dataset_api/__init__.py
@@ -3,7 +3,7 @@
from timdex_dataset_api.dataset import TIMDEXDataset
from timdex_dataset_api.record import DatasetRecord
-__version__ = "0.2.0"
+__version__ = "0.3.0"
__all__ = [
"DatasetRecord",
diff --git a/timdex_dataset_api/dataset.py b/timdex_dataset_api/dataset.py
index 50ebdad..4ce10b4 100644
--- a/timdex_dataset_api/dataset.py
+++ b/timdex_dataset_api/dataset.py
@@ -1,6 +1,5 @@
"""timdex_dataset_api/dataset.py"""
-import datetime
import itertools
import time
import uuid
@@ -30,15 +29,16 @@
pa.field("run_type", pa.string()),
pa.field("run_id", pa.string()),
pa.field("action", pa.string()),
+ pa.field("year", pa.string()),
+ pa.field("month", pa.string()),
+ pa.field("day", pa.string()),
)
)
TIMDEX_DATASET_PARTITION_COLUMNS = [
- "source",
- "run_date",
- "run_type",
- "run_id",
- "action",
+ "year",
+ "month",
+ "day",
]
DEFAULT_BATCH_SIZE = 1_000
@@ -166,25 +166,22 @@ def write(
self,
records_iter: Iterator["DatasetRecord"],
*,
- partition_values: dict[str, str | datetime.datetime] | None = None,
batch_size: int = DEFAULT_BATCH_SIZE,
use_threads: bool = True,
) -> list[ds.WrittenFile]:
"""Write records to the TIMDEX parquet dataset.
- This method expects an iterator of DatasetRecord instances, with optional
- partition column values that will be applied to all rows written (often, these
- are the same for all rows written, eliminating the need to repeat those values
- in the iterator).
+ This method expects an iterator of DatasetRecord instances.
This method encapsulates all dataset writing mechanics and performance
optimizations (e.g. batching) so that the calling context can focus on yielding
data.
- For write, the configuration existing_data_behavior="delete_matching" is used.
- This means that during write, if any pre-existing files are found for the exact
- combinations of partitions for that batch, those pre-existing files will be
- deleted. This effectively makes a write idempotent to the TIMDEX dataset.
+ This method uses the configuration existing_data_behavior="overwrite_or_ignore",
+ which will ignore any existing data and will overwrite files with the same name
+ as the parquet file. Since a UUID is generated for each write via the
+ basename_template, this effectively makes a write idempotent to the
+ TIMDEX dataset.
A max_open_files=500 configuration is set to avoid AWS S3 503 error "SLOW_DOWN"
if too many PutObject calls are made in parallel. Testing suggests this does not
@@ -192,7 +189,6 @@ def write(
Args:
- records_iter: Iterator of DatasetRecord instances
- - partition_values: dictionary of static partition column name/value pairs
- batch_size: size for batches to yield and write, directly affecting row
group size in final parquet files
- use_threads: boolean if threads should be used for writing
@@ -207,7 +203,6 @@ def write(
record_batches_iter = self.get_dataset_record_batches(
records_iter,
- partition_values=partition_values,
batch_size=batch_size,
)
@@ -215,7 +210,7 @@ def write(
record_batches_iter,
base_dir=self.source,
basename_template="%s-{i}.parquet" % (str(uuid.uuid4())), # noqa: UP031
- existing_data_behavior="delete_matching",
+ existing_data_behavior="overwrite_or_ignore",
filesystem=self.filesystem,
file_visitor=lambda written_file: self._written_files.append(written_file), # type: ignore[arg-type]
format="parquet",
@@ -235,32 +230,24 @@ def get_dataset_record_batches(
self,
records_iter: Iterator["DatasetRecord"],
*,
- partition_values: dict[str, str | datetime.datetime] | None = None,
batch_size: int = DEFAULT_BATCH_SIZE,
) -> Iterator[pa.RecordBatch]:
"""Yield pyarrow.RecordBatches for writing.
- This method expects an iterator of DatasetRecord instances, with optional
- partition column values that will be applied to all rows written (often, these
- are the same for all rows written, eliminating the need to repeat those values
- in the iterator).
+ This method expects an iterator of DatasetRecord instances.
Each DatasetRecord is validated and serialized to a dictionary before added to a
pyarrow.RecordBatch for writing.
Args:
- records_iter: Iterator of DatasetRecord instances
- - partition_values: dictionary of static partition column name/value pairs
- batch_size: size for batches to yield and write, directly affecting row
group size in final parquet files
"""
for i, record_batch in enumerate(itertools.batched(records_iter, batch_size)):
batch_start_time = time.perf_counter()
batch = pa.RecordBatch.from_pylist(
- [
- record.to_dict(partition_values=partition_values)
- for record in record_batch
- ]
+ [record.to_dict() for record in record_batch]
)
logger.debug(
f"Batch {i + 1} yielded for writing, "
diff --git a/timdex_dataset_api/record.py b/timdex_dataset_api/record.py
index ddc2b3d..96b5a38 100644
--- a/timdex_dataset_api/record.py
+++ b/timdex_dataset_api/record.py
@@ -1,12 +1,15 @@
"""timdex_dataset_api/record.py"""
-import datetime
-from dataclasses import asdict, dataclass
+from datetime import UTC, datetime
-from timdex_dataset_api.exceptions import InvalidDatasetRecordError
+from attrs import asdict, define, field
-@dataclass
+def strict_date_parse(date_string: str) -> datetime:
+ return datetime.strptime(date_string, "%Y-%m-%d").astimezone(UTC)
+
+
+@define
class DatasetRecord:
"""Container for single dataset record.
@@ -16,40 +19,34 @@ class DatasetRecord:
"""
# primary columns
- timdex_record_id: str
- source_record: bytes
- transformed_record: bytes
-
- # partition columns
- source: str | None = None
- run_date: str | datetime.datetime | None = None
- run_type: str | None = None
- run_id: str | None = None
- action: str | None = None
+ timdex_record_id: str = field()
+ source_record: bytes = field()
+ transformed_record: bytes = field()
+ source: str = field()
+ run_date: datetime = field(converter=strict_date_parse)
+ run_type: str = field()
+ run_id: str = field()
+ action: str = field()
+
+ @property
+ def year(self) -> str:
+ return self.run_date.strftime("%Y")
+
+ @property
+ def month(self) -> str:
+ return self.run_date.strftime("%m")
+
+ @property
+ def day(self) -> str:
+ return self.run_date.strftime("%d")
def to_dict(
self,
- *,
- partition_values: dict[str, str | datetime.datetime] | None = None,
- validate: bool = True,
) -> dict:
- """Serialize instance as dictionary, setting partition values if passed."""
- if partition_values:
- for key, value in partition_values.items():
- setattr(self, key, value)
- if validate:
- self.validate()
- return asdict(self)
-
- def validate(self) -> None:
- """Validate DatasetRecord for writing."""
- # ensure all partition columns are set
- missing_partition_values = [
- field
- for field in ["source", "run_date", "run_type", "run_id", "action"]
- if getattr(self, field) is None
- ]
- if missing_partition_values:
- raise InvalidDatasetRecordError(
- f"Partition values are missing: {', '.join(missing_partition_values)}"
- )
+ """Serialize instance as dictionary."""
+ return {
+ **asdict(self),
+ "year": self.year,
+ "month": self.month,
+ "day": self.day,
+ }