From 6aec26fad165102963197217e660552b7cccb1d7 Mon Sep 17 00:00:00 2001 From: Devin Matte Date: Sun, 7 Apr 2024 20:57:39 -0400 Subject: [PATCH] Move LAMP code to mbta-performance repo --- .github/CODEOWNERS | 1 - devops/helpers.sh | 1 - ingestor/.chalice/config.json | 5 - ingestor/.chalice/policy-lamp-ingest.json | 19 --- ingestor/.chalice/resources.json | 18 --- ingestor/app.py | 7 - ingestor/chalicelib/lamp/__init__.py | 3 - ingestor/chalicelib/lamp/backfill/__init__.py | 0 ingestor/chalicelib/lamp/backfill/main.py | 0 ingestor/chalicelib/lamp/ingest.py | 150 ------------------ ingestor/chalicelib/lamp/tests/__init__.py | 0 ingestor/chalicelib/lamp/tests/test_util.py | 29 ---- ingestor/chalicelib/lamp/utils.py | 25 --- poetry.lock | 50 +----- pyproject.toml | 1 - 15 files changed, 1 insertion(+), 308 deletions(-) delete mode 100644 ingestor/.chalice/policy-lamp-ingest.json delete mode 100644 ingestor/chalicelib/lamp/__init__.py delete mode 100644 ingestor/chalicelib/lamp/backfill/__init__.py delete mode 100644 ingestor/chalicelib/lamp/backfill/main.py delete mode 100644 ingestor/chalicelib/lamp/ingest.py delete mode 100644 ingestor/chalicelib/lamp/tests/__init__.py delete mode 100644 ingestor/chalicelib/lamp/tests/test_util.py delete mode 100644 ingestor/chalicelib/lamp/utils.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1ff278c..6095068 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1 @@ * @austinjpaul @devinmatte -ingestor/chalicelib/lamp @devinmatte @hamima-halim diff --git a/devops/helpers.sh b/devops/helpers.sh index 2304be3..94225a6 100644 --- a/devops/helpers.sh +++ b/devops/helpers.sh @@ -9,7 +9,6 @@ function shrink { zip -d -qq cfn/layer-deployment.zip '**/NOTICE*' zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/numpy*/tests/**/* zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/pandas*/tests/**/* - zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/pyarrow*/tests/**/* zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/examples/* zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/a*/* zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/b*/* diff --git a/ingestor/.chalice/config.json b/ingestor/.chalice/config.json index d696ca4..39c8a61 100644 --- a/ingestor/.chalice/config.json +++ b/ingestor/.chalice/config.json @@ -106,11 +106,6 @@ "lambda_memory_size": 1024, "lambda_timeout": 60, "max_ebs_size_gb": 2 - }, - "process_daily_lamp": { - "iam_policy_file": "policy-lamp-ingest.json", - "lambda_timeout": 900, - "lambda_memory_size": 2048 } } } diff --git a/ingestor/.chalice/policy-lamp-ingest.json b/ingestor/.chalice/policy-lamp-ingest.json deleted file mode 100644 index 83c33cf..0000000 --- a/ingestor/.chalice/policy-lamp-ingest.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Effect": "Allow", - "Resource": "arn:*:logs:*:*:*" - }, - { - "Action": ["s3:GetObject", "s3:PutObject"], - "Effect": "Allow", - "Resource": ["arn:aws:s3:::tm-mbta-performance/Events-lamp/daily-data/*"] - } - ] -} diff --git a/ingestor/.chalice/resources.json b/ingestor/.chalice/resources.json index d196df6..7fc844c 100644 --- a/ingestor/.chalice/resources.json +++ b/ingestor/.chalice/resources.json @@ -321,24 +321,6 @@ } } }, - "ProcessDailyLamp": { - "Type": "AWS::Serverless::Function", - "Properties": { - "Environment": { - "Variables": { - "DD_API_KEY": { - "Ref": "DDApiKey" - }, - "DD_VERSION": { - "Ref": "GitVersion" - }, - "DD_TAGS": { - "Ref": "DDTags" - } - } - } - } - }, "DeliveredTripMetricsDB": { "Type": "AWS::DynamoDB::Table", "Properties": { diff --git a/ingestor/app.py b/ingestor/app.py index c33709e..bd6e788 100644 --- a/ingestor/app.py +++ b/ingestor/app.py @@ -15,7 +15,6 @@ predictions, landing, trip_metrics, - lamp, yankee, ) @@ -164,9 +163,3 @@ def store_landing_data(event): @app.schedule(Cron("0", "0-6,9-23", "*", "*", "?", "*")) def update_yankee_shuttles(event): yankee.update_shuttles() - - -# Runs every 60 minutes from either 4 AM -> 1:55AM or 5 AM -> 2:55 AM depending on DST -@app.schedule(Cron("0", "0-6,9-23", "*", "*", "?", "*")) -def process_daily_lamp(event): - lamp.ingest_lamp_data() diff --git a/ingestor/chalicelib/lamp/__init__.py b/ingestor/chalicelib/lamp/__init__.py deleted file mode 100644 index bcc7cac..0000000 --- a/ingestor/chalicelib/lamp/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -__all__ = ["ingest_lamp_data"] - -from .ingest import ingest_lamp_data diff --git a/ingestor/chalicelib/lamp/backfill/__init__.py b/ingestor/chalicelib/lamp/backfill/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ingestor/chalicelib/lamp/backfill/main.py b/ingestor/chalicelib/lamp/backfill/main.py deleted file mode 100644 index e69de29..0000000 diff --git a/ingestor/chalicelib/lamp/ingest.py b/ingestor/chalicelib/lamp/ingest.py deleted file mode 100644 index 9019b44..0000000 --- a/ingestor/chalicelib/lamp/ingest.py +++ /dev/null @@ -1,150 +0,0 @@ -from datetime import date -import io -from typing import Tuple -import requests -import pandas as pd - -from .utils import format_dateint, get_current_service_date -from .. import parallel -from .. import s3 - - -LAMP_INDEX_URL = "https://performancedata.mbta.com/lamp/subway-on-time-performance-v1/index.csv" -RAPID_DAILY_URL_TEMPLATE = "https://performancedata.mbta.com/lamp/subway-on-time-performance-v1/{YYYY_MM_DD}-subway-on-time-performance-v1.parquet" -S3_BUCKET = "tm-mbta-performance" -# month and day are not zero-padded -S3_KEY_TEMPLATE = "Events-lamp/daily-data/{stop_id}/Year={YYYY}/Month={_M}/Day={_D}/events.csv" - - -# LAMP columns to fetch from parquet files -INPUT_COLUMNS = [ - "service_date", - "route_id", - "trip_id", - "stop_id", - "direction_id", - "stop_sequence", - "vehicle_id", - "vehicle_label", - "move_timestamp", # departure time from the previous station - "stop_timestamp", # arrival time at the current station -] - -# columns that should be output to s3 events.csv -OUTPUT_COLUMNS = [ - "service_date", - "route_id", - "trip_id", - "direction_id", - "stop_id", - "stop_sequence", - "vehicle_id", - "vehicle_label", - "event_type", - "event_time", -] - - -def _local_save(s3_key, stop_events): - """TODO remove this temp code, it saves the output files locally!""" - import os - - s3_key = ".temp/" + s3_key - if not os.path.exists(os.path.dirname(s3_key)): - os.makedirs(os.path.dirname(s3_key)) - stop_events.to_csv(s3_key) - - -def _process_arrival_departure_times(pq_df: pd.DataFrame) -> pd.DataFrame: - """Process and collate arrivals and departures for a timetable of events. - - Before: TODO add example - After: TODO add example - """ - # NB: While generally, we can trust df dtypes fetched from parquet files as the files are compressed with columnar metadata, - # theres some numerical imprecisions that numpy seem to be throwing on M1 machines - # that are affecting how epoch timestamps are being cased to datetimes. Maybe not a problem on the AWS machines, though? - pq_df["dep_time"] = pd.to_datetime(pq_df["move_timestamp"], unit="s", utc=True).dt.tz_convert("US/Eastern") - pq_df["arr_time"] = pd.to_datetime(pq_df["stop_timestamp"], unit="s", utc=True).dt.tz_convert("US/Eastern") - - # explode departure and arrival times - arr_df = pq_df[pq_df["arr_time"].notna()] - arr_df = arr_df.assign(event_type="ARR").rename(columns={"arr_time": "event_time"}) - arr_df = arr_df[OUTPUT_COLUMNS] - - dep_df = pq_df[pq_df["dep_time"].notna()] - dep_df = dep_df.assign(event_type="DEP").rename(columns={"dep_time": "event_time"}).drop(columns=["arr_time"]) - - # these departures are from the the previous stop! so set them to the previous stop id - # find the stop id for the departure whose sequence number precences the recorded one - # stop sequences don't necessarily increment by 1 or with a reliable pattern - dep_df = dep_df.sort_values(by=["stop_sequence"]) - dep_df = pd.merge_asof( - dep_df, - dep_df, - on=["stop_sequence"], - by=[ - "service_date", # comment for faster performance - "route_id", - "trip_id", - "vehicle_id", - "vehicle_label", # comment for faster performance - "direction_id", - "event_type", # comment for faster performance - ], - direction="backward", - suffixes=("_curr", "_prev"), - allow_exact_matches=False, # don't want to match on itself - ) - # use CURRENT time, but PREVIOUS stop id - dep_df = dep_df.rename(columns={"event_time_curr": "event_time", "stop_id_prev": "stop_id"})[OUTPUT_COLUMNS] - - # stitch together arrivals and departures - return pd.concat([arr_df, dep_df]) - - -def fetch_pq_file_from_remote(service_date: date) -> pd.DataFrame: - """Fetch a parquet file from LAMP for a given service date.""" - # TODO(check if file exists in index, throw if it doesn't) - url = RAPID_DAILY_URL_TEMPLATE.format(YYYY_MM_DD=service_date.strftime("%Y-%m-%d")) - result = requests.get(url) - return pd.read_parquet(io.BytesIO(result.content), columns=INPUT_COLUMNS, engine="pyarrow") - - -def ingest_pq_file(pq_df: pd.DataFrame) -> pd.DataFrame: - """Process and tranform columns for the full day's events.""" - pq_df["direction_id"] = pq_df["direction_id"].astype("int16") - pq_df["service_date"] = pq_df["service_date"].apply(format_dateint) - - processed_daily_events = _process_arrival_departure_times(pq_df) - return processed_daily_events.sort_values(by=["event_time"]) - - -def upload_to_s3(stop_id_and_events: Tuple[str, pd.DataFrame], service_date: date) -> None: - """Upload events to s3 as a .csv file.""" - # unpack from iterable - stop_id, stop_events = stop_id_and_events - - # Upload to s3 as csv - s3_key = S3_KEY_TEMPLATE.format(stop_id=stop_id, YYYY=service_date.year, _M=service_date.month, _D=service_date.day) - # _local_save(s3_key, stop_events) - s3.upload_df_as_csv(S3_BUCKET, s3_key, stop_events) - return [stop_id] - - -_parallel_upload = parallel.make_parallel(upload_to_s3) - - -def ingest_lamp_data(): - """Ingest and upload today's LAMP data.""" - service_date = get_current_service_date() - pq_df = fetch_pq_file_from_remote(service_date) - processed_daily_events = ingest_pq_file(pq_df) - - # split daily events by stop_id and parallel upload to s3 - stop_event_groups = processed_daily_events.groupby("stop_id") - _parallel_upload(stop_event_groups, service_date) - - -if __name__ == "__main__": - ingest_lamp_data() diff --git a/ingestor/chalicelib/lamp/tests/__init__.py b/ingestor/chalicelib/lamp/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ingestor/chalicelib/lamp/tests/test_util.py b/ingestor/chalicelib/lamp/tests/test_util.py deleted file mode 100644 index ae1248f..0000000 --- a/ingestor/chalicelib/lamp/tests/test_util.py +++ /dev/null @@ -1,29 +0,0 @@ -from datetime import date, datetime -from ..utils import EASTERN_TIME, service_date - - -def test_service_date(): - assert service_date(datetime(2023, 12, 15, 3, 0, 0)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 15, 5, 45, 0)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 15, 7, 15, 0)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 15, 23, 59, 59)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 16, 0, 0, 0)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 16, 2, 59, 59)) == date(2023, 12, 15) - - -def test_localized_datetime(): - assert service_date(datetime(2023, 12, 15, 3, 0, 0, tzinfo=EASTERN_TIME)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 15, 5, 45, 0, tzinfo=EASTERN_TIME)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 15, 7, 15, 0, tzinfo=EASTERN_TIME)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 15, 23, 59, 59, tzinfo=EASTERN_TIME)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 16, 0, 0, 0, tzinfo=EASTERN_TIME)) == date(2023, 12, 15) - assert service_date(datetime(2023, 12, 16, 2, 59, 59, tzinfo=EASTERN_TIME)) == date(2023, 12, 15) - - -def test_edt_vs_est_datetimes(): - assert service_date(datetime(2023, 11, 5, 23, 59, 59, tzinfo=EASTERN_TIME)) == date(2023, 11, 5) - assert service_date(datetime(2023, 11, 6, 0, 0, 0, tzinfo=EASTERN_TIME)) == date(2023, 11, 5) - assert service_date(datetime(2023, 11, 6, 1, 0, 0, tzinfo=EASTERN_TIME)) == date(2023, 11, 5) - assert service_date(datetime(2023, 11, 6, 2, 0, 0, tzinfo=EASTERN_TIME)) == date(2023, 11, 5) - # 3am EST is 4am EDT - assert service_date(datetime(2023, 11, 6, 3, 0, 0, tzinfo=EASTERN_TIME)) == date(2023, 11, 6) diff --git a/ingestor/chalicelib/lamp/utils.py b/ingestor/chalicelib/lamp/utils.py deleted file mode 100644 index 4255c4a..0000000 --- a/ingestor/chalicelib/lamp/utils.py +++ /dev/null @@ -1,25 +0,0 @@ -from datetime import date, datetime, timedelta -from zoneinfo import ZoneInfo - -EASTERN_TIME = ZoneInfo("US/Eastern") - - -def service_date(ts: datetime) -> date: - # In practice a None TZ is UTC, but we want to be explicit - # In many places we have an implied eastern - ts = ts.replace(tzinfo=EASTERN_TIME) - - if ts.hour >= 3 and ts.hour <= 23: - return date(ts.year, ts.month, ts.day) - - prior = ts - timedelta(days=1) - return date(prior.year, prior.month, prior.day) - - -def get_current_service_date() -> date: - return service_date(datetime.now(EASTERN_TIME)) - - -def format_dateint(dtint: int) -> str: - """Safely takes a dateint of YYYYMMDD to YYYY-MM-DD.""" - return datetime.strptime(str(dtint), "%Y%m%d").strftime("%Y-%m-%d") diff --git a/poetry.lock b/poetry.lock index b401015..090fa40 100644 --- a/poetry.lock +++ b/poetry.lock @@ -997,54 +997,6 @@ files = [ {file = "protobuf-5.26.1.tar.gz", hash = "sha256:8ca2a1d97c290ec7b16e4e5dff2e5ae150cc1582f55b5ab300d45cb0dfa90e51"}, ] -[[package]] -name = "pyarrow" -version = "15.0.2" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pyarrow-15.0.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:88b340f0a1d05b5ccc3d2d986279045655b1fe8e41aba6ca44ea28da0d1455d8"}, - {file = "pyarrow-15.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eaa8f96cecf32da508e6c7f69bb8401f03745c050c1dd42ec2596f2e98deecac"}, - {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23c6753ed4f6adb8461e7c383e418391b8d8453c5d67e17f416c3a5d5709afbd"}, - {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f639c059035011db8c0497e541a8a45d98a58dbe34dc8fadd0ef128f2cee46e5"}, - {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:290e36a59a0993e9a5224ed2fb3e53375770f07379a0ea03ee2fce2e6d30b423"}, - {file = "pyarrow-15.0.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:06c2bb2a98bc792f040bef31ad3e9be6a63d0cb39189227c08a7d955db96816e"}, - {file = "pyarrow-15.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:f7a197f3670606a960ddc12adbe8075cea5f707ad7bf0dffa09637fdbb89f76c"}, - {file = "pyarrow-15.0.2-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:5f8bc839ea36b1f99984c78e06e7a06054693dc2af8920f6fb416b5bca9944e4"}, - {file = "pyarrow-15.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5e81dfb4e519baa6b4c80410421528c214427e77ca0ea9461eb4097c328fa33"}, - {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a4f240852b302a7af4646c8bfe9950c4691a419847001178662a98915fd7ee7"}, - {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e7d9cfb5a1e648e172428c7a42b744610956f3b70f524aa3a6c02a448ba853e"}, - {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2d4f905209de70c0eb5b2de6763104d5a9a37430f137678edfb9a675bac9cd98"}, - {file = "pyarrow-15.0.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:90adb99e8ce5f36fbecbbc422e7dcbcbed07d985eed6062e459e23f9e71fd197"}, - {file = "pyarrow-15.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:b116e7fd7889294cbd24eb90cd9bdd3850be3738d61297855a71ac3b8124ee38"}, - {file = "pyarrow-15.0.2-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:25335e6f1f07fdaa026a61c758ee7d19ce824a866b27bba744348fa73bb5a440"}, - {file = "pyarrow-15.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:90f19e976d9c3d8e73c80be84ddbe2f830b6304e4c576349d9360e335cd627fc"}, - {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a22366249bf5fd40ddacc4f03cd3160f2d7c247692945afb1899bab8a140ddfb"}, - {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2a335198f886b07e4b5ea16d08ee06557e07db54a8400cc0d03c7f6a22f785f"}, - {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3e6d459c0c22f0b9c810a3917a1de3ee704b021a5fb8b3bacf968eece6df098f"}, - {file = "pyarrow-15.0.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:033b7cad32198754d93465dcfb71d0ba7cb7cd5c9afd7052cab7214676eec38b"}, - {file = "pyarrow-15.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:29850d050379d6e8b5a693098f4de7fd6a2bea4365bfd073d7c57c57b95041ee"}, - {file = "pyarrow-15.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:7167107d7fb6dcadb375b4b691b7e316f4368f39f6f45405a05535d7ad5e5058"}, - {file = "pyarrow-15.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e85241b44cc3d365ef950432a1b3bd44ac54626f37b2e3a0cc89c20e45dfd8bf"}, - {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:248723e4ed3255fcd73edcecc209744d58a9ca852e4cf3d2577811b6d4b59818"}, - {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ff3bdfe6f1b81ca5b73b70a8d482d37a766433823e0c21e22d1d7dde76ca33f"}, - {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:f3d77463dee7e9f284ef42d341689b459a63ff2e75cee2b9302058d0d98fe142"}, - {file = "pyarrow-15.0.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:8c1faf2482fb89766e79745670cbca04e7018497d85be9242d5350cba21357e1"}, - {file = "pyarrow-15.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:28f3016958a8e45a1069303a4a4f6a7d4910643fc08adb1e2e4a7ff056272ad3"}, - {file = "pyarrow-15.0.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:89722cb64286ab3d4daf168386f6968c126057b8c7ec3ef96302e81d8cdb8ae4"}, - {file = "pyarrow-15.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cd0ba387705044b3ac77b1b317165c0498299b08261d8122c96051024f953cd5"}, - {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad2459bf1f22b6a5cdcc27ebfd99307d5526b62d217b984b9f5c974651398832"}, - {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58922e4bfece8b02abf7159f1f53a8f4d9f8e08f2d988109126c17c3bb261f22"}, - {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:adccc81d3dc0478ea0b498807b39a8d41628fa9210729b2f718b78cb997c7c91"}, - {file = "pyarrow-15.0.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:8bd2baa5fe531571847983f36a30ddbf65261ef23e496862ece83bdceb70420d"}, - {file = "pyarrow-15.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6669799a1d4ca9da9c7e06ef48368320f5856f36f9a4dd31a11839dda3f6cc8c"}, - {file = "pyarrow-15.0.2.tar.gz", hash = "sha256:9c9bc803cb3b7bfacc1e96ffbfd923601065d9d3f911179d81e72d99fd74a3d9"}, -] - -[package.dependencies] -numpy = ">=1.16.6,<2" - [[package]] name = "pycodestyle" version = "2.11.1" @@ -1671,4 +1623,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "4b078be8386ecbb2be6b9cd00a8403d83ec117d75f71f3dc0b1157ca633a15dc" +content-hash = "63e927bc3d9a38209b6597e0758376450474cd141ccc47931d0fb98904ece562" diff --git a/pyproject.toml b/pyproject.toml index f956050..8f35088 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ dynamodb-json = "^1.3" datadog_lambda = "5.90.0" ddtrace = "2.7.2" tqdm = "^4.66.1" -pyarrow = "^15.0.2" [tool.poetry.dev-dependencies] chalice = "^1.31.0"