Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ETL-690] Add additional expectations for fitbit data types #153

Merged
merged 4 commits into from
Nov 15, 2024
Merged

[ETL-690] Add additional expectations for fitbit data types #153

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
40dfd8d
Add additional expectations for fitbit data types
philerooski Nov 13, 2024
bcfef0e
upgrade to v2 of ecr login action
philerooski Nov 14, 2024
c724bdb
Revert "upgrade to v2 of ecr login action"
philerooski Nov 15, 2024
a311def
Fix remove artifacts command in cleanup workflow
philerooski Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 5 additions & 9 deletions .github/workflows/upload-and-deploy.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,14 +193,9 @@ jobs:
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
with:
mask-password: 'false'

- name: Get ECR secret names
id: ecr
run: |
usernameKey=docker_username_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _)
echo "username-key=$usernameKey" >> $GITHUB_OUTPUT
passwordKey=docker_password_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _)
echo "password-key=$passwordKey" >> $GITHUB_OUTPUT
- uses: actions/checkout@v3

- name: Set up Docker Buildx
Expand All @@ -215,10 +210,11 @@ jobs:
file: ${{ matrix.dockerfile }}
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache

outputs:
ecr-registry: ${{ steps.login-ecr.outputs.registry }}
ecr-username: ${{ steps.login-ecr.outputs[steps.ecr.outputs.username-key] }}
ecr-password: ${{ steps.login-ecr.outputs[steps.ecr.outputs.password-key] }}
ecr-username: ${{ steps.login-ecr.outputs['docker_username_914833433684_dkr_ecr_us_east_1_amazonaws_com'] }}
ecr-password: ${{ steps.login-ecr.outputs['docker_password_914833433684_dkr_ecr_us_east_1_amazonaws_com'] }}

glue-unit-tests:
name: Run Pytest unit tests for AWS glue
Expand Down
235 changes: 233 additions & 2 deletions src/glue/resources/data_values_expectations.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,18 @@
{
"fitbitactivitylogs": {
"expectations": [
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "AverageHeartRate",
"min_value": 25,
"max_value": 300
}
}
],
"expectation_suite_name": "fitbitactivitylogs_expectations"
},
"fitbitdailydata": {
"expectation_suite_name": "fitbitdailydata_expectations",
"expectations": [
{
"expectation_type": "expect_column_values_to_be_between",
Expand All @@ -25,8 +37,227 @@
"min_value": 4,
"max_value": 40
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "MinutesFairlyActive",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "MinutesLightlyActive",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "MinutesSedentary",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "MinutesVeryActive",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "RestingHeartRate",
"min_value": 25,
"max_value": 300
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "Hrv_DailyRmssd",
"min_value": 3,
"max_value": 210
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SpO2_Avg",
"min_value": 50,
"max_value": 100
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SpO2_Min",
"min_value": 50,
"max_value": 100
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SpO2_Max",
"min_value": 50,
"max_value": 100
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "TempSkin",
"min_value": -15,
"max_value": 15
}
}
]
],
"expectation_suite_name": "fitbitdailydata_expectations"
},
"fitbitintradaycombined": {
"expectations": [
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "DeepSleepSummaryBreathRate",
"min_value": 4,
"max_value": 40
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "RemSleepSummaryBreathRate",
"min_value": 4,
"max_value": 40
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "FullSleepSummaryBreathRate",
"min_value": 4,
"max_value": 40
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "LightSleepSummaryBreathRate",
"min_value": 4,
"max_value": 40
}
}
],
"expectation_suite_name": "fitbitintradaycombined_expectations"
},
"fitbitsleeplogs": {
"expectations": [
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "MinutesAfterWakeup",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "MinutesAsleep",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "MinutesAwake",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "MinutesToFallAsleep",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "TimeInBed",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SleepLevelAwake",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SleepLevelAsleep",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SleepLevelDeep",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SleepLevelLight",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SleepLevelRem",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SleepLevelRestless",
"min_value": 0,
"max_value": 1440
}
},
{
"expectation_type": "expect_column_values_to_be_between",
"kwargs": {
"column": "SleepLevelWake",
"min_value": 0,
"max_value": 1440
}
}
],
"expectation_suite_name": "fitbitsleeplogs_expectations"
},
"healthkitv2workouts": {
"expectation_suite_name": "healthkitv2workouts_expectations",
Expand Down
Loading