Skip to content

Commit

Permalink
Merge pull request #338 from cityofaustin/0.6.0-release-candidate
Browse files Browse the repository at this point in the history
promote 0.6.0 to production
  • Loading branch information
amenity authored Jun 11, 2021
2 parents 4d87c3e + a8f9f92 commit fc50e18
Show file tree
Hide file tree
Showing 34 changed files with 26,106 additions and 298 deletions.
68 changes: 68 additions & 0 deletions .github/workflows/atd_moped_build_stage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,12 @@ jobs:
MOPED_PRBODY: ${{ github.event.pull_request.body }}
run: |
export BRANCH_NAME=${GITHUB_REF##*/}
echo "Moped Test Build @ ${BRANCH_NAME}: ${{ github.event.inputs.description }}"
echo "SHA: ${GITHUB_SHA}"
echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}"
echo "MOPED_PRNUM: ${MOPED_PRNUM}"
echo "PWD: $(pwd)"
source $(pwd)/.github/workflows/aws-heroku-helper.sh
echo "Commented out editor build"
build_editor
Expand Down Expand Up @@ -77,6 +79,7 @@ jobs:
MOPED_PRBODY: ${{ github.event.pull_request.body }}
run: |
export BRANCH_NAME=${GITHUB_REF##*/}
echo "Moped Test Build @ ${BRANCH_NAME}: ${{ github.event.inputs.description }}"
echo "SHA: ${GITHUB_SHA}"
echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}"
echo "MOPED_PRNUM: ${MOPED_PRNUM}"
Expand All @@ -86,3 +89,68 @@ jobs:
clone_hasura_repo
build_database
run_database_migration
build_activity_log_sqs:
name: Build Activity Log SQS
runs-on: ubuntu-latest
steps:
- uses: actions/setup-python@v2
with:
python-version: "3.8"
architecture: "x64"
# Get the code first
- name: "Checkout"
uses: actions/checkout@v2
# Then install the AWC CLI tools & boto3
- name: "Install CLI Requirements: AWS Cli"
run: |
sudo apt-get install python3-setuptools
pip3 install awscli boto3
# Run the shell commands using the AWS environment variables
- name: "Moped Database: Re-deploy"
env:
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
ATD_MOPED_EVENTS_ROLE: ${{ secrets.ATD_MOPED_EVENTS_ROLE }}
PR_NUMBER: ${{ github.event.pull_request.number }}
MOPED_TEST_STAGE: "TRUE"
run: |
export BRANCH_NAME=${GITHUB_REF##*/}
echo "Moped Test Build @ ${BRANCH_NAME}: ${{ github.event.inputs.description }}"
echo "SHA: ${GITHUB_SHA}"
echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}"
echo "MOPED_PRNUM: ${MOPED_PRNUM}"
echo "PWD: $(pwd)"
source $(pwd)/.github/workflows/aws-moped-sqs-helper.sh
deploy_event_function "activity_log"
build_moped_test_api:
name: Build Moped Test API
runs-on: ubuntu-20.04
steps:
- uses: actions/setup-python@v2
with:
python-version: "3.8"
architecture: "x64"
# Get the code first
- name: "Checkout"
uses: actions/checkout@v2
# Then install the AWC CLI tools & boto3
- name: "Install AWS Cli"
run: |
sudo apt-get install python3-setuptools
pip3 install awscli boto3 zappa virtualenv
# Run the shell commands using the AWS environment variables
- name: "Build"
env:
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_MOPED_API_ZAPPA_CONFIGURAITON_FILE: ${{ secrets.AWS_MOPED_API_ZAPPA_CONFIGURAITON_FILE }}
run: |
export BRANCH_NAME=${GITHUB_REF##*/}
echo "SHA: ${GITHUB_SHA}"
echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}"
echo "GR: ${GITHUB_REF}"
echo "PWD: $(pwd)"
source $(pwd)/.github/workflows/aws-moped-api-helper.sh
deploy_moped_api
25 changes: 21 additions & 4 deletions .github/workflows/aws-moped-api-helper.sh
Original file line number Diff line number Diff line change
@@ -1,14 +1,31 @@
#!/usr/bin/env bash

#
# Determine working stage based on branch name
#
case "${BRANCH_NAME}" in
"production")
export WORKING_STAGE="production"
"production")
export WORKING_STAGE="production";
;;
"staging")
export WORKING_STAGE="staging";
;;
*)
export WORKING_STAGE="staging"
*)
export WORKING_STAGE="moped_test";
;;
esac

#
# Override the working stage if this is a moped-test build.
#
if [[ "${MOPED_TEST_STAGE}" = "TRUE" ]]; then
export WORKING_STAGE="test";
export BUILD_TYPE="moped_test";
else
export BUILD_TYPE="git push";
fi;

echo "BUILD_TYPE: ${BUILD_TYPE}";
echo "SOURCE -> BRANCH_NAME: ${BRANCH_NAME}";
echo "SOURCE -> WORKING_STAGE: ${WORKING_STAGE}";
echo "PR_NUMBER: '${PR_NUMBER}'";
Expand Down
20 changes: 19 additions & 1 deletion .github/workflows/aws-moped-sqs-helper.sh
Original file line number Diff line number Diff line change
@@ -1,14 +1,31 @@
#!/usr/bin/env bash

#
# Determine working stage based on branch name
#
case "${BRANCH_NAME}" in
"production")
export WORKING_STAGE="production";
;;
*)
"staging")
export WORKING_STAGE="staging";
;;
*)
export WORKING_STAGE="moped-test";
;;
esac

#
# Override the working stage if this is a moped-test build.
#
if [[ "${MOPED_TEST_STAGE}" = "TRUE" ]]; then
export WORKING_STAGE="test";
export BUILD_TYPE="moped-test";
else
export BUILD_TYPE="git push";
fi;

echo "BUILD_TYPE: ${BUILD_TYPE}";
echo "SOURCE -> BRANCH_NAME: ${BRANCH_NAME}";
echo "SOURCE -> WORKING_STAGE: ${WORKING_STAGE}";
echo "PR_NUMBER: '${PR_NUMBER}'";
Expand Down Expand Up @@ -39,6 +56,7 @@ function bundle_function {
#
function generate_env_vars {
FUNCTION_NAME_CONFIG=$1
echo "Generating Environment Variables for SQS";
aws secretsmanager get-secret-value \
--secret-id "ATD_MOPED_EVENT_SQS_ENV_${WORKING_STAGE^^}" | \
jq -rc ".SecretString" > handler_config.json;
Expand Down
30 changes: 27 additions & 3 deletions moped-api/claims.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,18 @@
)


def lower_case_email(user_email: str) -> str:
"""
Attempts to lower case a user email address
:param user_email: The user email address in question
:return:
"""
try:
return str(user_email).lower()
except:
return user_email


def get_claims(payload: LocalProxy) -> dict:
"""
It's a handy way to extract the hasura claims from a valid payload.
Expand Down Expand Up @@ -91,6 +103,13 @@ def is_valid_user(current_cognito_jwt: str) -> bool:
cognito_username = user_dict.get("cognito:username", "")
is_email_verified = user_dict.get("email_verified", False)

# If not a string, then not valid user
if not isinstance(user_email, str):
return False

# Lower-case the email address
user_email = lower_case_email(user_email)

# If not verified, then check it is an azure coa account
if not is_email_verified:
if str(cognito_username).startswith("azuread_") and str(
Expand All @@ -112,13 +131,14 @@ def is_valid_user(current_cognito_jwt: str) -> bool:
return True


def is_coa_staff(email: str) -> bool:
def is_coa_staff(user_email: str) -> bool:
"""
Returns True if the email address ends with city postfix
:param str email: The email address to be evaluated
:param str user_email: The email address to be evaluated
:return bool:
"""
return email.endswith("@austintexas.gov")
user_email = lower_case_email(user_email)
return user_email.endswith("@austintexas.gov")


def generate_iso_timestamp() -> str:
Expand Down Expand Up @@ -152,6 +172,7 @@ def retrieve_user_profile(user_email: str) -> dict:
:return dict: The user profile as a dictionary
"""
dynamodb = boto3.client("dynamodb", region_name="us-east-1")
user_email = lower_case_email(user_email)
user_profile = dynamodb.get_item(
TableName=AWS_COGNITO_DYNAMO_TABLE_NAME,
Key={
Expand All @@ -171,6 +192,7 @@ def load_claims(user_email: str) -> dict:
:param str user_email: The user email to retrieve the claims for
:return dict: The claims JSON
"""
user_email = lower_case_email(user_email)
profile = retrieve_user_profile(user_email=user_email)
claims_encrypted = profile["claims"]["S"]
cognito_uuid = profile["cognito_uuid"]["S"]
Expand Down Expand Up @@ -210,6 +232,7 @@ def put_claims(user_email: str, user_claims: dict, cognito_uuid: str = None, dat
claims_str = json.dumps(user_claims)
encrypted_claims = encrypt(fernet_key=AWS_COGNITO_DYNAMO_SECRET_KEY, content=claims_str)
dynamodb = boto3.client("dynamodb", region_name="us-east-1")
user_email = lower_case_email(user_email)
dynamodb.put_item(
TableName=AWS_COGNITO_DYNAMO_TABLE_NAME,
Item={
Expand All @@ -228,6 +251,7 @@ def delete_claims(user_email: str):
:param str user_email: The user email to set the claims for
"""
dynamodb = boto3.client("dynamodb", region_name="us-east-1")
user_email = lower_case_email(user_email)
dynamodb.delete_item(
TableName=AWS_COGNITO_DYNAMO_TABLE_NAME,
Key={"user_id": {"S": user_email}},
Expand Down
47 changes: 47 additions & 0 deletions moped-api/requirements/moped_test.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
argcomplete==1.12.0
attrs==20.2.0
boto==2.49.0
boto3==1.15.5
botocore==1.18.5
Cerberus==1.3.2
certifi==2020.6.20
cffi==1.14.3
cfn-flip==1.2.3
chardet==3.0.4
click==7.1.2
cognitojwt==1.2.2
cryptography==3.3.2
durationpy==0.5
ecdsa==0.14.1
Flask==1.1.2
Flask-Cognito==1.16
Flask-Cors==3.0.9
future==0.18.2
hjson==3.0.2
idna==2.10
itsdangerous==1.1.0
Jinja2==2.11.3
jmespath==0.10.0
kappa==0.6.0
MarkupSafe==1.1.1
pip-tools==5.3.1
placebo==0.9.0
pyasn1==0.4.8
pycparser==2.20
python-dateutil==2.6.1
python-jose==3.2.0
python-slugify==4.0.1
pytz==2020.1
PyYAML==5.4
requests==2.24.0
rsa==4.7
s3transfer==0.3.3
six==1.15.0
text-unidecode==1.3
toml==0.10.1
tqdm==4.49.0
troposphere==2.6.2
urllib3==1.25.10
Werkzeug==0.16.1
wsgi-request-logger==0.4.6
zappa==0.52.0
3 changes: 3 additions & 0 deletions moped-database/metadata/tables.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4525,6 +4525,7 @@
- fiscal_year
- is_retired
- milestone_id
- phase_name
- project_description
- project_description_public
- project_id
Expand Down Expand Up @@ -4554,6 +4555,7 @@
- fiscal_year
- is_retired
- milestone_id
- phase_name
- project_description
- project_description_public
- project_id
Expand Down Expand Up @@ -4583,6 +4585,7 @@
- fiscal_year
- is_retired
- milestone_id
- phase_name
- project_description
- project_description_public
- project_id
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ALTER TABLE "public"."moped_proj_notes" ALTER COLUMN "comm_id" SET NOT NULL;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ALTER TABLE "public"."moped_proj_notes" ALTER COLUMN "comm_id" DROP NOT NULL;
32 changes: 32 additions & 0 deletions moped-database/migrations/1623358779761_run_sql_migration/up.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
CREATE OR REPLACE VIEW "public"."project_list_view" AS
SELECT mp.project_uuid,
mp.project_id,
mp.project_name,
mp.project_description,
mp.project_description_public,
mp.ecapris_subproject_id,
mp.project_importance,
mp.project_order,
mp.current_status,
mp.timeline_id,
mp.current_phase,
mp.end_date,
mp.project_length,
mp.start_date,
mp.fiscal_year,
mp.capitally_funded,
mp.project_priority,
mp.date_added,
mp.added_by,
mp.is_retired,
mp.milestone_id,
mp.status_id,
string_agg(concat(mu.first_name, ' ', mu.last_name, ':', mpr.project_role_name), ','::text) AS project_team_members,
mp.updated_at,
mpph.phase_name
FROM ((((moped_project mp
LEFT JOIN moped_proj_personnel mpp ON (((mp.project_id = mpp.project_id) AND (mpp.status_id = 1))))
LEFT JOIN moped_users mu ON ((mpp.user_id = mu.user_id)))
LEFT JOIN moped_project_roles mpr ON ((mpp.role_id = mpr.project_role_id)))
LEFT JOIN moped_proj_phases mpph ON ((mp.project_id = mpph.project_id)) AND mpph.is_current_phase = true)
GROUP BY mp.project_uuid, mp.project_id, mp.project_name, mp.project_description, mp.project_description_public, mp.ecapris_subproject_id, mp.project_importance, mp.project_order, mp.current_status, mp.timeline_id, mp.current_phase, mp.end_date, mp.project_length, mp.start_date, mp.fiscal_year, mp.capitally_funded, mp.project_priority, mp.date_added, mp.added_by, mp.is_retired, mp.milestone_id, mp.status_id, mp.updated_at, mpph.phase_name;
Loading

0 comments on commit fc50e18

Please sign in to comment.