From 9e9ed30e66d167b3901692e0fbe7b607d6f3a5c0 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Fri, 20 Aug 2021 12:45:38 +0000 Subject: [PATCH 1/3] Update module github.com/plaid/plaid-go to v1 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c24c2c77..449fec7c 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/nleeper/goment v1.4.2 github.com/nyaruka/phonenumbers v1.0.71 github.com/pkg/errors v0.9.2-0.20201214064552-5dd12d0cfe7f - github.com/plaid/plaid-go v0.0.0-20210804185432-0d3f02cddaa8 + github.com/plaid/plaid-go v1.0.0 github.com/prometheus/client_golang v1.11.0 github.com/robfig/cron v1.2.0 // indirect github.com/sirupsen/logrus v1.8.1 From 6580f579227319164da330c971a0a0a8f75cab19 Mon Sep 17 00:00:00 2001 From: Elliot Courant Date: Sat, 21 Aug 2021 15:51:03 -0500 Subject: [PATCH 2/3] Rewrote plaid implementation. This includes a complete rewrite of how we were accessing plaids API. It removes the plaid_helper package and replaces it with the platypus package. This package only really exposes interfaces externally. Allowing us to make more changes there in our own layer to better support larger API changes in the future. It also includes more code coverage. While the code coverage right now is pretty basic and very happy path, it will help iterate on our plaid code much much faster in the future. --- .github/workflows/docker-build.yml | 28 -- .github/workflows/docs-deploy.yaml | 23 -- .github/workflows/docs.yml | 17 - .github/workflows/environments.yaml | 16 +- .github/workflows/go.yml | 48 --- .github/workflows/incoming.yml | 102 ++++++ .github/workflows/main.yaml | 90 +++++ .github/workflows/release.yaml | 5 +- .gitlab-ci.yml | 20 +- Makefile | 13 + Makefile.gitlab-ci | 16 - cloudbuild.yaml | 4 - go.mod | 136 +++++++- go.sum | 5 +- minikube/vault.yaml | 4 +- minikube/vault/auth.tf | 19 +- minikube/vault/mounts.tf | 2 +- minikube/vault/rest-api-service.tf | 11 + pkg/config/configuration.go | 3 + pkg/controller/controller.go | 12 +- pkg/controller/links.go | 9 +- pkg/controller/main_test.go | 17 +- pkg/controller/plaid.go | 174 +++------- pkg/internal/cmd/controllers.go | 4 +- pkg/internal/cmd/controllers_ui.go | 4 +- pkg/internal/cmd/serve.go | 39 +-- pkg/internal/mock_http_helper/responder.go | 12 +- .../mock_http_helper/responder_test.go | 39 +++ pkg/internal/mock_mail/mail_test.go | 2 +- pkg/internal/mock_plaid/accounts.go | 97 +++++- pkg/internal/mock_plaid/accounts_test.go | 11 + pkg/internal/mock_plaid/authentication.go | 43 +++ pkg/internal/mock_plaid/link.go | 33 ++ pkg/internal/mock_plaid/mock_exchange.go | 21 +- pkg/internal/mock_plaid/transactions.go | 87 +++++ pkg/internal/mock_plaid/transactions_test.go | 37 ++ pkg/internal/mock_plaid/verification.go | 43 +++ pkg/internal/myownsanity/number_test.go | 12 + pkg/internal/myownsanity/numbers.go | 25 ++ pkg/internal/plaid_helper/client.go | 303 ----------------- pkg/internal/plaid_helper/client_test.go | 95 ------ pkg/internal/plaid_helper/verification.go | 125 ------- pkg/internal/platypus/account.go | 123 +++++++ pkg/internal/platypus/account_test.go | 81 +++++ pkg/internal/platypus/client.go | 266 +++++++++++++++ pkg/internal/platypus/client_test.go | 130 ++++++++ pkg/internal/platypus/platypus.go | 315 ++++++++++++++++++ pkg/internal/platypus/platypus_test.go | 43 +++ pkg/internal/platypus/token.go | 52 +++ pkg/internal/platypus/transaction.go | 119 +++++++ pkg/internal/platypus/webhook.go | 212 ++++++++++++ pkg/internal/platypus/webhook_test.go | 42 +++ pkg/internal/testutils/account.go | 14 + pkg/internal/testutils/plaid.go | 2 +- pkg/internal/testutils/seed.go | 56 ++-- pkg/internal/vault_helper/vault.go | 19 ++ pkg/jobs/jobs.go | 10 +- pkg/jobs/non_distributed.go | 4 +- pkg/jobs/pull_account_balances.go | 32 +- pkg/jobs/pull_account_balances_test.go | 16 +- pkg/jobs/pull_historical_transactions.go | 151 +-------- pkg/jobs/pull_initial_transactions.go | 100 ++---- pkg/jobs/pull_latest_transactions.go | 175 +--------- pkg/jobs/transactions.go | 143 ++++++++ pkg/jobs/update_institutions.go | 104 ------ pkg/logging/pg.go | 4 +- pkg/repository/plaid_link.go | 64 +++- pkg/repository/plaid_link_test.go | 111 ++++++ templates/api-config.yaml | 1 + tests/pg/foreign_keys.sql | 9 + tests/pg/indexes.sql | 12 + values.acceptance.yaml | 3 +- values.dog.yaml | 9 +- values.staging.yaml | 3 +- values.yaml | 1 + 75 files changed, 2812 insertions(+), 1420 deletions(-) delete mode 100644 .github/workflows/docker-build.yml delete mode 100644 .github/workflows/docs-deploy.yaml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/go.yml create mode 100644 .github/workflows/incoming.yml create mode 100644 .github/workflows/main.yaml create mode 100644 pkg/internal/mock_http_helper/responder_test.go create mode 100644 pkg/internal/mock_plaid/accounts_test.go create mode 100644 pkg/internal/mock_plaid/authentication.go create mode 100644 pkg/internal/mock_plaid/link.go create mode 100644 pkg/internal/mock_plaid/transactions.go create mode 100644 pkg/internal/mock_plaid/transactions_test.go create mode 100644 pkg/internal/mock_plaid/verification.go create mode 100644 pkg/internal/myownsanity/number_test.go create mode 100644 pkg/internal/myownsanity/numbers.go delete mode 100644 pkg/internal/plaid_helper/client.go delete mode 100644 pkg/internal/plaid_helper/client_test.go delete mode 100644 pkg/internal/plaid_helper/verification.go create mode 100644 pkg/internal/platypus/account.go create mode 100644 pkg/internal/platypus/account_test.go create mode 100644 pkg/internal/platypus/client.go create mode 100644 pkg/internal/platypus/client_test.go create mode 100644 pkg/internal/platypus/platypus.go create mode 100644 pkg/internal/platypus/platypus_test.go create mode 100644 pkg/internal/platypus/token.go create mode 100644 pkg/internal/platypus/transaction.go create mode 100644 pkg/internal/platypus/webhook.go create mode 100644 pkg/internal/platypus/webhook_test.go create mode 100644 pkg/internal/testutils/account.go create mode 100644 pkg/jobs/transactions.go delete mode 100644 pkg/jobs/update_institutions.go create mode 100644 pkg/repository/plaid_link_test.go create mode 100644 tests/pg/foreign_keys.sql create mode 100644 tests/pg/indexes.sql diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml deleted file mode 100644 index 525a921b..00000000 --- a/.github/workflows/docker-build.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Build - -on: - push: - branches: - - '!renovate/*' - - staging - - trying -# pull_request: -# branches: -# - main - -jobs: - docker: - name: Docker - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Build Image - id: docker_build - uses: docker/build-push-action@v2 - with: - push: false - tags: containers.monetr.dev/rest-api diff --git a/.github/workflows/docs-deploy.yaml b/.github/workflows/docs-deploy.yaml deleted file mode 100644 index 359a0f0e..00000000 --- a/.github/workflows/docs-deploy.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: Documentation - -on: - push: - branches: - - main - -jobs: - build: - name: Generate - runs-on: ubuntu-latest - container: ghcr.io/monetr/build-containers/ubuntu:20.04-2021.05.01 - steps: - - uses: actions/checkout@v2 - - run: yarn install - - run: make dependencies - - run: make docs - - run: $GITHUB_WORKSPACE/node_modules/.bin/redoc-cli bundle $GITHUB_WORKSPACE/docs/swagger.yaml -o $GITHUB_WORKSPACE/docs/index.html - - name: Deploy - uses: JamesIves/github-pages-deploy-action@4.1.5 - with: - branch: gh-pages - folder: docs diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 83d3b54e..00000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Documentation - -on: - pull_request: - branches: [ staging, trying ] - -jobs: - build: - name: Generate - runs-on: ubuntu-latest - container: ghcr.io/monetr/build-containers/ubuntu:20.04-2021.05.01 - steps: - - uses: actions/checkout@v2 - - run: yarn install - - run: make dependencies - - run: make docs - - run: $GITHUB_WORKSPACE/node_modules/.bin/redoc-cli bundle $GITHUB_WORKSPACE/docs/swagger.yaml -o $GITHUB_WORKSPACE/docs/index.html diff --git a/.github/workflows/environments.yaml b/.github/workflows/environments.yaml index 8eb951d2..50cfaf62 100644 --- a/.github/workflows/environments.yaml +++ b/.github/workflows/environments.yaml @@ -6,10 +6,22 @@ on: workflow_dispatch: { } jobs: + release-acceptance: + name: Acceptance + runs-on: ubuntu-latest + container: golang:1.17.1 + steps: + - uses: actions/checkout@v2 + with: + ref: 'main' + - name: fetch + run: git fetch --prune --unshallow + - name: Push To Dog Food + run: git push origin $(go run github.com/monetr/rest-api/tools/releaser --since=-24h):acceptance release-dog: name: Dog Food runs-on: ubuntu-latest - container: golang:1.16.3 + container: golang:1.17.1 steps: - uses: actions/checkout@v2 with: @@ -21,7 +33,7 @@ jobs: release-production: name: Production runs-on: ubuntu-latest - container: golang:1.16.3 + container: golang:1.17.1 steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 5425009d..00000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Go - -on: - push: - branches: [ main, staging, trying ] -# pull_request: -# branches: [ main ] - -jobs: - build: - name: Test - runs-on: ubuntu-latest - container: golang:1.17.0 - env: - POSTGRES_HOST: postgres - POSTGRES_PASSWORD: "" - POSTGRES_USER: api-testing - POSTGRES_DB: test-db - services: - postgres: - image: postgres:13 - env: - POSTGRES_HOST_AUTH_METHOD: trust - POSTGRES_USER: api-testing - POSTGRES_DB: test-db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - uses: actions/checkout@v2 - - name: Dependencies - run: make dependencies - - name: Build - run: make build - - name: Setup Schema - run: make apply-schema-ci - - name: Test - run: make test -# license: -# name: License -# runs-on: ubuntu-latest -# container: ghcr.io/monetr/build-containers/ubuntu:20.04-2021.08.17 -# steps: -# - uses: actions/checkout@v2 -# - name: Licenses -# run: make license diff --git a/.github/workflows/incoming.yml b/.github/workflows/incoming.yml new file mode 100644 index 00000000..be9384e5 --- /dev/null +++ b/.github/workflows/incoming.yml @@ -0,0 +1,102 @@ +name: GitHub + +on: + push: + branches: + - staging + - trying + +jobs: + test: + name: Test + runs-on: ubuntu-latest + container: golang:1.17.0 + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: api-testing + POSTGRES_DB: test-db + services: + postgres: + image: postgres:13 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: api-testing + POSTGRES_DB: test-db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + - name: Dependencies + run: make dependencies + - name: Build + run: make build + - name: Setup Schema + run: make apply-schema-ci + - name: Test + run: make test + pg-test: + name: PostgreSQL Test + runs-on: ubuntu-latest + container: ghcr.io/monetr/pgtest:latest + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: postgres + POSTGRES_DB: test-db + POSTGRES_HOST_AUTH_METHOD: trust + services: + postgres: + image: ghcr.io/monetr/pgtest:latest + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: postgres + POSTGRES_DB: test-db + POSTGRES_HOST_AUTH_METHOD: trust + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + - name: Test + run: make pg_test + - name: Publish Test Report + uses: mikepenz/action-junit-report@v2 + if: always() # always run even if the previous step fails + with: + report_paths: '/junit.xml' + check_name: 'PostgreSQL Test Summary' + docs-generate: + name: Generate Documentation + runs-on: ubuntu-latest + container: ghcr.io/monetr/build-containers/ubuntu:20.04 + steps: + - uses: actions/checkout@v2 + - run: yarn install + - run: make dependencies + - run: make docs + - run: $GITHUB_WORKSPACE/node_modules/.bin/redoc-cli bundle $GITHUB_WORKSPACE/docs/swagger.yaml -o $GITHUB_WORKSPACE/docs/index.html + docker: + needs: + - test + name: Docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build Image + id: docker_build + uses: docker/build-push-action@v2 + with: + push: false + tags: containers.monetr.dev/rest-api + diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 00000000..99fbc2b1 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,90 @@ +name: Main + +on: + push: + branches: + - main + +jobs: + test: + name: Test + runs-on: ubuntu-latest + container: golang:1.17.0 + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: api-testing + POSTGRES_DB: test-db + services: + postgres: + image: postgres:13 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: api-testing + POSTGRES_DB: test-db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + - name: Dependencies + run: make dependencies + - name: Build + run: make build + - name: Setup Schema + run: make apply-schema-ci + - name: Test + run: make test + pg-test: + name: PostgreSQL Test + runs-on: ubuntu-latest + container: ghcr.io/monetr/pgtest:latest + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: postgres + POSTGRES_DB: test-db + POSTGRES_HOST_AUTH_METHOD: trust + services: + postgres: + image: ghcr.io/monetr/pgtest:latest + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: postgres + POSTGRES_DB: test-db + POSTGRES_HOST_AUTH_METHOD: trust + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + - name: Test + run: make pg_test + - name: Publish Test Report + uses: mikepenz/action-junit-report@v2 + if: always() # always run even if the previous step fails + with: + report_paths: '/junit.xml' + check_name: 'PostgreSQL Test Summary' + docs-deploy: + needs: + - test + name: Deploy Documentation + runs-on: ubuntu-latest + container: ghcr.io/monetr/build-containers/ubuntu:20.04 + steps: + - uses: actions/checkout@v2 + - run: yarn install + - run: make dependencies + - run: make docs + - run: $GITHUB_WORKSPACE/node_modules/.bin/redoc-cli bundle $GITHUB_WORKSPACE/docs/swagger.yaml -o $GITHUB_WORKSPACE/docs/index.html + - name: Deploy + uses: JamesIves/github-pages-deploy-action@4.1.5 + with: + branch: gh-pages + folder: docs diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d61556b9..9bda5435 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,8 +1,9 @@ name: Daily Release on: - schedule: - - cron: 0 23 * * * + push: + branches: + - acceptance workflow_dispatch: { } jobs: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e0bec041..918c006b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ # exception for staging branches that bors creates. workflow: rules: - - if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "staging" || $CI_COMMIT_BRANCH == "trying"' + - if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "staging" || $CI_COMMIT_BRANCH == "trying" || $CI_COMMIT_BRANCH == "dog" || $CI_COMMIT_BRANCH == "acceptance"' when: always - if: '$CI_COMMIT_BRANCH == "staging" || $CI_COMMIT_BRANCH == "trying"' when: never @@ -12,8 +12,8 @@ workflow: when: never - if: $CI_COMMIT_TAG when: always - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != "main"' - when: never +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != "main"' +# when: never stages: - Release @@ -47,7 +47,7 @@ Release Testing: Go Dependencies: needs: [ ] stage: Dependencies - image: containers.monetr.dev/golang:1.16.3 + image: containers.monetr.dev/golang:1.17.0 tags: - k8s:shared - arch:amd64 @@ -63,7 +63,7 @@ Binary: needs: - "Go Dependencies" stage: Build - image: containers.monetr.dev/golang:1.16.3 + image: containers.monetr.dev/golang:1.17.0 tags: - k8s:shared - arch:amd64 @@ -89,7 +89,7 @@ Go Tests: POSTGRES_DB: test-db POSTGRES_HOST_AUTH_METHOD: trust stage: Test - image: containers.monetr.dev/golang:1.16.3 + image: containers.monetr.dev/golang:1.17.0 tags: - k8s:shared - arch:amd64 @@ -154,6 +154,7 @@ Yeet Coverage: refs: - main - tags + - acceptance image: proxy.monetr.dev/docker:19.03.15 variables: DOCKER_HOST: tcp://docker:2375 @@ -241,6 +242,9 @@ Generate - Acceptance: expire_in: 7 days Generate - Dog: + only: + refs: + - dog needs: [ ] stage: Prepare image: containers.monetr.dev/ubuntu:20.04 @@ -301,7 +305,7 @@ Dry - Staging: Dry - Acceptance: only: refs: - - tags + - accceptance environment: name: Acceptance url: https://api.acceptance.monetr.dev @@ -356,7 +360,7 @@ Deploy - Staging: Deploy - Acceptance: only: refs: - - tags + - accceptance environment: name: Acceptance url: https://api.acceptance.monetr.dev diff --git a/Makefile b/Makefile index 9560e2e6..0f1e4a09 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,19 @@ ifdef GITHUB_ACTION include Makefile.github-actions endif +# PostgreSQL tests currently only work in CI pipelines. +ifdef CI +PG_TEST_EXTENSION_QUERY = "CREATE EXTENSION pgtap;" +JUNIT_OUTPUT_FILE=/junit.xml +pg_test: + @for FILE in $(PWD)/schema/*.up.sql; do \ + echo "Applying $$FILE"; \ + psql -q -d $(POSTGRES_DB) -U $(POSTGRES_USER) -h $(POSTGRES_HOST) -f $$FILE || exit 1; \ + done; + psql -q -d $(POSTGRES_DB) -U $(POSTGRES_USER) -h $(POSTGRES_HOST) -c $(PG_TEST_EXTENSION_QUERY) + -JUNIT_OUTPUT_FILE=$(JUNIT_OUTPUT_FILE) pg_prove -h $(POSTGRES_HOST) -U $(POSTGRES_USER) -d $(POSTGRES_DB) -f -c $(PWD)/tests/pg/*.sql --verbose --harness TAP::Harness::JUnit +endif + include Makefile.release include Makefile.docker diff --git a/Makefile.gitlab-ci b/Makefile.gitlab-ci index 363544a9..3dd0c110 100644 --- a/Makefile.gitlab-ci +++ b/Makefile.gitlab-ci @@ -15,19 +15,3 @@ ifdef GOPATH apply-schema-ci: go run $(MONETR_CLI_PACKAGE) database migrate -d $(POSTGRES_DB) -U $(POSTGRES_USER) -H $(POSTGRES_HOST) endif - -apply-schema-ci-psql: - @for FILE in $(CI_PROJECT_DIR)/schema/*.up.sql; do \ - echo "Applying $$FILE"; \ - psql -q -d $(POSTGRES_DB) -U $(POSTGRES_USER) -h $(POSTGRES_HOST) -f $$FILE || exit 1; \ - done; - -PG_PROVE_TESTS = "$(PWD)/tests/pg/*.sql" -PG_TEST_EXTENSION_QUERY = "CREATE EXTENSION pgtap;" - -pg_test: apply-schema-ci-psql - psql -q -d $(POSTGRES_DB) -U $(POSTGRES_USER) -h $(POSTGRES_HOST) -c $(PG_TEST_EXTENSION_QUERY) - make pg_prove_tests - -pg_prove_tests: - pg_prove -h $(POSTGRES_HOST) -U $(POSTGRES_USER) -d $(POSTGRES_DB) -f ./tests/pg/*.sql \ No newline at end of file diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 26ea5e00..21cca300 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -7,7 +7,3 @@ steps: waitFor: [ 'build-docker' ] name: 'gcr.io/cloud-builders/docker' args: [ 'push', 'gcr.io/$PROJECT_ID/github.com/monetr/rest-api:$COMMIT_SHA' ] - - id: 'test-gcloud' - waitFor: [ 'build-docker' ] - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' - args: [ 'gcloud', 'container', 'clusters', 'list' ] diff --git a/go.mod b/go.mod index 449fec7c..093b795a 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ module github.com/monetr/rest-api -go 1.16 +go 1.17 require ( cloud.google.com/go/logging v1.4.2 github.com/MicahParks/keyfunc v0.4.0 + github.com/OneOfOne/xxhash v1.2.2 + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/alicebob/miniredis/v2 v2.15.1 github.com/brianvoe/gofakeit/v6 v6.7.1 github.com/form3tech-oss/jwt-go v3.2.5+incompatible @@ -42,3 +44,135 @@ require ( google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af gopkg.in/ezzarghili/recaptcha-go.v4 v4.3.0 ) + +require ( + cloud.google.com/go v0.81.0 // indirect + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect + github.com/CloudyKit/jet/v5 v5.1.1 // indirect + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/ajg/form v1.5.1 // indirect + github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect + github.com/andybalholm/brotli v1.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v3 v3.0.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/chris-ramon/douceur v0.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect + github.com/emirpasic/gods v1.12.0 // indirect + github.com/fatih/structs v1.1.0 // indirect + github.com/flosch/pongo2/v4 v4.0.1 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/spec v0.20.3 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-pg/zerochecker v0.2.0 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/google/go-querystring v1.0.0 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/gorilla/css v1.0.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.1 // indirect + github.com/hashicorp/go-multierror v1.1.0 // indirect + github.com/hashicorp/go-retryablehttp v0.6.6 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/vault/sdk v0.2.1 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/imkira/go-interpol v1.1.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/iris-contrib/httpexpect/v2 v2.0.5 // indirect + github.com/iris-contrib/jade v1.1.4 // indirect + github.com/iris-contrib/schema v0.0.6 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/kataras/blocks v0.0.4 // indirect + github.com/kataras/golog v0.1.6 // indirect + github.com/kataras/pio v0.0.10 // indirect + github.com/kataras/sitemap v0.0.5 // indirect + github.com/kataras/tunnel v0.0.2 // indirect + github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect + github.com/klauspost/compress v1.11.3 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/microcosm-cc/bluemonday v1.0.4 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pierrec/lz4 v2.5.2+incompatible // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/schollz/closestmatch v2.1.0+incompatible // indirect + github.com/sergi/go-diff v1.1.0 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/tdewolff/minify/v2 v2.9.10 // indirect + github.com/tdewolff/parse/v2 v2.5.5 // indirect + github.com/tkuchiki/go-timezone v0.2.0 // indirect + github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/vmihailenco/bufpool v0.1.11 // indirect + github.com/vmihailenco/tagparser v0.1.2 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/xanzy/ssh-agent v0.3.0 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect + github.com/yosssi/ace v0.0.5 // indirect + github.com/yudai/gojsondiff v1.0.0 // indirect + github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect + github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect + golang.org/x/tools v0.1.5 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/api v0.46.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/grpc v1.40.0 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/square/go-jose.v2 v2.5.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + mellium.im/sasl v0.2.1 // indirect + moul.io/http2curl v1.0.0 // indirect +) diff --git a/go.sum b/go.sum index 50e8002c..de0cb47d 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,7 @@ github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tT github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= @@ -594,8 +595,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.2-0.20201214064552-5dd12d0cfe7f h1:lJqhwddJVYAkyp72a4pwzMClI20xTwL7miDdm2W/KBM= github.com/pkg/errors v0.9.2-0.20201214064552-5dd12d0cfe7f/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/plaid/plaid-go v0.0.0-20210804185432-0d3f02cddaa8 h1:+fef2TcO+bG6vkiYljmYpBxpTRXg5jllX5Ku0w/kg8E= -github.com/plaid/plaid-go v0.0.0-20210804185432-0d3f02cddaa8/go.mod h1:c7cDT1Lkcr0AgKJGVIG+oCa07jOrrg4Um8nduQ1eQN0= +github.com/plaid/plaid-go v1.0.0 h1:JQEDXHt0K7wNs/WaL/E9DLH1RNYIybO98GiIjekMxYo= +github.com/plaid/plaid-go v1.0.0/go.mod h1:jsPs/+TSYwDPNxMhY2uwlpDUJBnqppGg+pNXNgdITc0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= diff --git a/minikube/vault.yaml b/minikube/vault.yaml index e8dee26f..93adeba1 100644 --- a/minikube/vault.yaml +++ b/minikube/vault.yaml @@ -268,7 +268,7 @@ metadata: spec: type: ClusterIP ports: - - port: 443 + - port: 80 targetPort: http protocol: TCP name: http @@ -300,7 +300,7 @@ spec: - path: / backend: serviceName: vault - servicePort: 443 + servicePort: 80 --- apiVersion: v1 diff --git a/minikube/vault/auth.tf b/minikube/vault/auth.tf index d6a93a18..ec69a308 100644 --- a/minikube/vault/auth.tf +++ b/minikube/vault/auth.tf @@ -9,4 +9,21 @@ resource "vault_kubernetes_auth_backend_config" "example" { token_reviewer_jwt = var.kubernetes_reviewer_jwt issuer = "kubernetes.io/serviceaccount" disable_iss_validation = true -} \ No newline at end of file +} + +resource "vault_auth_backend" "userpass" { + type = "userpass" +} + +resource "vault_generic_endpoint" "monetr-user" { + depends_on = [vault_auth_backend.userpass] + path = "auth/userpass/users/monetr" + ignore_absent_fields = true + + data_json = jsonencode({ + policies = [ + vault_policy.rest-api-service-policy.name, + ] + password = "password" + }) +} diff --git a/minikube/vault/mounts.tf b/minikube/vault/mounts.tf index 138c9f6b..08cb5c71 100644 --- a/minikube/vault/mounts.tf +++ b/minikube/vault/mounts.tf @@ -1,5 +1,5 @@ resource "vault_mount" "plaid-client-secrets" { - path = "plaid/clients" + path = "customers/plaid" type = "kv-v2" description = "KV store used for Plaid client credentials" } \ No newline at end of file diff --git a/minikube/vault/rest-api-service.tf b/minikube/vault/rest-api-service.tf index 779ff7a3..550f1def 100644 --- a/minikube/vault/rest-api-service.tf +++ b/minikube/vault/rest-api-service.tf @@ -1,4 +1,15 @@ data "vault_policy_document" "rest-api-policy" { + rule { + path = "${vault_mount.plaid-client-secrets.path}/*" + capabilities = [ + "create", + "read", + "update", + "delete", + ] + description = "Allow the REST API to manage client secrets." + } + rule { path = "${vault_mount.plaid-client-secrets.path}/data/*" capabilities = [ diff --git a/pkg/config/configuration.go b/pkg/config/configuration.go index 04340a05..59462272 100644 --- a/pkg/config/configuration.go +++ b/pkg/config/configuration.go @@ -127,6 +127,7 @@ type Plaid struct { WebhooksEnabled bool WebhooksDomain string + OAuthDomain string } type CORS struct { @@ -185,6 +186,7 @@ type Vault struct { Auth string Token string TokenFile string + Username, Password string Role string CertificatePath string KeyPath string @@ -262,6 +264,7 @@ func setupEnv(v *viper.Viper) { v.BindEnv("Plaid.EnableReturningUserExperience", "MONETR_PLAID_RETURNING_EXPERIENCE") v.BindEnv("Plaid.WebhooksEnabled", "MONETR_PLAID_WEBHOOKS_ENABLED") v.BindEnv("Plaid.WebhooksDomain", "MONETR_PLAID_WEBHOOKS_DOMAIN") + v.BindEnv("Plaid.OAuthDomain", "MONETR_PLAID_OAUTH_DOMAIN") v.BindEnv("PostgreSQL.Address", "MONETR_PG_ADDRESS") v.BindEnv("PostgreSQL.Port", "MONETR_PG_PORT") v.BindEnv("PostgreSQL.Username", "MONETR_PG_USERNAME") diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 855b1016..a3670c94 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/mail" "net/http" "net/smtp" @@ -19,7 +20,6 @@ import ( "github.com/monetr/rest-api/pkg/build" "github.com/monetr/rest-api/pkg/cache" "github.com/monetr/rest-api/pkg/config" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/jobs" "github.com/monetr/rest-api/pkg/metrics" @@ -38,8 +38,8 @@ type Controller struct { db *pg.DB configuration config.Configuration captcha *recaptcha.ReCAPTCHA - plaid plaid_helper.Client - plaidWebhookVerification plaid_helper.WebhookVerification + plaid platypus.Platypus + plaidWebhookVerification platypus.WebhookVerification plaidSecrets secrets.PlaidSecretsProvider smtp *smtp.Client mailVerifyCode *gotp.HOTP @@ -61,7 +61,7 @@ func NewController( configuration config.Configuration, db *pg.DB, job jobs.JobManager, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, stripe stripe_helper.Stripe, cachePool *redis.Pool, @@ -85,12 +85,14 @@ func NewController( pubSub := pubsub.NewPostgresPubSub(log, db) basicBilling := billing.NewBasicBilling(log, accountsRepo, pubSub) + plaidWebhookVerification := platypus.NewInMemoryWebhookVerification(log, plaidClient, 5*time.Minute) + return &Controller{ captcha: &captcha, configuration: configuration, db: db, plaid: plaidClient, - plaidWebhookVerification: plaid_helper.NewMemoryWebhookVerificationCache(log, plaidClient), + plaidWebhookVerification: plaidWebhookVerification, plaidSecrets: plaidSecrets, log: log, job: job, diff --git a/pkg/controller/links.go b/pkg/controller/links.go index 0ef61f12..ffa01486 100644 --- a/pkg/controller/links.go +++ b/pkg/controller/links.go @@ -269,7 +269,14 @@ func (c *Controller) deleteLink(ctx iris.Context) { return } - if err = c.plaid.RemoveItem(c.getContext(ctx), accessToken); err != nil { + client, err := c.plaid.NewClient(c.getContext(ctx), link, accessToken) + if err != nil { + // ERROR THINGS + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create plaid client") + return + } + + if err = client.RemoveItem(c.getContext(ctx)); err != nil { crumbs.Error(c.getContext(ctx), "Failed to remove item", "plaid", map[string]interface{}{ "linkId": link.LinkId, "itemId": link.PlaidLink.ItemId, diff --git a/pkg/controller/main_test.go b/pkg/controller/main_test.go index 6e8cbfa5..a9347df4 100644 --- a/pkg/controller/main_test.go +++ b/pkg/controller/main_test.go @@ -11,10 +11,12 @@ import ( "github.com/monetr/rest-api/pkg/config" "github.com/monetr/rest-api/pkg/controller" "github.com/monetr/rest-api/pkg/internal/mock_secrets" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/internal/testutils" "github.com/monetr/rest-api/pkg/jobs" + "github.com/monetr/rest-api/pkg/repository" + "github.com/monetr/rest-api/pkg/secrets" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" "net/http" @@ -35,7 +37,9 @@ func NewTestApplicationConfig(t *testing.T) config.Configuration { SMTP: config.SMTPClient{}, ReCAPTCHA: config.ReCAPTCHA{}, Plaid: config.Plaid{ - Environment: plaid.Sandbox, + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, }, CORS: config.CORS{ Debug: false, @@ -54,12 +58,9 @@ func NewTestApplication(t *testing.T) *httptest.Expect { func NewTestApplicationWithConfig(t *testing.T, configuration config.Configuration) *httptest.Expect { log := testutils.GetLog(t) db := testutils.GetPgDatabase(t) - plaidClient := plaid_helper.NewPlaidClient(log, plaid.ClientOptions{ - ClientID: configuration.Plaid.ClientID, - Secret: configuration.Plaid.ClientSecret, - Environment: configuration.Plaid.Environment, - HTTPClient: http.DefaultClient, - }) + secretProvider := secrets.NewPostgresPlaidSecretsProvider(log, db) + plaidRepo := repository.NewPlaidRepository(db) + plaidClient := platypus.NewPlaid(log, secretProvider, plaidRepo, configuration.Plaid) miniRedis := miniredis.NewMiniRedis() require.NoError(t, miniRedis.Start()) diff --git a/pkg/controller/plaid.go b/pkg/controller/plaid.go index e98b6a66..66829469 100644 --- a/pkg/controller/plaid.go +++ b/pkg/controller/plaid.go @@ -6,6 +6,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/kataras/iris/v12" "github.com/monetr/rest-api/pkg/crumbs" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/models" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -14,7 +16,6 @@ import ( "time" "github.com/kataras/iris/v12/core/router" - "github.com/plaid/plaid-go/plaid" ) func (c *Controller) handlePlaidLinkEndpoints(p router.Party) { @@ -125,67 +126,35 @@ func (c *Controller) newPlaidToken(ctx iris.Context) { } } - plaidProducts := []string{ - "transactions", - } - legalName := "" if len(me.LastName) > 0 { legalName = fmt.Sprintf("%s %s", me.FirstName, me.LastName) } - var phoneNumber string + var phoneNumber *string if me.Login.PhoneNumber != nil { - phoneNumber = me.Login.PhoneNumber.E164() - } - - var webhook string - if c.configuration.Plaid.WebhooksEnabled { - domain := c.configuration.Plaid.WebhooksDomain - if domain != "" { - webhook = fmt.Sprintf("%s/plaid/webhook", c.configuration.Plaid.WebhooksDomain) - } else { - c.log.Errorf("plaid webhooks are enabled, but they cannot be registered with without a domain") - } + phoneNumber = myownsanity.StringP(me.Login.PhoneNumber.E164()) } - redirectUri := fmt.Sprintf("https://%s/plaid/oauth-return", c.configuration.UIDomainName) - - token, err := c.plaid.CreateLinkToken(c.getContext(ctx), plaid.LinkTokenConfigs{ - User: &plaid.LinkTokenUser{ - ClientUserID: strconv.FormatUint(userId, 10), - LegalName: legalName, - PhoneNumber: phoneNumber, - EmailAddress: me.Login.Email, - // TODO (elliotcourant) I'm going to leave these be for now but we need - // to loop back and add this once email/phone verification is working. - PhoneNumberVerifiedTime: time.Time{}, - EmailAddressVerifiedTime: time.Time{}, - }, - ClientName: "monetr", - Products: plaidProducts, - CountryCodes: []string{ - "US", - }, - Webhook: webhook, - AccountFilters: nil, - CrossAppItemAdd: nil, - PaymentInitiation: nil, - Language: "en", - LinkCustomizationName: "", - RedirectUri: redirectUri, + token, err := c.plaid.CreateLinkToken(c.getContext(ctx), platypus.LinkTokenOptions{ + ClientUserID: strconv.FormatUint(userId, 10), + LegalName: legalName, + PhoneNumber: phoneNumber, + PhoneNumberVerifiedTime: nil, + EmailAddress: me.Login.Email, + EmailAddressVerifiedTime: nil, }) if err != nil { c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create link token") return } - if err = c.storeLinkTokenInCache(c.getContext(ctx), log, me.UserId, token.LinkToken, token.Expiration); err != nil { + if err = c.storeLinkTokenInCache(c.getContext(ctx), log, me.UserId, token.Token(), token.Expiration()); err != nil { log.WithError(err).Warn("failed to cache link token") } ctx.JSON(map[string]interface{}{ - "linkToken": token.LinkToken, + "linkToken": token.Token(), }) } @@ -235,72 +204,24 @@ func (c *Controller) updatePlaidLink(ctx iris.Context) { return } - legalName := "" - if len(me.LastName) > 0 { - legalName = fmt.Sprintf("%s %s", me.FirstName, me.LastName) - } else { - // TODO Handle a missing last name, we need a legal name Plaid. - // Should this be considered an error state? - } - - var phoneNumber string - if me.Login.PhoneNumber != nil { - phoneNumber = me.Login.PhoneNumber.E164() - } - - var webhook string - if c.configuration.Plaid.WebhooksEnabled { - domain := c.configuration.Plaid.WebhooksDomain - if domain != "" { - webhook = fmt.Sprintf("%s/plaid/webhook", c.configuration.Plaid.WebhooksDomain) - } else { - log.Errorf("plaid webhooks are enabled, but they cannot be registered with without a domain") - } - } - - accessToken, err := c.plaidSecrets.GetAccessTokenForPlaidLinkId(c.getContext(ctx), repo.AccountId(), link.PlaidLink.ItemId) + client, err := c.plaid.NewClientFromLink(c.getContext(ctx), me.AccountId, linkId) if err != nil { - log.WithError(err).Errorf("failed to retrieve current access token") - c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to retrieve current access token") + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create Plaid client for link") return } - redirectUri := fmt.Sprintf("https://%s/plaid/oauth-return", c.configuration.UIDomainName) - - token, err := c.plaid.CreateLinkToken(c.getContext(ctx), plaid.LinkTokenConfigs{ - User: &plaid.LinkTokenUser{ - ClientUserID: strconv.FormatUint(me.UserId, 10), - LegalName: legalName, - PhoneNumber: phoneNumber, - EmailAddress: me.Login.Email, - // TODO Add in email/phone verification. - PhoneNumberVerifiedTime: time.Time{}, - EmailAddressVerifiedTime: time.Time{}, - }, - ClientName: "monetr", - CountryCodes: []string{ - "US", - }, - Webhook: webhook, - AccountFilters: nil, - CrossAppItemAdd: nil, - PaymentInitiation: nil, - Language: "en", - LinkCustomizationName: "", - RedirectUri: redirectUri, - AccessToken: accessToken, - }) + token, err := client.UpdateItem(c.getContext(ctx)) if err != nil { - c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create link token") + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create link token to update Plaid link") return } - if err = c.storeLinkTokenInCache(c.getContext(ctx), log, me.UserId, token.LinkToken, token.Expiration); err != nil { + if err = c.storeLinkTokenInCache(c.getContext(ctx), log, me.UserId, token.Token(), token.Expiration()); err != nil { log.WithError(err).Warn("failed to cache link token") } ctx.JSON(map[string]interface{}{ - "linkToken": token.LinkToken, + "linkToken": token.Token(), }) } @@ -424,19 +345,6 @@ func (c *Controller) plaidTokenCallback(ctx iris.Context) { return } - plaidAccounts, err := c.plaid.GetAccounts(c.getContext(ctx), result.AccessToken, plaid.GetAccountsOptions{ - AccountIDs: callbackRequest.AccountIds, - }) - if err != nil { - c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to retrieve accounts") - return - } - - if len(plaidAccounts) == 0 { - c.returnError(ctx, http.StatusInternalServerError, "could not retrieve details for any accounts") - return - } - repo := c.mustGetAuthenticatedRepository(ctx) var webhook string @@ -452,7 +360,7 @@ func (c *Controller) plaidTokenCallback(ctx iris.Context) { if err = c.plaidSecrets.UpdateAccessTokenForPlaidLinkId( c.getContext(ctx), repo.AccountId(), - result.ItemID, + result.ItemId, result.AccessToken, ); err != nil { log.WithError(err).Errorf("failed to store access token") @@ -461,7 +369,7 @@ func (c *Controller) plaidTokenCallback(ctx iris.Context) { } plaidLink := models.PlaidLink{ - ItemId: result.ItemID, + ItemId: result.ItemId, Products: []string{ // TODO (elliotcourant) Make this based on what product's we sent in the create link token request. "transactions", @@ -488,22 +396,42 @@ func (c *Controller) plaidTokenCallback(ctx iris.Context) { return } + // Create a plaid client for the new link. + client, err := c.plaid.NewClient(c.getContext(ctx), &link, result.AccessToken) + if err != nil { + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create Plaid client") + return + } + + // Then use that client to retrieve that link's bank accounts. + plaidAccounts, err := client.GetAccounts(c.getContext(ctx), callbackRequest.AccountIds...) + if err != nil { + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to retrieve accounts") + return + } + + if len(plaidAccounts) == 0 { + c.returnError(ctx, http.StatusInternalServerError, "could not retrieve details for any accounts") + return + } + now := time.Now().UTC() accounts := make([]models.BankAccount, len(plaidAccounts)) for i, plaidAccount := range plaidAccounts { accounts[i] = models.BankAccount{ AccountId: repo.AccountId(), LinkId: link.LinkId, - PlaidAccountId: plaidAccount.AccountID, - AvailableBalance: int64(plaidAccount.Balances.Available * 100), - CurrentBalance: int64(plaidAccount.Balances.Current * 100), - Name: plaidAccount.Name, - Mask: plaidAccount.Mask, - PlaidName: plaidAccount.Name, - PlaidOfficialName: plaidAccount.OfficialName, - Type: models.BankAccountType(plaidAccount.Type), - SubType: models.BankAccountSubType(plaidAccount.Subtype), - LastUpdated: now, + PlaidAccountId: plaidAccount.GetAccountId(), + AvailableBalance: plaidAccount.GetBalances().GetAvailable(), + CurrentBalance: plaidAccount.GetBalances().GetCurrent(), + Name: plaidAccount.GetName(), + Mask: plaidAccount.GetMask(), + PlaidName: plaidAccount.GetName(), + PlaidOfficialName: plaidAccount.GetOfficialName(), + // THIS MIGHT BREAK SOMETHING. + //Type: models.BankAccountType(plaidAccount.Type), + //SubType: models.BankAccountSubType(plaidAccount.Subtype), + LastUpdated: now, } } if err = repo.CreateBankAccounts(c.getContext(ctx), accounts...); err != nil { diff --git a/pkg/internal/cmd/controllers.go b/pkg/internal/cmd/controllers.go index b03ce59d..1f1323f7 100644 --- a/pkg/internal/cmd/controllers.go +++ b/pkg/internal/cmd/controllers.go @@ -9,7 +9,7 @@ import ( "github.com/monetr/rest-api/pkg/billing" "github.com/monetr/rest-api/pkg/config" "github.com/monetr/rest-api/pkg/controller" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/jobs" "github.com/monetr/rest-api/pkg/metrics" @@ -22,7 +22,7 @@ func getControllers( configuration config.Configuration, db *pg.DB, job jobs.JobManager, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, stripe stripe_helper.Stripe, cache *redis.Pool, diff --git a/pkg/internal/cmd/controllers_ui.go b/pkg/internal/cmd/controllers_ui.go index 561f23d1..da3c5f55 100644 --- a/pkg/internal/cmd/controllers_ui.go +++ b/pkg/internal/cmd/controllers_ui.go @@ -9,7 +9,7 @@ import ( "github.com/monetr/rest-api/pkg/billing" "github.com/monetr/rest-api/pkg/config" "github.com/monetr/rest-api/pkg/controller" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/jobs" "github.com/monetr/rest-api/pkg/metrics" @@ -23,7 +23,7 @@ func getControllers( configuration config.Configuration, db *pg.DB, job jobs.JobManager, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, stripe stripe_helper.Stripe, cache *redis.Pool, diff --git a/pkg/internal/cmd/serve.go b/pkg/internal/cmd/serve.go index f84d6a23..4ca325af 100644 --- a/pkg/internal/cmd/serve.go +++ b/pkg/internal/cmd/serve.go @@ -6,13 +6,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" - "net" - "net/http" - "os" - "path/filepath" - "time" - "github.com/getsentry/sentry-go" "github.com/go-pg/pg/v10" "github.com/kataras/iris/v12" @@ -24,16 +17,21 @@ import ( "github.com/monetr/rest-api/pkg/internal/certhelper" "github.com/monetr/rest-api/pkg/internal/migrations" "github.com/monetr/rest-api/pkg/internal/myownsanity" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/internal/vault_helper" "github.com/monetr/rest-api/pkg/jobs" "github.com/monetr/rest-api/pkg/logging" "github.com/monetr/rest-api/pkg/metrics" + "github.com/monetr/rest-api/pkg/repository" "github.com/monetr/rest-api/pkg/secrets" "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" "github.com/spf13/cobra" + "io/ioutil" + "net" + "os" + "path/filepath" + "time" ) func init() { @@ -73,11 +71,13 @@ func RunServer() error { var vault vault_helper.VaultHelper if configuration.Vault.Enabled { client, err := vault_helper.NewVaultHelper(log, vault_helper.Config{ - Address: "http://vault.monetr.in:443", - Role: "rest-api", - Auth: "kubernetes", + Address: configuration.Vault.Address, + Role: configuration.Vault.Role, + Auth: configuration.Vault.Auth, Timeout: 30 * time.Second, IdleConnTimeout: 9 * time.Minute, + Username: configuration.Vault.Username, + Password: configuration.Vault.Password, }) if err != nil { log.WithError(err).Fatalf("failed to create vault helper") @@ -315,15 +315,6 @@ func RunServer() error { basicPaywall = billing.NewBasicPaywall(log, accountRepo) } - plaidHelper := plaid_helper.NewPlaidClient(log, plaid.ClientOptions{ - ClientID: configuration.Plaid.ClientID, - Secret: configuration.Plaid.ClientSecret, - Environment: configuration.Plaid.Environment, - HTTPClient: &http.Client{ - Timeout: 30 * time.Second, - }, - }) - if configuration.Plaid.WebhooksEnabled { log.Debugf("plaid webhooks are enabled and will be sent to: %s", configuration.Plaid.WebhooksDomain) } @@ -339,11 +330,13 @@ func RunServer() error { plaidSecrets = secrets.NewPostgresPlaidSecretsProvider(log, db) } + plaidClient := platypus.NewPlaid(log, plaidSecrets, repository.NewPlaidRepository(db), configuration.Plaid) + jobManager := jobs.NewJobManager( log, redisController.Pool(), db, - plaidHelper, + plaidClient, stats, plaidSecrets, ) @@ -354,7 +347,7 @@ func RunServer() error { configuration, db, jobManager, - plaidHelper, + plaidClient, stats, stripe, redisController.Pool(), diff --git a/pkg/internal/mock_http_helper/responder.go b/pkg/internal/mock_http_helper/responder.go index 1bff2325..e2386fad 100644 --- a/pkg/internal/mock_http_helper/responder.go +++ b/pkg/internal/mock_http_helper/responder.go @@ -17,7 +17,12 @@ func NewHttpMockJsonResponder( headersFn func(t *testing.T, request *http.Request, response interface{}, status int) map[string][]string, ) { httpmock.RegisterResponder(method, path, func(request *http.Request) (*http.Response, error) { - result, status := handler(t, request) + var result interface{} + var status int + require.NotPanics(t, func() { + result, status = handler(t, request) + }, "handle must not panic") + body := bytes.NewBuffer(nil) require.NoError(t, json.NewEncoder(body).Encode(result), "must encode response body") response := &http.Response{ @@ -28,6 +33,11 @@ func NewHttpMockJsonResponder( Body: ioutil.NopCloser(body), ContentLength: int64(body.Len()), Request: request, + Header: map[string][]string{ + "Content-Type": { + "application/json", + }, + }, } headers := headersFn(t, request, result, status) diff --git a/pkg/internal/mock_http_helper/responder_test.go b/pkg/internal/mock_http_helper/responder_test.go new file mode 100644 index 00000000..fe80c6c7 --- /dev/null +++ b/pkg/internal/mock_http_helper/responder_test.go @@ -0,0 +1,39 @@ +package mock_http_helper + +import ( + "encoding/json" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" + "io/ioutil" + "net/http" + "testing" +) + +func TestNewHttpMockJsonResponder(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + url := "https://monetr.test/thing" + NewHttpMockJsonResponder(t, "GET", url, func(t *testing.T, request *http.Request) (interface{}, int) { + return map[string]interface{}{ + "value": 123, + }, http.StatusOK + }, func(t *testing.T, request *http.Request, response interface{}, status int) map[string][]string { + return nil + }) + + response, err := http.Get(url) + assert.NoError(t, err, "http get request must succeed") + assert.Equal(t, http.StatusOK, response.StatusCode, "status code must be 200") + + body, err := ioutil.ReadAll(response.Body) + assert.NoError(t, err, "must be able to read the response body") + assert.NotEmpty(t, body, "body must not be empty") + + var result map[string]interface{} + err = json.Unmarshal(body, &result) + assert.NoError(t, err, "must be able to unmarshal response") + + assert.EqualValues(t, 123, result["value"], "value must match") + assert.Len(t, result, 1, "must only have one key") +} diff --git a/pkg/internal/mock_mail/mail_test.go b/pkg/internal/mock_mail/mail_test.go index 1d5b8919..2cea9921 100644 --- a/pkg/internal/mock_mail/mail_test.go +++ b/pkg/internal/mock_mail/mail_test.go @@ -25,7 +25,7 @@ func TestMockMailCommunication_Send(t *testing.T) { assert.Len(t, mockMail.Sent, 1, "should have added request to list") }) - t.Run("simple", func(t *testing.T) { + t.Run("none sent", func(t *testing.T) { mockMail := NewMockMail() mockMail.ShouldFail = true diff --git a/pkg/internal/mock_plaid/accounts.go b/pkg/internal/mock_plaid/accounts.go index 9d47e09b..29f3f22c 100644 --- a/pkg/internal/mock_plaid/accounts.go +++ b/pkg/internal/mock_plaid/accounts.go @@ -2,42 +2,104 @@ package mock_plaid import ( "encoding/json" + "fmt" + "github.com/brianvoe/gofakeit/v6" "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/internal/testutils" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" "net/http" + "strings" "testing" ) +func BankAccountFixture(t *testing.T) plaid.AccountBase { + accountNumber := gofakeit.AchAccount() + require.NotEmpty(t, accountNumber, "account number cannot be empty") + + accountType := gofakeit.RandomString([]string{ + string(plaid.ACCOUNTTYPE_DEPOSITORY), + string(plaid.ACCOUNTTYPE_CREDIT), + string(plaid.ACCOUNTTYPE_INVESTMENT), + string(plaid.ACCOUNTTYPE_LOAN), + }) + + var accountSubType plaid.AccountSubtype + switch plaid.AccountType(accountType) { + case plaid.ACCOUNTTYPE_DEPOSITORY: + accountSubType = plaid.AccountSubtype(gofakeit.RandomString([]string{ + string(plaid.ACCOUNTSUBTYPE_CHECKING), + string(plaid.ACCOUNTSUBTYPE_SAVINGS), + string(plaid.ACCOUNTSUBTYPE_PAYPAL), + })) + case plaid.ACCOUNTTYPE_CREDIT: + accountSubType = plaid.AccountSubtype(gofakeit.RandomString([]string{ + string(plaid.ACCOUNTSUBTYPE_CREDIT_CARD), + string(plaid.ACCOUNTSUBTYPE_PAYPAL), + })) + case plaid.ACCOUNTTYPE_INVESTMENT: + accountSubType = plaid.AccountSubtype(gofakeit.RandomString([]string{ + string(plaid.ACCOUNTSUBTYPE_IRA), + string(plaid.ACCOUNTSUBTYPE_ROTH), + })) + case plaid.ACCOUNTTYPE_LOAN: + accountSubType = plaid.AccountSubtype(gofakeit.RandomString([]string{ + string(plaid.ACCOUNTSUBTYPE_AUTO), + string(plaid.ACCOUNTSUBTYPE_HOME), + })) + } + + mask := accountNumber[len(accountNumber)-4:] + + currencyCode := "USD" + + current := gofakeit.Float32Range(100, 500) + available := gofakeit.Float32Range(current-10, current) + limit := gofakeit.Float32Range(current, current+100) + + return plaid.AccountBase{ + AccountId: gofakeit.Generate("????????????????"), + Balances: plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(myownsanity.Float32P(available)), + Current: *plaid.NewNullableFloat32(myownsanity.Float32P(current)), + Limit: *plaid.NewNullableFloat32(myownsanity.Float32P(limit)), + IsoCurrencyCode: *plaid.NewNullableString(myownsanity.StringP(currencyCode)), + UnofficialCurrencyCode: *plaid.NewNullableString(myownsanity.StringP(currencyCode)), + }, + Mask: *plaid.NewNullableString(myownsanity.StringP(mask)), + Name: fmt.Sprintf("Personal Account - %s", mask), + OfficialName: *plaid.NewNullableString(myownsanity.StringP(fmt.Sprintf("%s - %s", strings.ToUpper(accountType), mask))), + Type: plaid.AccountType(accountType), + Subtype: *plaid.NewNullableAccountSubtype(&accountSubType), + } +} + func MockGetAccountsExtended(t *testing.T, plaidData *testutils.MockPlaidData) { mock_http_helper.NewHttpMockJsonResponder( t, "POST", Path(t, "/accounts/get"), func(t *testing.T, request *http.Request) (interface{}, int) { + accessToken := ValidatePlaidAuthentication(t, request, RequireAccessToken) var getAccountsRequest struct { - ClientId string `json:"client_id"` - Secret string `json:"secret"` - AccessToken string `json:"access_token"` Options struct { - AccountIds []string `json:"account_ids"` + AccountIds []string `json:"account_ids"` } `json:"options"` } require.NoError(t, json.NewDecoder(request.Body).Decode(&getAccountsRequest), "must decode request") - accounts, ok := plaidData.BankAccounts[getAccountsRequest.AccessToken] - if !ok { - panic("invalid access token mocking not implemented") - } + accounts, ok := plaidData.BankAccounts[accessToken] + require.True(t, ok, "invalid access token mocking not implemented") - response := plaid.GetAccountsResponse{ - Accounts: make([]plaid.Account, 0), - Item: plaid.Item{}, // Not yet populating this. + response := plaid.AccountsGetResponse{ + RequestId: gofakeit.UUID(), + Accounts: make([]plaid.AccountBase, 0), + Item: plaid.Item{}, // Not yet populating this. } for _, accountId := range getAccountsRequest.Options.AccountIds { account, ok := accounts[accountId] if !ok { - panic("bad account id handling not yet implemented") + panic("bad account id handling not yet implemented") } response.Accounts = append(response.Accounts, account) @@ -49,7 +111,7 @@ func MockGetAccountsExtended(t *testing.T, plaidData *testutils.MockPlaidData) { ) } -func MockGetAccounts(t *testing.T, accounts []plaid.Account) { +func MockGetAccounts(t *testing.T, accounts []plaid.AccountBase) { mock_http_helper.NewHttpMockJsonResponder( t, "POST", Path(t, "/accounts/get"), @@ -87,7 +149,14 @@ func MockGetAccountsError(t *testing.T, plaidError plaid.Error) { } require.NoError(t, json.NewDecoder(request.Body).Decode(&getAccountsRequest), "must decode request") - return plaidError, plaidError.StatusCode + var status int + if s := plaidError.Status.Get(); s != nil { + status = int(*s) + } else { + status = http.StatusInternalServerError + } + + return plaidError, status }, PlaidHeaders, ) diff --git a/pkg/internal/mock_plaid/accounts_test.go b/pkg/internal/mock_plaid/accounts_test.go new file mode 100644 index 00000000..a86635e8 --- /dev/null +++ b/pkg/internal/mock_plaid/accounts_test.go @@ -0,0 +1,11 @@ +package mock_plaid + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestBankAccountFixture(t *testing.T) { + account := BankAccountFixture(t) + assert.NotEmpty(t, account, "account must not be empty") +} diff --git a/pkg/internal/mock_plaid/authentication.go b/pkg/internal/mock_plaid/authentication.go new file mode 100644 index 00000000..55e8677b --- /dev/null +++ b/pkg/internal/mock_plaid/authentication.go @@ -0,0 +1,43 @@ +package mock_plaid + +import ( + "bytes" + "encoding/json" + "github.com/stretchr/testify/require" + "io" + "net/http" + "strings" + "testing" +) + +const ( + RequireAccessToken = true + DoNotRequireAccessToken = false +) + +func ValidatePlaidAuthentication(t *testing.T, request *http.Request, requireAccessToken bool) (accessToken string) { + split := bytes.NewBuffer(nil) + bodyReader := io.TeeReader(request.Body, split) + request.Body = io.NopCloser(split) + + var authenticationInBody struct { + ClientId string `json:"client_id"` + Secret string `json:"secret"` + AccessToken string `json:"access_token"` + } + require.NoError(t, json.NewDecoder(bodyReader).Decode(&authenticationInBody), "must decode request") + + if strings.TrimSpace(authenticationInBody.ClientId) == "" { + require.NotEmpty(t, request.Header.Get("Plaid-Client-Id"), "client Id cannot be missing") + } + + if strings.TrimSpace(authenticationInBody.Secret) == "" { + require.NotEmpty(t, request.Header.Get("Plaid-Secret"), "secret cannot be missing") + } + + if requireAccessToken { + require.NotEmpty(t, authenticationInBody.AccessToken, "access token is required") + } + + return authenticationInBody.AccessToken +} diff --git a/pkg/internal/mock_plaid/link.go b/pkg/internal/mock_plaid/link.go new file mode 100644 index 00000000..ec12de75 --- /dev/null +++ b/pkg/internal/mock_plaid/link.go @@ -0,0 +1,33 @@ +package mock_plaid + +import ( + "encoding/json" + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/require" + "net/http" + "testing" + "time" +) + +func MockCreateLinkToken(t *testing.T) { + mock_http_helper.NewHttpMockJsonResponder( + t, + "POST", Path(t, "/link/token/create"), + func(t *testing.T, request *http.Request) (interface{}, int) { + ValidatePlaidAuthentication(t, request, DoNotRequireAccessToken) + var createLinkTokenRequest plaid.LinkTokenCreateRequest + require.NoError(t, json.NewDecoder(request.Body).Decode(&createLinkTokenRequest), "must decode request") + require.NotEmpty(t, createLinkTokenRequest.ClientName, "client name is required") + require.NotEmpty(t, createLinkTokenRequest.Language, "language is required") + + return plaid.LinkTokenCreateResponse{ + LinkToken: gofakeit.UUID(), + Expiration: time.Now().Add(30 * time.Second), + RequestId: gofakeit.UUID(), + }, http.StatusOK + }, + PlaidHeaders, + ) +} diff --git a/pkg/internal/mock_plaid/mock_exchange.go b/pkg/internal/mock_plaid/mock_exchange.go index e29cde35..b6f5b2e5 100644 --- a/pkg/internal/mock_plaid/mock_exchange.go +++ b/pkg/internal/mock_plaid/mock_exchange.go @@ -4,6 +4,7 @@ import ( "encoding/json" "github.com/brianvoe/gofakeit/v6" "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" "net/http" @@ -20,26 +21,28 @@ func MockExchangePublicToken(t *testing.T) string { t, "POST", Path(t, "/item/public_token/exchange"), func(t *testing.T, request *http.Request) (interface{}, int) { + ValidatePlaidAuthentication(t, request, DoNotRequireAccessToken) var exchangeRequest struct { - ClientID string `json:"client_id"` - Secret string `json:"secret"` PublicToken string `json:"public_token"` } require.NoError(t, json.NewDecoder(request.Body).Decode(&exchangeRequest), "must decode request") + requestId := gofakeit.UUID() if exchangeRequest.PublicToken != publicToken { return plaid.Error{ - ErrorType: "INVALID_REQUEST", - ErrorCode: "1234", - ErrorMessage: "public_token is not valid", - DisplayMessage: "public_token is not valid", - StatusCode: http.StatusBadRequest, + RequestId: &requestId, + ErrorType: "INVALID_REQUEST", + ErrorCode: "1234", + ErrorMessage: "public_token is not valid", + DisplayMessage: *plaid.NewNullableString(myownsanity.StringP("public_token is not valid")), + Status: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(http.StatusBadRequest))), }, http.StatusBadRequest } - return plaid.ExchangePublicTokenResponse{ + return plaid.ItemPublicTokenExchangeResponse{ + RequestId: requestId, AccessToken: gofakeit.UUID(), - ItemID: gofakeit.UUID(), + ItemId: gofakeit.UUID(), }, http.StatusOK }, PlaidHeaders, diff --git a/pkg/internal/mock_plaid/transactions.go b/pkg/internal/mock_plaid/transactions.go new file mode 100644 index 00000000..dc14b415 --- /dev/null +++ b/pkg/internal/mock_plaid/transactions.go @@ -0,0 +1,87 @@ +package mock_plaid + +import ( + "encoding/json" + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/require" + "net/http" + "testing" + "time" +) + +func GenerateTransactions(t *testing.T, start, end time.Time, numberOfTransactions int, bankAccountIds []string) []plaid.Transaction { + transactions := make([]plaid.Transaction, numberOfTransactions*len(bankAccountIds)) + for i := 0; i < len(transactions); i++ { + bankAccountId := bankAccountIds[i%len(bankAccountIds)] + + transaction := plaid.Transaction{} + transaction.SetAmount(gofakeit.Float32Range(0.99, 100)) + transaction.SetCategory([]string{ + "Bank Fees", + }) + transaction.SetCategoryId("10000000") + transaction.SetAccountId(bankAccountId) + transaction.SetIsoCurrencyCode("USD") + transaction.SetUnofficialCurrencyCode("USD") + + // Should break down transaction dates evenly over the provided range. + down := end.Add(-(end.Sub(start) / time.Duration(numberOfTransactions*len(bankAccountIds))) * time.Duration(i)) + + transaction.SetDate(down.Format("2006-01-02")) + transaction.SetName(gofakeit.Company()) + transaction.SetTransactionId(gofakeit.Generate("?????????????????????")) + + transactions[i] = transaction + } + + return transactions +} + +func MockGetRandomTransactions(t *testing.T, start, end time.Time, numberOfTransactions int, bankAccountIds []string) { + transactions := GenerateTransactions(t, start, end, numberOfTransactions, bankAccountIds) + mock_http_helper.NewHttpMockJsonResponder( + t, + "POST", Path(t, "/transactions/get"), + func(t *testing.T, request *http.Request) (interface{}, int) { + ValidatePlaidAuthentication(t, request, RequireAccessToken) + var getTransactionsRequest struct { + Options struct { + AccountIds []string `json:"account_ids"` + Count int `json:"count"` + Offset int `json:"offset"` + } `json:"options"` + StartDate string `json:"start_date"` + EndDate string `json:"end_date"` + } + require.NoError(t, json.NewDecoder(request.Body).Decode(&getTransactionsRequest), "must decode request") + + // Make sure our request dates are valid. + _, err := time.Parse("2006-01-02", getTransactionsRequest.StartDate) + require.NoError(t, err, "must provide a valid start date") + _, err = time.Parse("2006-01-02", getTransactionsRequest.EndDate) + require.NoError(t, err, "must provide a valid end date") + + if getTransactionsRequest.Options.Offset > len(transactions) { + return plaid.TransactionsGetResponse{}, http.StatusOK + } + + offset := getTransactionsRequest.Options.Offset + count := getTransactionsRequest.Options.Count + endingOffset := myownsanity.Min(len(transactions), offset+count) + data := transactions[offset:endingOffset] + + return plaid.TransactionsGetResponse{ + Accounts: nil, // Add some basic reporting here too + Transactions: data, + TotalTransactions: int32(len(transactions)), + Item: plaid.Item{}, + RequestId: gofakeit.UUID(), + AdditionalProperties: nil, + }, http.StatusOK + }, + PlaidHeaders, + ) +} diff --git a/pkg/internal/mock_plaid/transactions_test.go b/pkg/internal/mock_plaid/transactions_test.go new file mode 100644 index 00000000..ccce9dec --- /dev/null +++ b/pkg/internal/mock_plaid/transactions_test.go @@ -0,0 +1,37 @@ +package mock_plaid + +import ( + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestGenerateTransactions(t *testing.T) { + t.Run("2 days", func(t *testing.T) { + numberOfTransactions := 600 + bankAccounts := []string{ + "1234", + "5678", + } + end := time.Now() + start := time.Now().Add(-25 * time.Hour) + transactions := GenerateTransactions(t, start, end, numberOfTransactions, bankAccounts) + assert.Len(t, transactions, len(bankAccounts) * numberOfTransactions) + assert.Equal(t, end.Format("2006-01-02"), transactions[0].GetDate(), "date of first transaction should be end") + assert.Equal(t, start.Format("2006-01-02"), transactions[len(transactions)-1].GetDate(), "date of last transaction should be start") + }) + + t.Run("30 days", func(t *testing.T) { + numberOfTransactions := 3000 + bankAccounts := []string{ + "1234", + "5678", + } + end := time.Now() + start := time.Now().Add((-30 * 24 * time.Hour) + (-1 * time.Hour)) + transactions := GenerateTransactions(t, start, end, numberOfTransactions, bankAccounts) + assert.Len(t, transactions, len(bankAccounts) * numberOfTransactions) + assert.Equal(t, end.Format("2006-01-02"), transactions[0].GetDate(), "date of first transaction should be end") + assert.Equal(t, start.Format("2006-01-02"), transactions[len(transactions)-1].GetDate(), "date of last transaction should be start") + }) +} diff --git a/pkg/internal/mock_plaid/verification.go b/pkg/internal/mock_plaid/verification.go new file mode 100644 index 00000000..bbf2b4a1 --- /dev/null +++ b/pkg/internal/mock_plaid/verification.go @@ -0,0 +1,43 @@ +package mock_plaid + +import ( + "encoding/json" + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/require" + "net/http" + "testing" + "time" +) + +func MockGetWebhookVerificationKey(t *testing.T) { + mock_http_helper.NewHttpMockJsonResponder(t, + "POST", Path(t, "/webhook_verification_key/get"), + func(t *testing.T, request *http.Request) (interface{}, int) { + ValidatePlaidAuthentication(t, request, DoNotRequireAccessToken) + var getWebhookVerificationKeyRequest struct { + KeyId string `json:"kid"` + } + require.NoError(t, json.NewDecoder(request.Body).Decode(&getWebhookVerificationKeyRequest), "must decode request") + + return plaid.WebhookVerificationKeyGetResponse{ + Key: plaid.JWKPublicKey{ + Alg: "", + Crv: "", + Kid: "", + Kty: "", + Use: "", + X: "", + Y: "", + CreatedAt: int32(time.Now().Unix()), + ExpiredAt: *plaid.NewNullableInt32(myownsanity.Int32P(int32(time.Now().Add(10 * time.Second).Unix()))), + }, + RequestId: gofakeit.UUID(), + AdditionalProperties: nil, + }, http.StatusOK + }, + PlaidHeaders, + ) +} diff --git a/pkg/internal/myownsanity/number_test.go b/pkg/internal/myownsanity/number_test.go new file mode 100644 index 00000000..8e166385 --- /dev/null +++ b/pkg/internal/myownsanity/number_test.go @@ -0,0 +1,12 @@ +package myownsanity + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestMax(t *testing.T) { + assert.Equal(t, 2, Max(1, 2)) + assert.Equal(t, 1000, Max(1000, 100)) + assert.Equal(t, 500, Max(500, 500)) +} diff --git a/pkg/internal/myownsanity/numbers.go b/pkg/internal/myownsanity/numbers.go new file mode 100644 index 00000000..e0e68cb4 --- /dev/null +++ b/pkg/internal/myownsanity/numbers.go @@ -0,0 +1,25 @@ +package myownsanity + +func Float32P(value float32) *float32 { + return &value +} + +func Int32P(value int32) *int32 { + return &value +} + +func Max(a, b int) int { + if a > b { + return a + } + + return b +} + +func Min(a, b int) int { + if a < b { + return a + } + + return b +} \ No newline at end of file diff --git a/pkg/internal/plaid_helper/client.go b/pkg/internal/plaid_helper/client.go deleted file mode 100644 index 4deecad2..00000000 --- a/pkg/internal/plaid_helper/client.go +++ /dev/null @@ -1,303 +0,0 @@ -package plaid_helper - -import ( - "context" - "github.com/getsentry/sentry-go" - "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" - "github.com/sirupsen/logrus" - "strings" - "time" -) - -type Client interface { - CreateLinkToken(ctx context.Context, config plaid.LinkTokenConfigs) (*plaid.CreateLinkTokenResponse, error) - ExchangePublicToken(ctx context.Context, publicToken string) (*plaid.ExchangePublicTokenResponse, error) - GetAccounts(ctx context.Context, accessToken string, options plaid.GetAccountsOptions) ([]plaid.Account, error) - GetAllTransactions(ctx context.Context, accessToken string, start, end time.Time, accountIds []string) ([]plaid.Transaction, error) - GetAllInstitutions(ctx context.Context, countryCodes []string, options plaid.GetInstitutionsOptions) ([]plaid.Institution, error) - GetInstitutions(ctx context.Context, count, offset int, countryCodes []string, options plaid.GetInstitutionsOptions) (total int, _ []plaid.Institution, _ error) - GetInstitution(ctx context.Context, institutionId string, includeMetadata bool, countryCodes []string) (*plaid.Institution, error) - GetWebhookVerificationKey(ctx context.Context, keyId string) (plaid.GetWebhookVerificationKeyResponse, error) - RemoveItem(ctx context.Context, accessToken string) error - Close() error -} - -var ( - _ Client = &plaidClient{} -) - -func NewPlaidClient(log *logrus.Entry, options plaid.ClientOptions) Client { - client, err := plaid.NewClient(options) - if err != nil { - // There currently isn't a code path that actually returns an error from the client. So if something happens - // then its new. - panic(err) - } - - return &plaidClient{ - log: log, - client: client, - institutionTicker: time.NewTicker(2400 * time.Millisecond), // Limit our institution API calls to 25 per minute. - } -} - -type plaidClient struct { - log *logrus.Entry - client *plaid.Client - institutionTicker *time.Ticker -} - -func (p *plaidClient) CreateLinkToken(ctx context.Context, config plaid.LinkTokenConfigs) (*plaid.CreateLinkTokenResponse, error) { - span := sentry.StartSpan(ctx, "Plaid - CreateLinkToken") - defer span.Finish() - - span.Data = map[string]interface{}{} - - result, err := p.client.CreateLinkToken(config) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - err = errors.Wrap(err, "failed to create link token") - } else { - span.Status = sentry.SpanStatusOK - } - - return &result, err -} - -func (p *plaidClient) ExchangePublicToken(ctx context.Context, publicToken string) (*plaid.ExchangePublicTokenResponse, error) { - span := sentry.StartSpan(ctx, "Plaid - ExchangePublicToken") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - - result, err := p.client.ExchangePublicToken(publicToken) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - return nil, errors.Wrap(err, "failed to exchange public token") - } - - span.Status = sentry.SpanStatusOK - - return &result, nil -} - -func (p *plaidClient) GetAccounts(ctx context.Context, accessToken string, options plaid.GetAccountsOptions) ([]plaid.Account, error) { - span := sentry.StartSpan(ctx, "Plaid - GetAccounts") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - span.Data["options"] = options - - result, err := p.client.GetAccountsWithOptions(accessToken, options) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - return nil, errors.Wrap(err, "failed to retrieve plaid accounts") - } - - return result.Accounts, nil -} - -func (p *plaidClient) GetAllTransactions(ctx context.Context, accessToken string, start, end time.Time, accountIds []string) ([]plaid.Transaction, error) { - span := sentry.StartSpan(ctx, "Plaid - GetAllTransactions") - defer span.Finish() - span.Data = map[string]interface{}{ - "start": start, - "end": end, - } - if len(accountIds) > 0 { - span.Data["accountIds"] = accountIds - } - - perPage := 100 - - options := plaid.GetTransactionsOptions{ - StartDate: start.Format("2006-01-02"), - EndDate: end.Format("2006-01-02"), - AccountIDs: accountIds, - Count: perPage, - Offset: 0, - } - - transactions := make([]plaid.Transaction, 0) - for { - options.Offset = len(transactions) - total, items, err := p.GetTransactions(span.Context(), accessToken, options) - if err != nil { - return nil, err - } - - transactions = append(transactions, items...) - - if len(items) < perPage { - break - } - - if len(transactions) >= total { - break - } - } - - return transactions, nil -} - -func (p *plaidClient) GetTransactions(ctx context.Context, accessToken string, options plaid.GetTransactionsOptions) (total int, _ []plaid.Transaction, _ error) { - span := sentry.StartSpan(ctx, "Plaid - GetTransactions") - defer span.Finish() - span.Data = map[string]interface{}{ - "options": options, - } - - result, err := p.client.GetTransactionsWithOptions(accessToken, options) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - return 0, nil, errors.Wrap(err, "failed to retrieve plaid transactions") - } - - return result.TotalTransactions, result.Transactions, nil -} - -func (p *plaidClient) GetInstitution(ctx context.Context, institutionId string, includeMetadata bool, countryCodes []string) (*plaid.Institution, error) { - span := sentry.StartSpan(ctx, "Plaid - GetInstitution") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - - span.Data["institutionId"] = institutionId - span.Data["includeMetadata"] = includeMetadata - span.Data["countryCodes"] = countryCodes - - result, err := p.client.GetInstitutionByIDWithOptions(institutionId, countryCodes, plaid.GetInstitutionByIDOptions{ - IncludeOptionalMetadata: true, - IncludePaymentInitiationMetadata: false, - IncludeStatus: false, - }) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - return nil, errors.Wrap(err, "failed to retrieve plaid institution") - } - - return &result.Institution, nil -} - -func (p *plaidClient) GetAllInstitutions(ctx context.Context, countryCodes []string, options plaid.GetInstitutionsOptions) ([]plaid.Institution, error) { - span := sentry.StartSpan(ctx, "Plaid - GetAllInstitutions") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - span.Data["countryCodes"] = countryCodes - span.Data["options"] = options - - perPage := 500 - institutions := make([]plaid.Institution, 0) - for { - total, items, err := p.GetInstitutions(span.Context(), perPage, len(institutions), countryCodes, options) - if err != nil { - span.Status = sentry.SpanStatusInternalError - return nil, err - } - - institutions = append(institutions, items...) - - // If we received fewer items than we requested, then we have reached the end. - if len(items) < perPage { - break - } - - // If we have received at least what we expect to be the total amount, then we are also done. - if len(institutions) >= total { - break - } - } - - return institutions, nil -} - -func (p *plaidClient) GetInstitutions(ctx context.Context, count, offset int, countryCodes []string, options plaid.GetInstitutionsOptions) (total int, _ []plaid.Institution, _ error) { - span := sentry.StartSpan(ctx, "Plaid - GetInstitutions") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - - span.Data["count"] = count - span.Data["offset"] = offset - span.Data["countryCodes"] = countryCodes - span.Data["options"] = options - - log := p.log.WithFields(logrus.Fields{ - "count": count, - "offset": offset, - "countryCodes": strings.Join(countryCodes, ","), - }) - - log.Debug("retrieving plaid institutions") - - rateLimitTimeout := time.NewTimer(30 * time.Second) - select { - // The institution ticker handles rate limiting for the get institutions endpoint. It makes sure that even - // concurrently, we should not be able to exceed our request limit. At least on a single replica. - case <-p.institutionTicker.C: - result, err := p.client.GetInstitutionsWithOptions(count, offset, countryCodes, options) - span.Data["plaidRequestId"] = result.RequestID - log = log.WithField("plaidRequestId", result.RequestID) - if err != nil { - span.Status = sentry.SpanStatusInternalError - log.WithError(err).Errorf("failed to retrieve plaid institutions") - return 0, nil, errors.Wrap(err, "failed to retrieve plaid institutions") - } - - log.Debugf("successfully retrieved %d institutions", len(result.Institutions)) - - span.Status = sentry.SpanStatusOK - - return result.Total, result.Institutions, nil - case <-rateLimitTimeout.C: - return 0, nil, errors.Errorf("timed out waiting for rate limit") - } -} - -func (p *plaidClient) GetWebhookVerificationKey(ctx context.Context, keyId string) (plaid.GetWebhookVerificationKeyResponse, error) { - span := sentry.StartSpan(ctx, "Plaid - GetWebhookVerificationKey") - defer span.Finish() - span.Data = map[string]interface{}{} - - result, err := p.client.GetWebhookVerificationKey(keyId) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - } else { - span.Status = sentry.SpanStatusOK - } - - return result, errors.Wrap(err, "failed to retrieve webhook verification key") -} - -func (p *plaidClient) RemoveItem(ctx context.Context, accessToken string) error { - span := sentry.StartSpan(ctx, "Plaid - RemoveItem") - defer span.Finish() - span.Data = map[string]interface{}{} - - result, err := p.client.RemoveItem(accessToken) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - } else { - span.Status = sentry.SpanStatusOK - } - - return errors.Wrap(err, "failed to remove Plaid item") -} - -func (p *plaidClient) Close() error { - p.client = nil - p.institutionTicker.Stop() - return nil -} diff --git a/pkg/internal/plaid_helper/client_test.go b/pkg/internal/plaid_helper/client_test.go deleted file mode 100644 index 89b0c03d..00000000 --- a/pkg/internal/plaid_helper/client_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package plaid_helper - -import ( - "context" - "github.com/brianvoe/gofakeit/v6" - "github.com/jarcoal/httpmock" - "github.com/monetr/rest-api/pkg/internal/mock_plaid" - "github.com/monetr/rest-api/pkg/internal/testutils" - "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" - "github.com/stretchr/testify/assert" - "net/http" - "testing" -) - -func TestPlaidClient_GetAccounts(t *testing.T) { - t.Run("simple", func(t *testing.T) { - httpmock.Activate() - defer httpmock.Deactivate() - - account := plaid.Account{ - AccountID: gofakeit.UUID(), - Balances: plaid.AccountBalances{ - Available: 10.00, - Current: 10.00, - Limit: 10.00, - ISOCurrencyCode: "USD", - UnofficialCurrencyCode: "USD", - }, - Mask: "1234", - Name: "Checking account", - OfficialName: "Super duper checking", - Subtype: "checking", - Type: "depository", - VerificationStatus: "verified?", - } - - mock_plaid.MockGetAccounts(t, []plaid.Account{ - account, - }) - - client := NewPlaidClient(testutils.GetLog(t), plaid.ClientOptions{ - ClientID: gofakeit.UUID(), - Secret: gofakeit.UUID(), - Environment: plaid.Sandbox, - HTTPClient: &http.Client{ - Transport: httpmock.DefaultTransport, - }, - }) - - result, err := client.GetAccounts(context.Background(), gofakeit.UUID(), plaid.GetAccountsOptions{ - AccountIDs: []string{ - account.AccountID, - }, - }) - assert.NoError(t, err, "should succeed") - assert.NotEmpty(t, result) - }) - - t.Run("error NO_ACCOUNTS", func(t *testing.T) { - httpmock.Activate() - defer httpmock.Deactivate() - - mock_plaid.MockGetAccountsError(t, plaid.Error{ - APIResponse: plaid.APIResponse{ - RequestID: gofakeit.UUID(), - }, - ErrorType: "ITEM_ERROR", - ErrorCode: "NO_ACCOUNTS", - ErrorMessage: "no valid accounts were found for this item", - DisplayMessage: "No valid accounts were found at the financial institution. Please visit your financial institution's website to confirm accounts are available.", - StatusCode: http.StatusBadRequest, - }) - - client := NewPlaidClient(testutils.GetLog(t), plaid.ClientOptions{ - ClientID: gofakeit.UUID(), - Secret: gofakeit.UUID(), - Environment: plaid.Sandbox, - HTTPClient: &http.Client{ - Transport: httpmock.DefaultTransport, - }, - }) - - result, err := client.GetAccounts(context.Background(), gofakeit.UUID(), plaid.GetAccountsOptions{ - AccountIDs: []string{ - gofakeit.UUID(), - }, - }) - assert.Error(t, err, "should fail with NO_ACCOUNTS error") - assert.Empty(t, result) - - cause := errors.Cause(err) - assert.IsType(t, plaid.Error{}, cause, "should be a plaid error") - }) -} diff --git a/pkg/internal/plaid_helper/verification.go b/pkg/internal/plaid_helper/verification.go deleted file mode 100644 index c2dcf166..00000000 --- a/pkg/internal/plaid_helper/verification.go +++ /dev/null @@ -1,125 +0,0 @@ -package plaid_helper - -import ( - "context" - "encoding/json" - "github.com/MicahParks/keyfunc" - "github.com/getsentry/sentry-go" - "github.com/gomodule/redigo/redis" - "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" - "github.com/sirupsen/logrus" - "sync" -) - -type WebhookVerification interface { - GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) - Close() error -} - -var ( - _ WebhookVerification = &redisWebhookVerification{} - _ WebhookVerification = &memoryWebhookVerification{} -) - -func NewMemoryWebhookVerificationCache(log *logrus.Entry, client Client) WebhookVerification { - return &memoryWebhookVerification{ - log: log, - plaidClient: client, - lock: sync.Mutex{}, - cache: map[string]*keyfunc.JWKS{}, - } -} - -type memoryWebhookVerification struct { - log *logrus.Entry - plaidClient Client - lock sync.Mutex - cache map[string]*keyfunc.JWKS -} - -func (m *memoryWebhookVerification) GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) { - span := sentry.StartSpan(ctx, "Cache - GetVerificationKey") - defer span.Finish() - - log := m.log.WithField("keyId", keyId).WithContext(ctx) - - m.lock.Lock() - defer m.lock.Unlock() - - jwksFunc, ok := m.cache[keyId] - if ok { - log.Trace("jwk function already present in cache, returning") - return jwksFunc, nil - } - - log.Trace("jwk function missing in cache, retrieving from plaid") - - verificationResponse, err := m.plaidClient.GetWebhookVerificationKey(span.Context(), keyId) - if err != nil { - return nil, errors.Wrap(err, "failed to retrieve public verification key") - } - - var keys = struct { - Keys []plaid.WebhookVerificationKey `json:"keys"` - }{ - Keys: []plaid.WebhookVerificationKey{ - verificationResponse.Key, - }, - } - - encodedKeys, err := json.Marshal(keys) - if err != nil { - return nil, errors.Wrap(err, "failed to convert plaid verification key to json") - } - - var jwksJSON json.RawMessage = encodedKeys - - jwksFunc, err = keyfunc.New(jwksJSON) - if err != nil { - return nil, errors.Wrap(err, "failed to create key function") - } - - m.cache[keyId] = jwksFunc - - return jwksFunc, nil -} - -func (m *memoryWebhookVerification) Close() error { - return m.plaidClient.Close() -} - -type redisWebhookVerification struct { - log *logrus.Entry - plaidClient Client - redisClient redis.Conn -} - -// TODO Finish building out caching of public verification keys. -func (r *redisWebhookVerification) GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) { - result, err := redis.String(r.redisClient.Do("GET", keyId)) - if err != nil { - return nil, err - } - - var jwksJSON json.RawMessage = []byte(result) - jwkKeyFunc, err := keyfunc.New(jwksJSON) - if err != nil { - return nil, errors.Wrap(err, "failed to create key function") - } - - return jwkKeyFunc, nil -} - -func (r *redisWebhookVerification) Close() error { - if err := r.plaidClient.Close(); err != nil { - r.log.WithError(err).Errorf("failed to close plaid client gracefully") - return errors.Wrap(err, "failed to close plaid client") - } - if err := r.redisClient.Close(); err != nil { - r.log.WithError(err).Errorf("failed to close redis client gracefully") - return errors.Wrap(err, "failed to close redis client") - } - - return nil -} diff --git a/pkg/internal/platypus/account.go b/pkg/internal/platypus/account.go new file mode 100644 index 00000000..f5de0e99 --- /dev/null +++ b/pkg/internal/platypus/account.go @@ -0,0 +1,123 @@ +package platypus + +import "github.com/plaid/plaid-go/plaid" + +type ( + BankAccount interface { + // GetAccountId will return Plaid's unique identifier for the bank account. + GetAccountId() string + // GetBalances will return the bank account's balances. + GetBalances() BankAccountBalances + // GetMask typically returns the last 4 of the bank account number. This could technically return anything that + // represents a small portion of the bank account's identification. I don't currently know enough about this to know + // what other values this might have. + GetMask() string + // GetName will return the name of the account specified by the user or the financial institution itself. + GetName() string + // GetOfficialName will return the name of the account specified by the financial institution itself. + GetOfficialName() string + } + + BankAccountBalances interface { + // GetAvailable returns the total amount available for the bank account in cents. + GetAvailable() int64 + // GetCurrent returns the current bank account balance in cents. This is typically the total account value excluding + // pending transactions. + GetCurrent() int64 + // GetLimit returns the limit of the account (this applies for credit accounts) in cents. + GetLimit() int64 + GetIsoCurrencyCode() string + GetUnofficialCurrencyCode() string + } +) + +var ( + _ BankAccountBalances = PlaidBankAccountBalances{} +) + +func NewPlaidBankAccountBalances(balances plaid.AccountBalance) (PlaidBankAccountBalances, error) { + return PlaidBankAccountBalances{ + // We work with all amounts in cents. So we need to convert all balances to cents in order to make them whole + // integers rather than floats. + Available: int64(balances.GetAvailable() * 100), + Current: int64(balances.GetCurrent() * 100), + Limit: int64(balances.GetLimit() * 100), + IsoCurrencyCode: balances.GetIsoCurrencyCode(), + UnofficialCurrencyCode: balances.GetUnofficialCurrencyCode(), + }, nil +} + +type PlaidBankAccountBalances struct { + Available int64 + Current int64 + Limit int64 + IsoCurrencyCode string + UnofficialCurrencyCode string +} + +func (p PlaidBankAccountBalances) GetAvailable() int64 { + return p.Available +} + +func (p PlaidBankAccountBalances) GetCurrent() int64 { + return p.Current +} + +func (p PlaidBankAccountBalances) GetLimit() int64 { + return p.Limit +} + +func (p PlaidBankAccountBalances) GetIsoCurrencyCode() string { + return p.IsoCurrencyCode +} + +func (p PlaidBankAccountBalances) GetUnofficialCurrencyCode() string { + return p.UnofficialCurrencyCode +} + +var ( + _ BankAccount = PlaidBankAccount{} +) + +func NewPlaidBankAccount(bankAccount plaid.AccountBase) (PlaidBankAccount, error) { + balances, err := NewPlaidBankAccountBalances(bankAccount.GetBalances()) + if err != nil { + return PlaidBankAccount{}, err + } + + return PlaidBankAccount{ + AccountId: bankAccount.GetAccountId(), + Balances: balances, + Mask: bankAccount.GetMask(), + Name: bankAccount.GetName(), + OfficialName: bankAccount.GetOfficialName(), + }, nil +} + +type PlaidBankAccount struct { + AccountId string + Balances PlaidBankAccountBalances + Mask string + Name string + OfficialName string +} + +func (p PlaidBankAccount) GetAccountId() string { + return p.AccountId +} + +func (p PlaidBankAccount) GetBalances() BankAccountBalances { + return p.Balances +} + +func (p PlaidBankAccount) GetMask() string { + return p.Mask +} + +func (p PlaidBankAccount) GetName() string { + return p.Name +} + +func (p PlaidBankAccount) GetOfficialName() string { + return p.OfficialName +} diff --git a/pkg/internal/platypus/account_test.go b/pkg/internal/platypus/account_test.go new file mode 100644 index 00000000..588a3f1d --- /dev/null +++ b/pkg/internal/platypus/account_test.go @@ -0,0 +1,81 @@ +package platypus + +import ( + "testing" + + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/assert" +) + +func TestNewPlaidBankAccountBalances(t *testing.T) { + t.Run("simple", func(t *testing.T) { + var available, current, limit float32 + available = 101.34 + current = 110.25 + limit = 500.13 + + plaidBalances := plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(&available), + Current: *plaid.NewNullableFloat32(¤t), + Limit: *plaid.NewNullableFloat32(&limit), + IsoCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), + UnofficialCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), + LastUpdatedDatetime: plaid.NullableTime{}, // Leave this be so that is has no value. + AdditionalProperties: map[string]interface{}{}, + } + + balances, err := NewPlaidBankAccountBalances(plaidBalances) + assert.NoError(t, err, "must be able to convert balances") + assert.NotEmpty(t, balances, "balances must not be empty") + assert.EqualValues(t, available*100, balances.GetAvailable(), "available should be converted to cents") + assert.EqualValues(t, current*100, balances.GetCurrent(), "current should be converted to cents") + assert.EqualValues(t, limit*100, balances.GetLimit(), "limit should be converted to cents") + assert.EqualValues(t, "USD", balances.GetIsoCurrencyCode(), "ISO currency code should match USD") + assert.EqualValues(t, "USD", balances.GetUnofficialCurrencyCode(), "unofficial currency code should match USD") + }) + + t.Run("missing value", func(t *testing.T) { + plaidBalances := plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(nil), + Current: *plaid.NewNullableFloat32(nil), + Limit: *plaid.NewNullableFloat32(nil), + IsoCurrencyCode: *plaid.NewNullableString(nil), + UnofficialCurrencyCode: *plaid.NewNullableString(nil), + } + + balances, err := NewPlaidBankAccountBalances(plaidBalances) + assert.NoError(t, err, "must be able to convert balances") + assert.EqualValues(t, 0, balances.GetAvailable(), "available should be 0 when no value is present") + assert.EqualValues(t, 0, balances.GetCurrent(), "current should be 0 when no value is present") + assert.EqualValues(t, 0, balances.GetLimit(), "limit should be 0 when no value is present") + assert.EqualValues(t, "", balances.GetIsoCurrencyCode(), "ISO currency code should be empty if no value is present") + assert.EqualValues(t, "", balances.GetUnofficialCurrencyCode(), "unofficial currency code should be empty if no value is present") + }) +} + +func TestNewPlaidBankAccount(t *testing.T) { + t.Run("simple", func(t *testing.T) { + subType := plaid.ACCOUNTSUBTYPE_CHECKING + plaidBank := plaid.AccountBase{ + AccountId: gofakeit.UUID(), + Balances: plaid.AccountBalance{}, + Mask: *plaid.NewNullableString(myownsanity.StringP("1234")), + Name: "Checking Account", + OfficialName: *plaid.NewNullableString(myownsanity.StringP("CHECKING - 1234")), + Type: plaid.ACCOUNTTYPE_DEPOSITORY, + Subtype: *plaid.NewNullableAccountSubtype(&subType), + } + + bank, err := NewPlaidBankAccount(plaidBank) + assert.NoError(t, err, "must be able to convert bank account") + assert.NotEmpty(t, bank, "bank account must not be empty") + assert.EqualValues(t, plaidBank.GetAccountId(), bank.GetAccountId(), "account Id must match") + assert.EqualValues(t, "1234", bank.GetMask(), "mask must match") + assert.EqualValues(t, "Checking Account", bank.GetName(), "name must match") + assert.EqualValues(t, "CHECKING - 1234", bank.GetOfficialName(), "official name must match") + + assert.IsType(t, PlaidBankAccountBalances{}, bank.GetBalances(), "must return plaid bank account balances") + }) +} diff --git a/pkg/internal/platypus/client.go b/pkg/internal/platypus/client.go new file mode 100644 index 00000000..fc30dcd1 --- /dev/null +++ b/pkg/internal/platypus/client.go @@ -0,0 +1,266 @@ +package platypus + +import ( + "context" + "fmt" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/crumbs" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "strconv" + "time" + + "github.com/getsentry/sentry-go" + "github.com/plaid/plaid-go/plaid" + "github.com/sirupsen/logrus" +) + +type ( + Client interface { + GetAccounts(ctx context.Context, accountIds ...string) ([]BankAccount, error) + GetAllTransactions(ctx context.Context, start, end time.Time, accountIds []string) ([]Transaction, error) + UpdateItem(ctx context.Context) (LinkToken, error) + RemoveItem(ctx context.Context) error + } +) + +var ( + _ Client = &PlaidClient{} +) + +type PlaidClient struct { + accountId uint64 + linkId uint64 + accessToken string + log *logrus.Entry + client *plaid.APIClient + config config.Plaid +} + +func (p *PlaidClient) getLog(span *sentry.Span) *logrus.Entry { + return p.log.WithContext(span.Context()).WithField("plaid", span.Op) +} + +func (p *PlaidClient) GetAccounts(ctx context.Context, accountIds ...string) ([]BankAccount, error) { + span := sentry.StartSpan(ctx, "Plaid - GetAccount") + defer span.Finish() + + log := p.getLog(span) + + // By default report the accountIds as "all accounts" to sentry. This way we know that if we are not requesting + // specific accounts then we are requesting all of them. + span.Data = map[string]interface{}{ + "accountIds": "ALL_BANK_ACCOUNTS", + } + + // If however we are requesting specific accounts, overwrite the value. + if len(accountIds) > 0 { + span.Data["accountIds"] = accountIds + } + + log.Trace("retrieving bank accounts from plaid") + + // Build the get accounts request. + request := p.client.PlaidApi. + AccountsGet(span.Context()). + AccountsGetRequest(plaid.AccountsGetRequest{ + AccessToken: p.accessToken, + Options: &plaid.AccountsGetRequestOptions{ + // This might not work, if it does not we should just add a nil check somehow here. + AccountIds: &accountIds, + }, + }) + + // Send the request. + result, response, err := request.Execute() + // And handle the response. + if err = after( + span, + response, + err, + "Retrieving bank accounts from Plaid", + "failed to retrieve bank accounts from plaid", + ); err != nil { + log.WithError(err).Errorf("failed to retrieve bank accounts from plaid") + return nil, err + } + + plaidAccounts := result.GetAccounts() + accounts := make([]BankAccount, len(plaidAccounts)) + + // Once we have our data, convert all of the results from our request to our own bank account interface. + for i, plaidAccount := range plaidAccounts { + accounts[i], err = NewPlaidBankAccount(plaidAccount) + if err != nil { + log.WithError(err). + WithField("bankAccountId", plaidAccount.GetAccountId()). + Errorf("failed to convert bank account") + crumbs.Error(span.Context(), "failed to convert bank account", "debug", map[string]interface{}{ + // Maybe we don't want to report the entire account object here, but it'll sure save us a ton of time + // if there is ever a problem with actually converting the account. This way we can actually see the + // account object that caused the problem -> when it caused the problem. + "bankAccount": plaidAccount, + }) + return nil, err + } + } + + return accounts, nil +} + +func (p *PlaidClient) GetAllTransactions(ctx context.Context, start, end time.Time, accountIds []string) ([]Transaction, error) { + span := sentry.StartSpan(ctx, "Plaid - GetAllTransactions") + defer span.Finish() + + transactions := make([]Transaction, 0) + + var perPage int32 = 500 + var offset int32 = 0 + for { + someTransactions, err := p.GetTransactions(span.Context(), start, end, perPage, offset, accountIds) + if err != nil { + return nil, err + } + + transactions = append(transactions, someTransactions...) + if retrieved := int32(len(someTransactions)); retrieved == perPage { + offset += retrieved + continue + } + + break + } + + return transactions, nil +} + +func (p *PlaidClient) GetTransactions(ctx context.Context, start, end time.Time, count, offset int32, bankAccountIds []string) ([]Transaction, error) { + span := sentry.StartSpan(ctx, "Plaid - GetTransactions") + defer span.Finish() + + log := p.getLog(span) + + log.Trace("retrieving transactions") + + request := p.client.PlaidApi. + TransactionsGet(span.Context()). + TransactionsGetRequest(plaid.TransactionsGetRequest{ + Options: &plaid.TransactionsGetRequestOptions{ + AccountIds: &bankAccountIds, + Count: &count, + Offset: &offset, + IncludeOriginalDescription: plaid.NullableBool{}, + }, + AccessToken: p.accessToken, + Secret: nil, + StartDate: start.Format("2006-01-02"), + EndDate: end.Format("2006-01-02"), + }) + + // Send the request. + result, response, err := request.Execute() + // And handle the response. + if err = after( + span, + response, + err, + "Retrieving transactions from Plaid", + "failed to retrieve transactions from plaid", + ); err != nil { + log.WithError(err).Errorf("failed to retrieve transactions from plaid") + return nil, err + } + + transactions := make([]Transaction, len(result.Transactions)) + for i, transaction := range result.Transactions { + transactions[i], err = NewTransactionFromPlaid(transaction) + if err != nil { + return nil, err + } + } + + return transactions, nil +} + +func (p *PlaidClient) UpdateItem(ctx context.Context) (LinkToken, error) { + span := sentry.StartSpan(ctx, "Plaid - UpdateItem") + defer span.Finish() + + log := p.getLog(span) + + log.Trace("creating link token for update") + + redirectUri := fmt.Sprintf("https://%s/plaid/oauth-return", p.config.OAuthDomain) + + var webhooksUrl *string + if p.config.WebhooksEnabled { + if p.config.WebhooksDomain == "" { + crumbs.Warn(span.Context(), "BUG: Plaid webhook domain is not present but webhooks are enabled.", "bug", nil) + } else { + webhooksUrl = myownsanity.StringP(fmt.Sprintf("https://%s/plaid/webhook", p.config.WebhooksDomain)) + } + } + + request := p.client.PlaidApi. + LinkTokenCreate(span.Context()). + LinkTokenCreateRequest(plaid.LinkTokenCreateRequest{ + ClientName: "monetr", + Language: PlaidLanguage, + CountryCodes: PlaidCountries, + User: plaid.LinkTokenCreateRequestUser{ + ClientUserId: strconv.FormatUint(p.accountId, 10), + }, + Webhook: webhooksUrl, + AccessToken: &p.accessToken, + LinkCustomizationName: nil, + RedirectUri: &redirectUri, + }) + + result, response, err := request.Execute() + if err = after( + span, + response, + err, + "Updating Plaid link token", + "failed to update Plaid link token", + ); err != nil { + log.WithError(err).Errorf("failed to create link token") + return nil, err + } + + return PlaidLinkToken{ + LinkToken: result.LinkToken, + Expires: result.Expiration, + }, nil +} + +func (p *PlaidClient) RemoveItem(ctx context.Context) error { + span := sentry.StartSpan(ctx, "Plaid - RemoveItem") + defer span.Finish() + + log := p.getLog(span) + + log.Trace("removing item") + + // Build the get accounts request. + request := p.client.PlaidApi. + ItemRemove(span.Context()). + ItemRemoveRequest(plaid.ItemRemoveRequest{ + AccessToken: p.accessToken, + }) + + // Send the request. + _, response, err := request.Execute() + // And handle the response. + if err = after( + span, + response, + err, + "Removing Plaid item", + "failed to remove Plaid item", + ); err != nil { + log.WithError(err).Errorf("failed to remove Plaid item") + return err + } + + return nil +} diff --git a/pkg/internal/platypus/client_test.go b/pkg/internal/platypus/client_test.go new file mode 100644 index 00000000..af4247e3 --- /dev/null +++ b/pkg/internal/platypus/client_test.go @@ -0,0 +1,130 @@ +package platypus + +import ( + "context" + "github.com/brianvoe/gofakeit/v6" + "github.com/jarcoal/httpmock" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/internal/mock_plaid" + "github.com/monetr/rest-api/pkg/internal/testutils" + "github.com/monetr/rest-api/pkg/models" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestPlaidClient_GetAccount(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + log := testutils.GetLog(t) + accountId := testutils.GetAccountIdForTest(t) + + accessToken := gofakeit.UUID() + + account := mock_plaid.BankAccountFixture(t) + + mock_plaid.MockGetAccounts(t, []plaid.AccountBase{ + account, + }) + + client := NewPlaid(log, nil, nil, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + }) + + link := &models.Link{ + LinkId: 1234, + AccountId: accountId, + } + + platypus, err := client.NewClient(context.Background(), link, accessToken) + assert.NoError(t, err, "should create platypus") + assert.NotNil(t, platypus, "should not be nil") + + accounts, err := platypus.GetAccounts(context.Background(), account.GetAccountId()) + assert.NoError(t, err, "should not return an error retrieving accounts") + assert.NotEmpty(t, accounts, "should return some accounts") + }) +} + +func TestPlaidClient_GetAllTransactions(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + log := testutils.GetLog(t) + accountId := testutils.GetAccountIdForTest(t) + + accessToken := gofakeit.UUID() + + account := mock_plaid.BankAccountFixture(t) + + end := time.Now() + start := end.Add(-7 * 24 * time.Hour) + mock_plaid.MockGetRandomTransactions(t, start, end, 5000, []string{ + account.GetAccountId(), + }) + + platypus := NewPlaid(log, nil, nil, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + }) + + link := &models.Link{ + LinkId: 1234, + AccountId: accountId, + } + + client, err := platypus.NewClient(context.Background(), link, accessToken) + assert.NoError(t, err, "should create platypus") + assert.NotNil(t, client, "should not be nil") + + transactions, err := client.GetAllTransactions(context.Background(), start, end, []string{ + account.GetAccountId(), + }) + assert.NoError(t, err, "should not return an error") + assert.NotEmpty(t, transactions, "should return a few transactions") + assert.Equal(t, map[string]int{ + "POST https://sandbox.plaid.com/transactions/get": 11, + }, httpmock.GetCallCountInfo(), "API calls should match") + }) +} + +func TestPlaidClient_UpdateItem(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + log := testutils.GetLog(t) + accountId := testutils.GetAccountIdForTest(t) + + accessToken := gofakeit.UUID() + + mock_plaid.MockCreateLinkToken(t) + + platypus := NewPlaid(log, nil, nil, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + OAuthDomain: "localhost", + }) + + link := &models.Link{ + LinkId: 1234, + AccountId: accountId, + } + + client, err := platypus.NewClient(context.Background(), link, accessToken) + assert.NoError(t, err, "should create client") + assert.NotNil(t, client, "should not be nil") + + linkToken, err := client.UpdateItem(context.Background()) + assert.NoError(t, err, "should not return an error creating an update link token") + assert.NotEmpty(t, linkToken.Token(), "must not be empty") + }) +} diff --git a/pkg/internal/platypus/platypus.go b/pkg/internal/platypus/platypus.go new file mode 100644 index 00000000..8e998aa1 --- /dev/null +++ b/pkg/internal/platypus/platypus.go @@ -0,0 +1,315 @@ +package platypus + +import ( + "context" + "encoding/json" + "fmt" + "github.com/getsentry/sentry-go" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/crumbs" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/monetr/rest-api/pkg/models" + "github.com/monetr/rest-api/pkg/repository" + "github.com/monetr/rest-api/pkg/secrets" + "github.com/pkg/errors" + "github.com/plaid/plaid-go/plaid" + "github.com/sirupsen/logrus" + "net/http" + "time" +) + +var ( + PlaidLanguage = "en" + PlaidCountries = []plaid.CountryCode{ + plaid.COUNTRYCODE_US, + } + PlaidProducts = []plaid.Products{ + plaid.PRODUCTS_TRANSACTIONS, + } +) + +type ( + Platypus interface { + CreateLinkToken(ctx context.Context, options LinkTokenOptions) (LinkToken, error) + ExchangePublicToken(ctx context.Context, publicToken string) (*ItemToken, error) + GetWebhookVerificationKey(ctx context.Context, keyId string) (*WebhookVerificationKey, error) + NewClientFromItemId(ctx context.Context, itemId string) (Client, error) + NewClientFromLink(ctx context.Context, accountId uint64, linkId uint64) (Client, error) + NewClient(ctx context.Context, link *models.Link, accessToken string) (Client, error) + Close() error + } +) + +// after is a wrapper around some of the basic operations we would want to perform after each request. Mainly that we +// want to keep track of things like the request Id and some information about the request itself. It also handles error +// wrapping. +func after(span *sentry.Span, response *http.Response, err error, message, errorMessage string) error { + if response != nil { + requestId := response.Header.Get("X-Request-Id") + if span.Data == nil { + span.Data = map[string]interface{}{} + } + + data := map[string]interface{}{} + + { + var extractedResponseBody map[string]interface{} + if e := json.NewDecoder(response.Body).Decode(&extractedResponseBody); e == nil { + if requestId == "" { + requestId = extractedResponseBody["request_id"].(string) + } + + if response.StatusCode != http.StatusOK { + data["body"] = extractedResponseBody + } + } + } + + data["X-RequestId"] = requestId + + span.Data["plaidRequestId"] = requestId + span.SetTag("plaidRequestId", requestId) + crumbs.HTTP( + span.Context(), + message, + "plaid", + response.Request.URL.String(), + response.Request.Method, + response.StatusCode, + data, + ) + } + if err != nil { + span.Status = sentry.SpanStatusInternalError + } + + span.Status = sentry.SpanStatusOK + + return errors.Wrap(err, errorMessage) +} + +var ( + _ Platypus = &Plaid{} +) + +func NewPlaid(log *logrus.Entry, secret secrets.PlaidSecretsProvider, repo repository.PlaidRepository, options config.Plaid) *Plaid { + httpClient := &http.Client{ + Timeout: 60 * time.Second, + } + + conf := plaid.NewConfiguration() + conf.HTTPClient = httpClient + conf.UseEnvironment(options.Environment) + conf.AddDefaultHeader("PLAID-CLIENT-ID", options.ClientID) + conf.AddDefaultHeader("PLAID-SECRET", options.ClientSecret) + + client := plaid.NewAPIClient(conf) + + return &Plaid{ + client: client, + log: log, + secret: secret, + repo: repo, + config: options, + } +} + +type Plaid struct { + client *plaid.APIClient + log *logrus.Entry + secret secrets.PlaidSecretsProvider + repo repository.PlaidRepository + config config.Plaid +} + +func (p *Plaid) CreateLinkToken(ctx context.Context, options LinkTokenOptions) (LinkToken, error) { + span := sentry.StartSpan(ctx, "Plaid - CreateLinkToken") + defer span.Finish() + + log := p.log + + redirectUri := fmt.Sprintf("https://%s/plaid/oauth-return", p.config.OAuthDomain) + + var webhooksUrl *string + if p.config.WebhooksEnabled { + if p.config.WebhooksDomain == "" { + crumbs.Warn(span.Context(), "BUG: Plaid webhook domain is not present but webhooks are enabled.", "bug", nil) + } else { + webhooksUrl = myownsanity.StringP(fmt.Sprintf("https://%s/plaid/webhook", p.config.WebhooksDomain)) + } + } + + request := p.client.PlaidApi. + LinkTokenCreate(span.Context()). + LinkTokenCreateRequest(plaid.LinkTokenCreateRequest{ + ClientName: "monetr", + Language: PlaidLanguage, + CountryCodes: PlaidCountries, + User: plaid.LinkTokenCreateRequestUser{ + ClientUserId: options.ClientUserID, + LegalName: &options.LegalName, + PhoneNumber: options.PhoneNumber, + PhoneNumberVerifiedTime: options.PhoneNumberVerifiedTime, + EmailAddress: &options.EmailAddress, + EmailAddressVerifiedTime: options.EmailAddressVerifiedTime, + Ssn: nil, + DateOfBirth: nil, + }, + Products: &PlaidProducts, + Webhook: webhooksUrl, + AccessToken: nil, + LinkCustomizationName: nil, + RedirectUri: &redirectUri, + AndroidPackageName: nil, + AccountFilters: nil, + EuConfig: nil, + InstitutionId: nil, + PaymentInitiation: nil, + DepositSwitch: nil, + IncomeVerification: nil, + Auth: nil, + }) + + result, response, err := request.Execute() + if err = after( + span, + response, + err, + "Creating link token with Plaid", + "failed to create link token", + ); err != nil { + log.WithError(err).Errorf("failed to create link token") + return nil, err + } + + return PlaidLinkToken{ + LinkToken: result.LinkToken, + Expires: result.Expiration, + }, nil +} + +func (p *Plaid) ExchangePublicToken(ctx context.Context, publicToken string) (*ItemToken, error) { + span := sentry.StartSpan(ctx, "Plaid - ExchangePublicToken") + defer span.Finish() + + log := p.log + + request := p.client.PlaidApi. + ItemPublicTokenExchange(span.Context()). + ItemPublicTokenExchangeRequest(plaid.ItemPublicTokenExchangeRequest{ + PublicToken: publicToken, + }) + + result, response, err := request.Execute() + if err = after( + span, + response, + err, + "Exchanging public token with Plaid", + "failed to exchange public token with Plaid", + ); err != nil { + log.WithError(err).Errorf("failed to exchange public token with Plaid") + return nil, err + } + + token, err := NewItemTokenFromPlaid(result) + if err != nil { + return nil, err + } + + return &token, nil +} + +func (p *Plaid) GetWebhookVerificationKey(ctx context.Context, keyId string) (*WebhookVerificationKey, error) { + span := sentry.StartSpan(ctx, "Plaid - GetWebhookVerificationKey") + defer span.Finish() + + log := p.log + + request := p.client.PlaidApi. + WebhookVerificationKeyGet(span.Context()). + WebhookVerificationKeyGetRequest(plaid.WebhookVerificationKeyGetRequest{ + KeyId: keyId, + }) + + result, response, err := request.Execute() + if err = after( + span, + response, + err, + "Retrieving webhook verification key", + "failed to retrieve webhook verification key from Plaid", + ); err != nil { + log.WithError(err).Errorf("failed to retrieve webhook verification key from Plaid") + return nil, err + } + + webhook, err := NewWebhookVerificationKeyFromPlaid(result.Key) + if err != nil { + return nil, err + } + + return &webhook, nil +} + +func (p *Plaid) NewClientFromItemId(ctx context.Context, itemId string) (Client, error) { + span := sentry.StartSpan(ctx, "Plaid - NewClientFromItemId") + defer span.Finish() + + link, err := p.repo.GetLinkByItemId(span.Context(), itemId) + if err != nil { + return nil, errors.Wrap(err, "cannot create client without link") + } + + return p.newClient(span.Context(), link) +} + +func (p *Plaid) NewClientFromLink(ctx context.Context, accountId uint64, linkId uint64) (Client, error) { + span := sentry.StartSpan(ctx, "Plaid - NewClientFromLink") + defer span.Finish() + + link, err := p.repo.GetLink(span.Context(), accountId, linkId) + if err != nil { + return nil, errors.Wrap(err, "cannot create Plaid client from link") + } + + return p.newClient(span.Context(), link) +} + +func (p *Plaid) NewClient(ctx context.Context, link *models.Link, accessToken string) (Client, error) { + return &PlaidClient{ + accountId: link.AccountId, + linkId: link.LinkId, + accessToken: accessToken, + log: p.log.WithFields(logrus.Fields{ + "accountId": link.AccountId, + "linkId": link.LinkId, + }), + client: p.client, + config: p.config, + }, nil +} + +func (p *Plaid) newClient(ctx context.Context, link *models.Link) (Client, error) { + span := sentry.StartSpan(ctx, "Plaid - newClient") + defer span.Finish() + + if link == nil { + return nil, errors.New("cannot create client without link") + } + + if link.PlaidLink == nil { + return nil, errors.New("cannot create client without link") + } + + accessToken, err := p.secret.GetAccessTokenForPlaidLinkId(span.Context(), link.AccountId, link.PlaidLink.ItemId) + if err != nil { + return nil, err + } + + return p.NewClient(span.Context(), link, accessToken) +} + +func (p *Plaid) Close() error { + panic("implement me") +} diff --git a/pkg/internal/platypus/platypus_test.go b/pkg/internal/platypus/platypus_test.go new file mode 100644 index 00000000..8bcb00c6 --- /dev/null +++ b/pkg/internal/platypus/platypus_test.go @@ -0,0 +1,43 @@ +package platypus + +import ( + "context" + "github.com/brianvoe/gofakeit/v6" + "github.com/jarcoal/httpmock" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/internal/mock_plaid" + "github.com/monetr/rest-api/pkg/internal/testutils" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestPlaid_CreateLinkToken(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + log := testutils.GetLog(t) + mock_plaid.MockCreateLinkToken(t) + + platypus := NewPlaid(log, nil, nil, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + OAuthDomain: "localhost", + }) + + linkToken, err := platypus.CreateLinkToken(context.Background(), LinkTokenOptions{ + ClientUserID: "1234", + LegalName: gofakeit.Name(), + PhoneNumber: nil, + PhoneNumberVerifiedTime: nil, + EmailAddress: gofakeit.Email(), + EmailAddressVerifiedTime: nil, + RedirectURI: "", + UpdateMode: false, + }) + assert.NoError(t, err, "should not return an error creating a link token") + assert.NotEmpty(t, linkToken.Token(), "must not be empty") + }) +} diff --git a/pkg/internal/platypus/token.go b/pkg/internal/platypus/token.go new file mode 100644 index 00000000..fddda585 --- /dev/null +++ b/pkg/internal/platypus/token.go @@ -0,0 +1,52 @@ +package platypus + +import ( + "github.com/plaid/plaid-go/plaid" + "time" +) + +type ItemToken struct { + AccessToken string + ItemId string +} + +func NewItemTokenFromPlaid(input plaid.ItemPublicTokenExchangeResponse) (ItemToken, error) { + return ItemToken{ + AccessToken: input.GetAccessToken(), + ItemId: input.GetItemId(), + }, nil +} + +type LinkTokenOptions struct { + ClientUserID string + LegalName string + PhoneNumber *string + PhoneNumberVerifiedTime *time.Time + EmailAddress string + EmailAddressVerifiedTime *time.Time + RedirectURI string + UpdateMode bool +} + +type LinkToken interface { + Token() string + Expiration() time.Time +} + +var ( + _ LinkToken = PlaidLinkToken{} +) + +type PlaidLinkToken struct { + LinkToken string + Expires time.Time +} + +func (p PlaidLinkToken) Token() string { + return p.LinkToken +} + +func (p PlaidLinkToken) Expiration() time.Time { + return p.Expires +} + diff --git a/pkg/internal/platypus/transaction.go b/pkg/internal/platypus/transaction.go new file mode 100644 index 00000000..79b3802c --- /dev/null +++ b/pkg/internal/platypus/transaction.go @@ -0,0 +1,119 @@ +package platypus + +import ( + "github.com/pkg/errors" + "github.com/plaid/plaid-go/plaid" + "time" +) + +type Transaction interface { + GetAmount() int64 + GetBankAccountId() string + GetCategory() []string + GetDate() time.Time + GetDateLocal(timezone *time.Location) time.Time + GetISOCurrencyCode() string + GetIsPending() bool + GetMerchantName() string + GetName() string + GetOriginalDescription() string + GetPendingTransactionId() *string + GetTransactionId() string + GetUnofficialCurrencyCode() string +} + +var ( + _ Transaction = PlaidTransaction{} +) + +type PlaidTransaction struct { + Amount int64 + BankAccountId string + Category []string + Date time.Time + ISOCurrencyCode string + UnofficialCurrencyCode string + IsPending bool + MerchantName string + Name string + OriginalDescription string + PendingTransactionId *string + TransactionId string +} + +func NewTransactionFromPlaid(input plaid.Transaction) (Transaction, error) { + date, err := time.Parse("2006-01-02", input.GetDate()) + if err != nil { + return nil, errors.Wrap(err, "failed to parse transaction date") + } + pendingTransactionId, _ := input.GetPendingTransactionIdOk() + + transaction := PlaidTransaction{ + Amount: int64(input.GetAmount() * 100), + BankAccountId: input.GetAccountId(), + Category: input.GetCategory(), + Date: date, + ISOCurrencyCode: input.GetIsoCurrencyCode(), + UnofficialCurrencyCode: input.GetUnofficialCurrencyCode(), + IsPending: input.GetPending(), + MerchantName: input.GetMerchantName(), + Name: input.GetName(), + OriginalDescription: input.GetOriginalDescription(), + PendingTransactionId: pendingTransactionId, + TransactionId: input.GetTransactionId(), + } + + return transaction, nil +} + +func (p PlaidTransaction) GetAmount() int64 { + return p.Amount +} + +func (p PlaidTransaction) GetBankAccountId() string { + return p.BankAccountId +} + +func (p PlaidTransaction) GetCategory() []string { + return p.Category +} + +func (p PlaidTransaction) GetDate() time.Time { + return p.Date +} + +func (p PlaidTransaction) GetDateLocal(timezone *time.Location) time.Time { + return p.Date.In(timezone) +} + +func (p PlaidTransaction) GetISOCurrencyCode() string { + return p.ISOCurrencyCode +} + +func (p PlaidTransaction) GetIsPending() bool { + return p.IsPending +} + +func (p PlaidTransaction) GetMerchantName() string { + return p.MerchantName +} + +func (p PlaidTransaction) GetName() string { + return p.Name +} + +func (p PlaidTransaction) GetOriginalDescription() string { + return p.OriginalDescription +} + +func (p PlaidTransaction) GetPendingTransactionId() *string { + return p.PendingTransactionId +} + +func (p PlaidTransaction) GetTransactionId() string { + return p.TransactionId +} + +func (p PlaidTransaction) GetUnofficialCurrencyCode() string { + return p.UnofficialCurrencyCode +} diff --git a/pkg/internal/platypus/webhook.go b/pkg/internal/platypus/webhook.go new file mode 100644 index 00000000..082e1fae --- /dev/null +++ b/pkg/internal/platypus/webhook.go @@ -0,0 +1,212 @@ +package platypus + +import ( + "context" + "encoding/json" + "github.com/MicahParks/keyfunc" + "github.com/getsentry/sentry-go" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/pkg/errors" + "github.com/plaid/plaid-go/plaid" + "github.com/sirupsen/logrus" + "sync" + "sync/atomic" + "time" +) + +type WebhookVerificationKey struct { + // The alg member identifies the cryptographic algorithm family used with the key. + Alg string `json:"alg"` + // The crv member identifies the cryptographic curve used with the key. + Crv string `json:"crv"` + // The kid (Key ID) member can be used to match a specific key. This can be used, for instance, to choose among a set of keys within the JWK during key rollover. + Kid string `json:"kid"` + // The kty (key type) parameter identifies the cryptographic algorithm family used with the key, such as RSA or EC. + Kty string `json:"kty"` + // The use (public key use) parameter identifies the intended use of the public key. + Use string `json:"use"` + // The x member contains the x coordinate for the elliptic curve point. + X string `json:"x"` + // The y member contains the y coordinate for the elliptic curve point. + Y string `json:"y"` + CreatedAt int32 `json:"created_at"` + ExpiredAt *int32 `json:"expired_at"` +} + +func NewWebhookVerificationKeyFromPlaid(input plaid.JWKPublicKey) (WebhookVerificationKey, error) { + return WebhookVerificationKey{ + Alg: input.GetAlg(), + Crv: input.GetCrv(), + Kid: input.GetKid(), + Kty: input.GetKty(), + Use: input.GetUse(), + X: input.GetX(), + Y: input.GetY(), + CreatedAt: input.GetCreatedAt(), + ExpiredAt: myownsanity.Int32P(input.GetExpiredAt()), + }, nil +} + +type WebhookVerification interface { + GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) + Close() error +} + +var ( + _ WebhookVerification = &memoryWebhookVerification{} +) + +func NewInMemoryWebhookVerification(log *logrus.Entry, plaid Platypus, cleanupInterval time.Duration) WebhookVerification { + verification := &memoryWebhookVerification{ + closed: 0, + log: log, + plaid: plaid, + lock: sync.Mutex{}, + cache: map[string]*keyCacheItem{}, + cleanupTicker: time.NewTicker(cleanupInterval), + closer: make(chan chan error, 1), + } + go verification.cacheWorker() // Start the background worker. + + return verification +} + +type keyCacheItem struct { + expiration time.Time + keyFunction *keyfunc.JWKS +} + +type memoryWebhookVerification struct { + closed uint32 + log *logrus.Entry + plaid Platypus + lock sync.Mutex + cache map[string]*keyCacheItem + cleanupTicker *time.Ticker + closer chan chan error +} + +func (m *memoryWebhookVerification) GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) { + if atomic.LoadUint32(&m.closed) > 0 { + return nil, errors.New("webhook verification is closed") + } + + span := sentry.StartSpan(ctx, "GetVerificationKey [InMemory]") + defer span.Finish() + + log := m.log.WithField("keyId", keyId).WithContext(ctx) + + m.lock.Lock() + defer m.lock.Unlock() + + item, ok := m.cache[keyId] + if ok { + if item.expiration.After(time.Now()) { + log.Trace("jwk function already present in cache, returning") + return item.keyFunction, nil + } + + log.Trace("jwk function present in cache, but is expired; the cached function will be removed and a new one will be retrieved") + delete(m.cache, keyId) + } + + log.Trace("retrieving jwk from plaid") + + result, err := m.plaid.GetWebhookVerificationKey(span.Context(), keyId) + if err != nil { + return nil, err + } + + var expiration time.Time + if result.ExpiredAt != nil { + expiration = time.Unix(int64(*result.ExpiredAt), 0) + } else { + // Making a huge assumption here, and this might end up causing problems later on. Maybe we should also add a + // check here to make sure that items that are close to expiration even here should not be cached? + expiration = time.Unix(int64(result.CreatedAt), 0).Add(30 * time.Minute) + } + + var keys = struct { + Keys []WebhookVerificationKey `json:"keys"` + }{ + Keys: []WebhookVerificationKey{ + *result, + }, + } + + encodedKeys, err := json.Marshal(keys) + if err != nil { + return nil, errors.Wrap(err, "failed to convert plaid verification key to json") + } + + var jwksJSON json.RawMessage = encodedKeys + + jwksFunc, err := keyfunc.New(jwksJSON) + if err != nil { + return nil, errors.Wrap(err, "failed to create key function") + } + + m.cache[keyId] = &keyCacheItem{ + expiration: expiration, + keyFunction: jwksFunc, + } + + return jwksFunc, nil +} + +func (m *memoryWebhookVerification) cacheWorker() { + for { + select { + case _ = <-m.cleanupTicker.C: + m.cleanup() + case promise := <-m.closer: + m.log.Debug("closing jwk cache, stopping background worker") + promise <- nil + return + } + } +} + +func (m *memoryWebhookVerification) cleanup() { + m.lock.Lock() + defer m.lock.Unlock() + + if len(m.cache) == 0 { + m.log.Debug("no items in Plaid jwk cache, nothing to cleanup") + return + } + + m.log.Debug("cleaning up Plaid jwk cache") + + itemsToRemove := make([]string, 0, len(m.cache)) + for key, item := range m.cache { + // If the item expiration is not in the future, then we need to add it to our list to be removed. + if !item.expiration.After(time.Now()) { + itemsToRemove = append(itemsToRemove, key) + } + } + + if len(itemsToRemove) == 0 { + m.log.Debug("no items have expired in cache") + return + } + + m.log.Debugf("found %d expired item(s); cleaning them up", len(itemsToRemove)) + + for _, key := range itemsToRemove { + delete(m.cache, key) + } + + return +} + +func (m *memoryWebhookVerification) Close() error { + if ok := atomic.CompareAndSwapUint32(&m.closed, 0, 1); !ok { + return errors.New("webhook verification is already closed") + } + + promise := make(chan error) + m.closer <- promise + + return <-promise +} diff --git a/pkg/internal/platypus/webhook_test.go b/pkg/internal/platypus/webhook_test.go new file mode 100644 index 00000000..a49255a5 --- /dev/null +++ b/pkg/internal/platypus/webhook_test.go @@ -0,0 +1,42 @@ +package platypus + +import ( + "context" + "github.com/brianvoe/gofakeit/v6" + "github.com/jarcoal/httpmock" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/internal/mock_plaid" + "github.com/monetr/rest-api/pkg/internal/testutils" + "github.com/monetr/rest-api/pkg/repository" + "github.com/monetr/rest-api/pkg/secrets" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestNewInMemoryWebhookVerification(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + mock_plaid.MockGetWebhookVerificationKey(t) + + log := testutils.GetLog(t) + db := testutils.GetPgDatabaseTxn(t) + secret := secrets.NewPostgresPlaidSecretsProvider(log, db) + plaidRepo := repository.NewPlaidRepository(db) + + plaid := NewPlaid(log, secret, plaidRepo, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + }) + + webhookVerification := NewInMemoryWebhookVerification(log, plaid, time.Second * 1) + + verify, err := webhookVerification.GetVerificationKey(context.Background(), gofakeit.UUID()) + assert.NoError(t, err, "must get verification") + assert.NotNil(t, verify, "verify must not be nil") + }) +} diff --git a/pkg/internal/testutils/account.go b/pkg/internal/testutils/account.go new file mode 100644 index 00000000..28e12f50 --- /dev/null +++ b/pkg/internal/testutils/account.go @@ -0,0 +1,14 @@ +package testutils + +import ( + "github.com/OneOfOne/xxhash" + "testing" +) + +// GetAccountIdForTest is used to create unique accountIds for individual tests. These accountIds are probably unique +// between tests. It is possible for two tests to have the same accountId, but it is very unlikely. This does make sure +// though that a test's accountId does stay the same between test runs as it is not random. This does not guarantee that +// the accountId is present in the database and is primarily aimed at mocks. +func GetAccountIdForTest(t *testing.T) uint64 { + return xxhash.Checksum64([]byte(t.Name())) +} diff --git a/pkg/internal/testutils/plaid.go b/pkg/internal/testutils/plaid.go index 5fb9d432..59024629 100644 --- a/pkg/internal/testutils/plaid.go +++ b/pkg/internal/testutils/plaid.go @@ -8,5 +8,5 @@ import ( type MockPlaidData struct { PlaidTokens map[string]models.PlaidToken PlaidLinks map[string]models.PlaidLink - BankAccounts map[string]map[string]plaid.Account + BankAccounts map[string]map[string]plaid.AccountBase } diff --git a/pkg/internal/testutils/seed.go b/pkg/internal/testutils/seed.go index 967fc1a7..2c08f05c 100644 --- a/pkg/internal/testutils/seed.go +++ b/pkg/internal/testutils/seed.go @@ -6,6 +6,7 @@ import ( "github.com/brianvoe/gofakeit/v6" "github.com/go-pg/pg/v10" "github.com/monetr/rest-api/pkg/hash" + "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/models" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" @@ -32,7 +33,7 @@ func SeedAccount(t *testing.T, db *pg.DB, options SeedAccountOption) (*models.Us plaidData := &MockPlaidData{ PlaidTokens: map[string]models.PlaidToken{}, PlaidLinks: map[string]models.PlaidLink{}, - BankAccounts: map[string]map[string]plaid.Account{}, + BankAccounts: map[string]map[string]plaid.AccountBase{}, } var user models.User @@ -190,38 +191,37 @@ func SeedAccount(t *testing.T, db *pg.DB, options SeedAccountOption) (*models.Us }, } - plaidData.BankAccounts[accessToken] = map[string]plaid.Account{ + checkingAccountSubType := plaid.ACCOUNTSUBTYPE_CHECKING + savingAccountSubType := plaid.ACCOUNTSUBTYPE_SAVINGS + + plaidData.BankAccounts[accessToken] = map[string]plaid.AccountBase{ checkingAccountId: { - AccountID: checkingAccountId, - Balances: plaid.AccountBalances{ - Available: float64(checkingBalance) / 100, - Current: float64(checkingBalance) / 100, - Limit: 0, - ISOCurrencyCode: "USD", - UnofficialCurrencyCode: "", + AccountId: checkingAccountId, + Balances: plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(checkingBalance) / 100)), + Current: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(checkingBalance) / 100)), + IsoCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), + UnofficialCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), }, - Mask: "1234", - Name: "Checking Account", - OfficialName: "Checking", - Subtype: "depository", - Type: "checking", - VerificationStatus: "", + Mask: *plaid.NewNullableString(myownsanity.StringP("1234")), + Name: "Checking Account", + OfficialName: *plaid.NewNullableString(myownsanity.StringP("Checking")), + Type: plaid.ACCOUNTTYPE_DEPOSITORY, + Subtype: *plaid.NewNullableAccountSubtype(&checkingAccountSubType), }, savingsAccountId: { - AccountID: savingsAccountId, - Balances: plaid.AccountBalances{ - Available: float64(savingBalance) / 100, - Current: float64(savingBalance) / 100, - Limit: 0, - ISOCurrencyCode: "USD", - UnofficialCurrencyCode: "", + AccountId: savingsAccountId, + Balances: plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(savingBalance) / 100)), + Current: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(savingBalance) / 100)), + IsoCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), + UnofficialCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), }, - Mask: "2345", - Name: "Savings Account", - OfficialName: "Savings", - Subtype: "depository", - Type: "saving", - VerificationStatus: "", + Mask: *plaid.NewNullableString(myownsanity.StringP("2345")), + Name: "Savings Account", + OfficialName: *plaid.NewNullableString(myownsanity.StringP("Savings")), + Type: plaid.ACCOUNTTYPE_DEPOSITORY, + Subtype: *plaid.NewNullableAccountSubtype(&savingAccountSubType), }, } diff --git a/pkg/internal/vault_helper/vault.go b/pkg/internal/vault_helper/vault.go index 92ddc2e8..9ffdd028 100644 --- a/pkg/internal/vault_helper/vault.go +++ b/pkg/internal/vault_helper/vault.go @@ -24,6 +24,7 @@ type Config struct { Auth string Token string TokenFile string + Username, Password string Timeout time.Duration TLSCertificatePath string TLSKeyPath string @@ -217,6 +218,24 @@ func (v *vaultBase) authenticate() error { log := v.log.WithField("method", v.config.Auth) switch v.config.Auth { + case "userpass": + log.Trace("authenticating to vault") + result, err := v.client.Logical().Write("auth/userpass/login/"+v.config.Username, map[string]interface{}{ + "password": v.config.Password, + "role": v.config.Role, + }) + if err != nil { + log.WithError(err).Errorf("failed to authenticate to vault") + return errors.Wrap(err, "failed to authenticate to vault") + } + + if result.Auth == nil { + log.WithError(err).Fatalf("no authentication returned from vault") + return errors.Errorf("no authentication returned from vault") + } + + v.client.SetToken(result.Auth.ClientToken) + log.Trace("successfully authenticated to vault") case "token": v.client.SetToken(v.config.Token) case "kubernetes": diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index 97577a85..90771484 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -2,13 +2,13 @@ package jobs import ( "context" + "github.com/monetr/rest-api/pkg/internal/platypus" "math" "time" "github.com/go-pg/pg/v10" "github.com/gocraft/work" "github.com/gomodule/redigo/redis" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" "github.com/monetr/rest-api/pkg/metrics" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/pubsub" @@ -32,13 +32,12 @@ var ( _ JobManager = &nonDistributedJobManager{} ) - type jobManagerBase struct { log *logrus.Entry work *work.WorkerPool queue *work.Enqueuer db *pg.DB - plaidClient plaid_helper.Client + plaidClient platypus.Platypus plaidSecrets secrets.PlaidSecretsProvider stats *metrics.Stats ps pubsub.PublishSubscribe @@ -48,7 +47,7 @@ func NewNonDistributedJobManager( log *logrus.Entry, pool *redis.Pool, db *pg.DB, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, plaidSecrets secrets.PlaidSecretsProvider, ) JobManager { @@ -69,7 +68,7 @@ func NewJobManager( log *logrus.Entry, pool *redis.Pool, db *pg.DB, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, plaidSecrets secrets.PlaidSecretsProvider, ) JobManager { @@ -97,7 +96,6 @@ func NewJobManager( manager.work.Job(PullLatestTransactions, manager.pullLatestTransactions) manager.work.Job(PullHistoricalTransactions, manager.pullHistoricalTransactions) manager.work.Job(RemoveTransactions, manager.removeTransactions) - manager.work.Job(UpdateInstitutions, manager.updateInstitutions) manager.work.Job(RemoveLink, manager.removeLink) // Every 30 minutes. 0 */30 * * * * diff --git a/pkg/jobs/non_distributed.go b/pkg/jobs/non_distributed.go index 5f8c067e..3641b015 100644 --- a/pkg/jobs/non_distributed.go +++ b/pkg/jobs/non_distributed.go @@ -4,7 +4,7 @@ import ( "context" "fmt" "github.com/go-pg/pg/v10" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/metrics" "github.com/monetr/rest-api/pkg/pubsub" "github.com/monetr/rest-api/pkg/secrets" @@ -15,7 +15,7 @@ import ( type nonDistributedJobManager struct { log *logrus.Entry db *pg.DB - plaidClient plaid_helper.Client + plaidClient platypus.Platypus plaidSecrets secrets.PlaidSecretsProvider stats *metrics.Stats ps pubsub.PublishSubscribe diff --git a/pkg/jobs/pull_account_balances.go b/pkg/jobs/pull_account_balances.go index 3ceb4d7b..02f2c84d 100644 --- a/pkg/jobs/pull_account_balances.go +++ b/pkg/jobs/pull_account_balances.go @@ -13,7 +13,6 @@ import ( "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/repository" "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" ) const ( @@ -162,40 +161,31 @@ func (j *jobManagerBase) pullAccountBalances(job *work.Job) (err error) { log.Debugf("requesting information for %d bank account(s)", len(itemBankAccountIds)) - result, err := j.plaidClient.GetAccounts( + platypus, err := j.plaidClient.NewClient(span.Context(), link, accessToken) + if err != nil { + log.WithError(err).Error("failed to create plaid client") + return err + } + + result, err := platypus.GetAccounts( span.Context(), - accessToken, - plaid.GetAccountsOptions{ - AccountIDs: itemBankAccountIds, - }, + itemBankAccountIds..., ) if err != nil { log.WithError(err).Error("failed to retrieve bank accounts from plaid") - switch plaidErr := errors.Cause(err).(type) { - case plaid.Error: - switch plaidErr.ErrorType { - case "ITEM_ERROR": - link.LinkStatus = models.LinkStatusError - link.ErrorCode = &plaidErr.ErrorCode - if updateErr := repo.UpdateLink(span.Context(), link); updateErr != nil { - log.WithError(updateErr).Error("failed to update link to be an error state") - } - } - } - return errors.Wrap(err, "failed to retrieve bank accounts from plaid") } updatedBankAccounts := make([]models.BankAccount, 0, len(result)) for _, item := range result { - bankAccount := plaidIdsToBank[item.AccountID] + bankAccount := plaidIdsToBank[item.GetAccountId()] bankLog := log.WithFields(logrus.Fields{ "bankAccountId": bankAccount.BankAccountId, "linkId": bankAccount.LinkId, }) shouldUpdate := false - available := int64(item.Balances.Available * 100) - current := int64(item.Balances.Current * 100) + available := item.GetBalances().GetAvailable() + current := item.GetBalances().GetCurrent() if bankAccount.CurrentBalance != current { bankLog = bankLog.WithField("currentBalanceChanged", true) diff --git a/pkg/jobs/pull_account_balances_test.go b/pkg/jobs/pull_account_balances_test.go index eeb54f0e..97a2ccce 100644 --- a/pkg/jobs/pull_account_balances_test.go +++ b/pkg/jobs/pull_account_balances_test.go @@ -6,15 +6,16 @@ import ( "github.com/go-pg/pg/v10" "github.com/gocraft/work" "github.com/jarcoal/httpmock" + "github.com/monetr/rest-api/pkg/config" "github.com/monetr/rest-api/pkg/internal/mock_plaid" "github.com/monetr/rest-api/pkg/internal/mock_secrets" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/testutils" "github.com/monetr/rest-api/pkg/repository" + "github.com/monetr/rest-api/pkg/secrets" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "net/http" "testing" "time" ) @@ -42,11 +43,12 @@ func TestPullAccountBalances(t *testing.T) { return nil }), "must retrieve linkId") - plaidClient := plaid_helper.NewPlaidClient(log, plaid.ClientOptions{ - ClientID: gofakeit.UUID(), - Secret: gofakeit.UUID(), - Environment: plaid.Sandbox, - HTTPClient: http.DefaultClient, + secretProvider := secrets.NewPostgresPlaidSecretsProvider(log, db) + plaidRepo := repository.NewPlaidRepository(db) + plaidClient := platypus.NewPlaid(log, secretProvider, plaidRepo, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, }) plaidSecrets := mock_secrets.NewMockPlaidSecrets() diff --git a/pkg/jobs/pull_historical_transactions.go b/pkg/jobs/pull_historical_transactions.go index 1872bb5b..b890fc63 100644 --- a/pkg/jobs/pull_historical_transactions.go +++ b/pkg/jobs/pull_historical_transactions.go @@ -9,7 +9,6 @@ import ( "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/repository" - "github.com/monetr/rest-api/pkg/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" "strconv" @@ -75,18 +74,6 @@ func (j *jobManagerBase) pullHistoricalTransactions(job *work.Job) (err error) { twoYearsAgo := time.Now().Add(-2 * 365 * 24 * time.Hour).UTC() return j.getRepositoryForJob(job, func(repo repository.Repository) error { - account, err := repo.GetAccount(span.Context()) - if err != nil { - log.WithError(err).Error("failed to retrieve account for job") - return err - } - - timezone, err := account.GetTimezone() - if err != nil { - log.WithError(err).Warn("failed to get account's time zone, defaulting to UTC") - timezone = time.UTC - } - link, err := repo.GetLink(span.Context(), linkId) if err != nil { log.WithError(err).Error("failed to retrieve link details to pull historical transactions") @@ -131,135 +118,29 @@ func (j *jobManagerBase) pullHistoricalTransactions(job *work.Job) (err error) { log.Debugf("retrieving transactions for %d bank account(s)", len(itemBankAccountIds)) - transactions, err := j.plaidClient.GetAllTransactions( - span.Context(), - accessToken, - twoYearsAgo, - time.Now(), - itemBankAccountIds, - ) - if err != nil { - log.WithError(err).Error("failed to retrieve transactions from plaid") - return errors.Wrap(err, "failed to retrieve transactions from plaid") - } - - plaidTransactionIds := make([]string, len(transactions)) - for i, transaction := range transactions { - plaidTransactionIds[i] = transaction.ID - } - transactionsByPlaidId, err := repo.GetTransactionsByPlaidId(span.Context(), linkId, plaidTransactionIds) + platypus, err := j.plaidClient.NewClient(span.Context(), link, accessToken) if err != nil { - log.WithError(err).Error("failed to retrieve transaction ids for updating plaid transactions") + log.WithError(err).Error("failed to create plaid client for link") return err } - transactionsToUpdate := make([]*models.Transaction, 0) - transactionsToInsert := make([]models.Transaction, 0) - now := time.Now().UTC() - for _, plaidTransaction := range transactions { - amount := int64(plaidTransaction.Amount * 100) - - date, _ := util.ParseInLocal("2006-01-02", plaidTransaction.Date, timezone) - var authorizedDate *time.Time - if plaidTransaction.AuthorizedDate != "" { - authDate, _ := util.ParseInLocal("2006-01-02", plaidTransaction.AuthorizedDate, timezone) - authorizedDate = &authDate - } - - var pendingPlaidTransactionId *string - if plaidTransaction.PendingTransactionID != "" { - pendingPlaidTransactionId = &plaidTransaction.PendingTransactionID - } - - transactionName := plaidTransaction.Name - - // We only want to make the transaction name be the merchant name if the merchant name is shorter. This is - // due to something I observed with a dominos transaction, where the merchant was improperly parsed and the - // transaction ended up being called `Mnuslindstrom` rather than `Domino's`. This should fix that problem. - if plaidTransaction.MerchantName != "" && len(plaidTransaction.MerchantName) < len(transactionName) { - transactionName = plaidTransaction.MerchantName - } - - existingTransaction, ok := transactionsByPlaidId[plaidTransaction.ID] - if !ok { - transactionsToInsert = append(transactionsToInsert, models.Transaction{ - AccountId: accountId, - BankAccountId: plaidIdsToBankIds[plaidTransaction.AccountID], - PlaidTransactionId: plaidTransaction.ID, - Amount: amount, - SpendingId: nil, - Spending: nil, - Categories: plaidTransaction.Category, - OriginalCategories: plaidTransaction.Category, - Date: date, - AuthorizedDate: authorizedDate, - Name: transactionName, - OriginalName: plaidTransaction.Name, - MerchantName: plaidTransaction.MerchantName, - OriginalMerchantName: plaidTransaction.MerchantName, - IsPending: plaidTransaction.Pending, - CreatedAt: now, - PendingPlaidTransactionId: pendingPlaidTransactionId, - }) - continue - } - - var shouldUpdate bool - if existingTransaction.Amount != amount { - shouldUpdate = true - } - - if existingTransaction.IsPending != plaidTransaction.Pending { - shouldUpdate = true - } - - if !myownsanity.TimesPEqual(existingTransaction.AuthorizedDate, authorizedDate) { - shouldUpdate = true - } - - if existingTransaction.PendingPlaidTransactionId != pendingPlaidTransactionId { - shouldUpdate = true - } - - existingTransaction.Amount = amount - existingTransaction.IsPending = plaidTransaction.Pending - existingTransaction.AuthorizedDate = authorizedDate - existingTransaction.PendingPlaidTransactionId = pendingPlaidTransactionId - - // Update old transactions calculated name as we can. - if existingTransaction.Name != transactionName { - existingTransaction.Name = transactionName - shouldUpdate = true - } - - // Fix timezone of records. - if !existingTransaction.Date.Equal(date) { - existingTransaction.Date = date - shouldUpdate = true - } - - if shouldUpdate { - transactionsToUpdate = append(transactionsToUpdate, &existingTransaction) - } - } - - if len(transactionsToUpdate) > 0 { - if err = repo.UpdateTransactions(span.Context(), transactionsToUpdate); err != nil { - log.WithError(err).Errorf("failed to update transactions for job") - return err - } + transactions, err := platypus.GetAllTransactions(span.Context(), twoYearsAgo, time.Now(), itemBankAccountIds) + if err != nil { + log.WithError(err).Error("failed to retrieve transactions from plaid") + return errors.Wrap(err, "failed to retrieve transactions from plaid") } - if len(transactionsToInsert) > 0 { - // Reverse the list so the oldest records are inserted first. - for i, j := 0, len(transactionsToInsert)-1; i < j; i, j = i+1, j-1 { - transactionsToInsert[i], transactionsToInsert[j] = transactionsToInsert[j], transactionsToInsert[i] - } - if err = repo.InsertTransactions(span.Context(), transactionsToInsert); err != nil { - log.WithError(err).Error("failed to insert new transactions") - return err - } + if err = j.upsertTransactions( + span.Context(), + log, + repo, + link, + plaidIdsToBankIds, + transactions, + ); err != nil { + log.WithError(err).Error("failed to upsert transactions from plaid") + return err } link.LastSuccessfulUpdate = myownsanity.TimeP(time.Now().UTC()) diff --git a/pkg/jobs/pull_initial_transactions.go b/pkg/jobs/pull_initial_transactions.go index dcf49236..8fff1cf5 100644 --- a/pkg/jobs/pull_initial_transactions.go +++ b/pkg/jobs/pull_initial_transactions.go @@ -8,9 +8,7 @@ import ( "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/repository" - "github.com/monetr/rest-api/pkg/util" "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" "strconv" "time" ) @@ -54,18 +52,6 @@ func (j *jobManagerBase) pullInitialTransactions(job *work.Job) (err error) { }) return j.getRepositoryForJob(job, func(repo repository.Repository) error { - account, err := repo.GetAccount(span.Context()) - if err != nil { - log.WithError(err).Error("failed to retrieve account for job") - return err - } - - timezone, err := account.GetTimezone() - if err != nil { - log.WithError(err).Warn("failed to get account's time zone, defaulting to UTC") - timezone = time.UTC - } - link, err := repo.GetLink(span.Context(), linkId) if err != nil { log.WithError(err).Error("cannot pull initial transactions for link provided") @@ -93,39 +79,26 @@ func (j *jobManagerBase) pullInitialTransactions(job *work.Job) (err error) { return nil } - bankAccountIdsByPlaid := map[string]uint64{} + plaidIdsToBankIds := map[string]uint64{} bankAccountIds := make([]string, len(link.BankAccounts)) for i, bankAccount := range link.BankAccounts { bankAccountIds[i] = bankAccount.PlaidAccountId - bankAccountIdsByPlaid[bankAccount.PlaidAccountId] = bankAccount.BankAccountId + plaidIdsToBankIds[bankAccount.PlaidAccountId] = bankAccount.BankAccountId } now := time.Now().UTC() - plaidTransactions, err := j.plaidClient.GetAllTransactions( - span.Context(), - accessToken, - now.Add(-30*24*time.Hour), - now, - bankAccountIds, - ) + platypus, err := j.plaidClient.NewClient(span.Context(), link, accessToken) if err != nil { - log.WithError(err).Error("failed to retrieve initial transactions") - - switch plaidErr := errors.Cause(err).(type) { - case plaid.Error: - switch plaidErr.ErrorType { - case "ITEM_ERROR": - link.LinkStatus = models.LinkStatusError - link.ErrorCode = &plaidErr.ErrorCode - if updateErr := repo.UpdateLink(span.Context(), link); updateErr != nil { - log.WithError(updateErr).Error("failed to update link to be an error state") - } - } - } - + log.WithError(err).Error("failed to create plaid client for link") return err } + plaidTransactions, err := platypus.GetAllTransactions(span.Context(), now.Add(-30*24*time.Hour), now, bankAccountIds) + if err != nil { + log.WithError(err).Error("failed to retrieve initial transactions from plaid") + return errors.Wrap(err, "failed to retrieve initial transactions from plaid") + } + if len(plaidTransactions) == 0 { log.Warn("no transactions were retrieved from plaid") return nil @@ -133,50 +106,15 @@ func (j *jobManagerBase) pullInitialTransactions(job *work.Job) (err error) { log.Debugf("retreived %d transaction(s) from plaid, processing now", len(plaidTransactions)) - transactions := make([]models.Transaction, len(plaidTransactions)) - for i, plaidTransaction := range plaidTransactions { - date, _ := util.ParseInLocal("2006-01-02", plaidTransaction.Date, timezone) - var authorizedDate *time.Time - if plaidTransaction.AuthorizedDate != "" { - authDate, _ := util.ParseInLocal("2006-01-02", plaidTransaction.AuthorizedDate, timezone) - authorizedDate = &authDate - } - - transactionName := plaidTransaction.Name - - // We only want to make the transaction name be the merchant name if the merchant name is shorter. This is - // due to something I observed with a dominos transaction, where the merchant was improperly parsed and the - // transaction ended up being called `Mnuslindstrom` rather than `Domino's`. This should fix that problem. - if plaidTransaction.MerchantName != "" && len(plaidTransaction.MerchantName) < len(transactionName) { - transactionName = plaidTransaction.MerchantName - } - - transactions[i] = models.Transaction{ - AccountId: repo.AccountId(), - BankAccountId: bankAccountIdsByPlaid[plaidTransaction.AccountID], - PlaidTransactionId: plaidTransaction.ID, - Amount: int64(plaidTransaction.Amount * 100), - SpendingId: nil, - Categories: plaidTransaction.Category, - OriginalCategories: plaidTransaction.Category, - Date: date, - AuthorizedDate: authorizedDate, - Name: transactionName, - OriginalName: plaidTransaction.Name, - MerchantName: plaidTransaction.MerchantName, - OriginalMerchantName: plaidTransaction.MerchantName, - IsPending: plaidTransaction.Pending, - CreatedAt: now, - } - } - - // Reverse the list so the oldest records are inserted first. - for i, j := 0, len(transactions)-1; i < j; i, j = i+1, j-1 { - transactions[i], transactions[j] = transactions[j], transactions[i] - } - - if err = repo.InsertTransactions(span.Context(), transactions); err != nil { - log.WithError(err).Error("failed to store initial transactions") + if err = j.upsertTransactions( + span.Context(), + log, + repo, + link, + plaidIdsToBankIds, + plaidTransactions, + ); err != nil { + log.WithError(err).Error("failed to upsert transactions from plaid") return err } diff --git a/pkg/jobs/pull_latest_transactions.go b/pkg/jobs/pull_latest_transactions.go index 23a25bca..bdc2c073 100644 --- a/pkg/jobs/pull_latest_transactions.go +++ b/pkg/jobs/pull_latest_transactions.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/monetr/rest-api/pkg/crumbs" "strconv" - "strings" "time" "github.com/getsentry/sentry-go" @@ -13,9 +12,7 @@ import ( "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/repository" - "github.com/monetr/rest-api/pkg/util" "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" "github.com/sirupsen/logrus" ) @@ -123,18 +120,6 @@ func (j *jobManagerBase) pullLatestTransactions(job *work.Job) (err error) { }) return j.getRepositoryForJob(job, func(repo repository.Repository) error { - account, err := repo.GetAccount(span.Context()) - if err != nil { - log.WithError(err).Error("failed to retrieve account for job") - return err - } - - timezone, err := account.GetTimezone() - if err != nil { - log.WithError(err).Warn("failed to get account's time zone, defaulting to UTC") - timezone = time.UTC - } - link, err := repo.GetLink(span.Context(), linkId) if err != nil { log.WithError(err).Error("failed to retrieve link details to pull transactions") @@ -193,156 +178,28 @@ func (j *jobManagerBase) pullLatestTransactions(job *work.Job) (err error) { } end := time.Now() - transactions, err := j.plaidClient.GetAllTransactions( - span.Context(), - accessToken, - start, - end, - itemBankAccountIds, - ) - if err != nil { - log.WithError(err).Error("failed to retrieve transactions from plaid") - switch plaidErr := errors.Cause(err).(type) { - case plaid.Error: - link.LinkStatus = models.LinkStatusError - link.ErrorCode = myownsanity.StringP(strings.Join([]string{ - plaidErr.ErrorType, - plaidErr.ErrorCode, - }, ".")) - if updateErr := repo.UpdateLink(span.Context(), link); updateErr != nil { - log.WithError(updateErr).Error("failed to update link to be an error state") - return err - } - - // Don't return an error here, we set the link to an error state and we don't want to have retry logic - // for this right now. - return nil - default: - log.WithError(err).Warnf("unknown error type from Plaid client: %T", plaidErr) - } - - return errors.Wrap(err, "failed to retrieve transactions from plaid") - } - - plaidTransactionIds := make([]string, len(transactions)) - for i, transaction := range transactions { - plaidTransactionIds[i] = transaction.ID - } - - transactionsByPlaidId, err := repo.GetTransactionsByPlaidId(span.Context(), linkId, plaidTransactionIds) + platypus, err := j.plaidClient.NewClient(span.Context(), link, accessToken) if err != nil { - log.WithError(err).Error("failed to retrieve transaction ids for updating plaid transactions") + log.WithError(err).Error("failed to create plaid client for link") return err } - transactionsToUpdate := make([]*models.Transaction, 0) - transactionsToInsert := make([]models.Transaction, 0) - now := time.Now().UTC() - for _, plaidTransaction := range transactions { - amount := int64(plaidTransaction.Amount * 100) - - date, _ := util.ParseInLocal("2006-01-02", plaidTransaction.Date, timezone) - var authorizedDate *time.Time - if plaidTransaction.AuthorizedDate != "" { - authDate, _ := util.ParseInLocal("2006-01-02", plaidTransaction.AuthorizedDate, timezone) - authorizedDate = &authDate - } - - var pendingPlaidTransactionId *string - if plaidTransaction.PendingTransactionID != "" { - pendingPlaidTransactionId = &plaidTransaction.PendingTransactionID - } - - transactionName := plaidTransaction.Name - - // We only want to make the transaction name be the merchant name if the merchant name is shorter. This is - // due to something I observed with a dominos transaction, where the merchant was improperly parsed and the - // transaction ended up being called `Mnuslindstrom` rather than `Domino's`. This should fix that problem. - if plaidTransaction.MerchantName != "" && len(plaidTransaction.MerchantName) < len(transactionName) { - transactionName = plaidTransaction.MerchantName - } - - existingTransaction, ok := transactionsByPlaidId[plaidTransaction.ID] - if !ok { - transactionsToInsert = append(transactionsToInsert, models.Transaction{ - AccountId: accountId, - BankAccountId: plaidIdsToBankIds[plaidTransaction.AccountID], - PlaidTransactionId: plaidTransaction.ID, - Amount: amount, - SpendingId: nil, - Spending: nil, - Categories: plaidTransaction.Category, - OriginalCategories: plaidTransaction.Category, - Date: date, - AuthorizedDate: authorizedDate, - Name: transactionName, - OriginalName: plaidTransaction.Name, - MerchantName: plaidTransaction.MerchantName, - OriginalMerchantName: plaidTransaction.MerchantName, - IsPending: plaidTransaction.Pending, - CreatedAt: now, - PendingPlaidTransactionId: pendingPlaidTransactionId, - }) - continue - } - - var shouldUpdate bool - if existingTransaction.Amount != amount { - shouldUpdate = true - } - - if existingTransaction.IsPending != plaidTransaction.Pending { - shouldUpdate = true - } - - if !myownsanity.TimesPEqual(existingTransaction.AuthorizedDate, authorizedDate) { - shouldUpdate = true - } - - if existingTransaction.PendingPlaidTransactionId != pendingPlaidTransactionId { - shouldUpdate = true - } - - existingTransaction.Amount = amount - existingTransaction.IsPending = plaidTransaction.Pending - existingTransaction.AuthorizedDate = authorizedDate - existingTransaction.PendingPlaidTransactionId = pendingPlaidTransactionId - - // Update old transactions calculated name as we can. - if existingTransaction.Name != transactionName { - existingTransaction.Name = transactionName - shouldUpdate = true - } - - // Fix timezone of records. - if !existingTransaction.Date.Equal(date) { - existingTransaction.Date = date - shouldUpdate = true - } - - if shouldUpdate { - transactionsToUpdate = append(transactionsToUpdate, &existingTransaction) - } - } - - if len(transactionsToUpdate) > 0 { - log.Infof("updating %d transactions", len(transactionsToUpdate)) - if err = repo.UpdateTransactions(span.Context(), transactionsToUpdate); err != nil { - log.WithError(err).Errorf("failed to update transactions for job") - return err - } + transactions, err := platypus.GetAllTransactions(span.Context(), start, end, itemBankAccountIds) + if err != nil { + log.WithError(err).Error("failed to retrieve transactions from plaid") + return errors.Wrap(err, "failed to retrieve transactions from plaid") } - if len(transactionsToInsert) > 0 { - log.Infof("creating %d transactions", len(transactionsToInsert)) - // Reverse the list so the oldest records are inserted first. - for i, j := 0, len(transactionsToInsert)-1; i < j; i, j = i+1, j-1 { - transactionsToInsert[i], transactionsToInsert[j] = transactionsToInsert[j], transactionsToInsert[i] - } - if err = repo.InsertTransactions(span.Context(), transactionsToInsert); err != nil { - log.WithError(err).Error("failed to insert new transactions") - return err - } + if err = j.upsertTransactions( + span.Context(), + log, + repo, + link, + plaidIdsToBankIds, + transactions, + ); err != nil { + log.WithError(err).Error("failed to upsert transactions from plaid") + return err } link.LastSuccessfulUpdate = myownsanity.TimeP(time.Now().UTC()) diff --git a/pkg/jobs/transactions.go b/pkg/jobs/transactions.go new file mode 100644 index 00000000..e5ff04c6 --- /dev/null +++ b/pkg/jobs/transactions.go @@ -0,0 +1,143 @@ +package jobs + +import ( + "context" + "github.com/getsentry/sentry-go" + "github.com/monetr/rest-api/pkg/internal/platypus" + "github.com/monetr/rest-api/pkg/models" + "github.com/monetr/rest-api/pkg/repository" + "github.com/sirupsen/logrus" + "time" +) + +func (j *jobManagerBase) upsertTransactions( + ctx context.Context, + log *logrus.Entry, + repo repository.BaseRepository, + link *models.Link, + plaidIdsToBankIds map[string]uint64, + plaidTransactions []platypus.Transaction, +) error { + span := sentry.StartSpan(ctx, "Job - Upsert Transactions") + defer span.Finish() + + account, err := repo.GetAccount(span.Context()) + if err != nil { + log.WithError(err).Error("failed to retrieve account for job") + return err + } + + timezone, err := account.GetTimezone() + if err != nil { + log.WithError(err).Warn("failed to get account's time zone, defaulting to UTC") + timezone = time.UTC + } + + plaidTransactionIds := make([]string, len(plaidTransactions)) + for i, transaction := range plaidTransactions { + plaidTransactionIds[i] = transaction.GetTransactionId() + } + + transactionsByPlaidId, err := repo.GetTransactionsByPlaidId(span.Context(), link.LinkId, plaidTransactionIds) + if err != nil { + log.WithError(err).Error("failed to retrieve transaction ids for updating plaid transactions") + return err + } + + transactionsToUpdate := make([]*models.Transaction, 0) + transactionsToInsert := make([]models.Transaction, 0) + now := time.Now().UTC() + for _, plaidTransaction := range plaidTransactions { + amount := plaidTransaction.GetAmount() + + date := plaidTransaction.GetDateLocal(timezone) + + transactionName := plaidTransaction.GetName() + + // We only want to make the transaction name be the merchant name if the merchant name is shorter. This is + // due to something I observed with a dominos transaction, where the merchant was improperly parsed and the + // transaction ended up being called `Mnuslindstrom` rather than `Domino's`. This should fix that problem. + if plaidTransaction.GetMerchantName() != "" && len(plaidTransaction.GetMerchantName()) < len(transactionName) { + transactionName = plaidTransaction.GetMerchantName() + } + + existingTransaction, ok := transactionsByPlaidId[plaidTransaction.GetTransactionId()] + if !ok { + transactionsToInsert = append(transactionsToInsert, models.Transaction{ + AccountId: repo.AccountId(), + BankAccountId: plaidIdsToBankIds[plaidTransaction.GetBankAccountId()], + PlaidTransactionId: plaidTransaction.GetTransactionId(), + Amount: amount, + SpendingId: nil, + Spending: nil, + Categories: plaidTransaction.GetCategory(), + OriginalCategories: plaidTransaction.GetCategory(), + Date: date, + Name: transactionName, + OriginalName: plaidTransaction.GetName(), + MerchantName: plaidTransaction.GetMerchantName(), + OriginalMerchantName: plaidTransaction.GetMerchantName(), + IsPending: plaidTransaction.GetIsPending(), + CreatedAt: now, + PendingPlaidTransactionId: plaidTransaction.GetPendingTransactionId(), + }) + continue + } + + var shouldUpdate bool + if existingTransaction.Amount != amount { + shouldUpdate = true + } + + if existingTransaction.IsPending != plaidTransaction.GetIsPending() { + shouldUpdate = true + } + + // This won't work quite right, I need to compare the values, not the pointers. + if existingTransaction.PendingPlaidTransactionId != plaidTransaction.GetPendingTransactionId() { + shouldUpdate = true + } + + existingTransaction.Amount = amount + existingTransaction.IsPending = plaidTransaction.GetIsPending() + existingTransaction.PendingPlaidTransactionId = plaidTransaction.GetPendingTransactionId() + + // Update old transactions calculated name as we can. + if existingTransaction.Name != transactionName { + existingTransaction.Name = transactionName + shouldUpdate = true + } + + // Fix timezone of records. + if !existingTransaction.Date.Equal(date) { + existingTransaction.Date = date + shouldUpdate = true + } + + if shouldUpdate { + transactionsToUpdate = append(transactionsToUpdate, &existingTransaction) + } + } + + if len(transactionsToUpdate) > 0 { + log.Infof("updating %d transactions", len(transactionsToUpdate)) + if err = repo.UpdateTransactions(span.Context(), transactionsToUpdate); err != nil { + log.WithError(err).Errorf("failed to update transactions for job") + return err + } + } + + if len(transactionsToInsert) > 0 { + log.Infof("creating %d transactions", len(transactionsToInsert)) + // Reverse the list so the oldest records are inserted first. + for i, j := 0, len(transactionsToInsert)-1; i < j; i, j = i+1, j-1 { + transactionsToInsert[i], transactionsToInsert[j] = transactionsToInsert[j], transactionsToInsert[i] + } + if err = repo.InsertTransactions(span.Context(), transactionsToInsert); err != nil { + log.WithError(err).Error("failed to insert new transactions") + return err + } + } + + return nil +} diff --git a/pkg/jobs/update_institutions.go b/pkg/jobs/update_institutions.go deleted file mode 100644 index 311f70c0..00000000 --- a/pkg/jobs/update_institutions.go +++ /dev/null @@ -1,104 +0,0 @@ -package jobs - -import ( - "context" - "github.com/getsentry/sentry-go" - "github.com/gocraft/work" - "github.com/monetr/rest-api/pkg/models" - "github.com/monetr/rest-api/pkg/repository" - "github.com/plaid/plaid-go/plaid" -) - -const ( - UpdateInstitutions = "UpdateInstitutions" -) - -func (j *jobManagerBase) updateInstitutions(job *work.Job) error { - span := sentry.StartSpan(context.Background(), "Job", sentry.TransactionName("Update Institutions")) - defer span.Finish() - - log := j.getLogForJob(job) - - institutions, err := j.plaidClient.GetAllInstitutions(span.Context(), []string{"US"}, plaid.GetInstitutionsOptions{ - Products: []string{ - "transactions", - }, - IncludeOptionalMetadata: true, - }) - if err != nil { - log.WithError(err).Error("failed to retrieve institutions for update") - return err - } - - if len(institutions) == 0 { - log.Warn("no institutions found") - return nil - } - - plaidInstitutionIds := make([]string, len(institutions)) - for i, institution := range institutions { - plaidInstitutionIds[i] = institution.ID - } - - return j.getJobHelperRepository(job, func(repo repository.JobRepository) error { - byPlaidId, err := repo.GetInstitutionsByPlaidID(span.Context(), plaidInstitutionIds) - if err != nil { - return err - } - - institutionsToUpdate := make([]*models.Institution, 0) - institutionsToCreate := make([]*models.Institution, 0) - for _, institution := range institutions { - existingInstitution, ok := byPlaidId[institution.ID] - if !ok { - var url, color, logo *string - if institution.URL != "" { - url = &institution.URL - } - - if institution.PrimaryColor != "" { - color = &institution.PrimaryColor - } - - if institution.Logo != "" { - logo = &institution.Logo - } - - institutionsToCreate = append(institutionsToCreate, &models.Institution{ - Name: institution.Name, - PlaidInstitutionId: &institution.ID, - PlaidProducts: institution.Products, - URL: url, - PrimaryColor: color, - Logo: logo, - }) - } - - if existingInstitution.Name != institution.Name { - existingInstitution.Name = institution.Name - institutionsToUpdate = append(institutionsToUpdate, &existingInstitution) - } - } - - if len(institutionsToUpdate) == 0 && len(institutionsToCreate) == 0 { - log.Infof("no institution changes") - return nil - } - - if len(institutionsToUpdate) > 0 { - log.Infof("updating %d institutions", len(institutionsToUpdate)) - if err = repo.UpdateInstitutions(span.Context(), institutionsToUpdate); err != nil { - return err - } - } - - if len(institutionsToCreate) > 0 { - log.Infof("creating %d institutions", len(institutionsToCreate)) - if err = repo.CreateInstitutions(span.Context(), institutionsToCreate); err != nil { - return err - } - } - - return nil - }) -} diff --git a/pkg/logging/pg.go b/pkg/logging/pg.go index 12c81694..d30c627c 100644 --- a/pkg/logging/pg.go +++ b/pkg/logging/pg.go @@ -88,7 +88,7 @@ func (h *PostgresHooks) AfterQuery(ctx context.Context, event *pg.QueryEvent) er Category: "postgres", Message: queryString, Data: map[string]interface{}{ - "queryTime": queryTime, + "queryTime": queryTime.String(), }, Level: "debug", Timestamp: event.StartTime, @@ -99,7 +99,7 @@ func (h *PostgresHooks) AfterQuery(ctx context.Context, event *pg.QueryEvent) er Category: "postgres", Message: queryString, Data: map[string]interface{}{ - "queryTime": queryTime, + "queryTime": queryTime.String(), "error": event.Err.Error(), }, Level: "error", diff --git a/pkg/repository/plaid_link.go b/pkg/repository/plaid_link.go index d50ae96c..45fa9697 100644 --- a/pkg/repository/plaid_link.go +++ b/pkg/repository/plaid_link.go @@ -5,6 +5,7 @@ package repository import ( "context" "github.com/getsentry/sentry-go" + "github.com/go-pg/pg/v10" "github.com/monetr/rest-api/pkg/models" "github.com/pkg/errors" ) @@ -24,4 +25,65 @@ func (r *repositoryBase) UpdatePlaidLink(ctx context.Context, link *models.Plaid span.SetTag("accountId", r.AccountIdStr()) _, err := r.txn.ModelContext(span.Context(), link).WherePK().Update(link) return errors.Wrap(err, "failed to update Plaid link") -} \ No newline at end of file +} + +type PlaidRepository interface { + GetLinkByItemId(ctx context.Context, itemId string) (*models.Link, error) + GetLink(ctx context.Context, accountId, linkId uint64) (*models.Link, error) +} + +func NewPlaidRepository(db pg.DBI) PlaidRepository { + return &plaidRepositoryBase{ + txn: db, + } +} + +type plaidRepositoryBase struct { + txn pg.DBI +} + +func (r *plaidRepositoryBase) GetLinkByItemId(ctx context.Context, itemId string) (*models.Link, error) { + span := sentry.StartSpan(ctx, "GetLinkByItemId") + defer span.Finish() + + span.Data = map[string]interface{}{ + "itemId": itemId, + } + + var link models.Link + err := r.txn.ModelContext(span.Context(), &link). + Relation("PlaidLink"). + Relation("BankAccounts"). + Where(`"plaid_link"."item_id" = ?`, itemId). + Limit(1). + Select(&link) + if err != nil { + return nil, errors.Wrap(err, "failed to retrieve link by item Id") + } + + return &link, nil +} + +func (r *plaidRepositoryBase) GetLink(ctx context.Context, accountId, linkId uint64) (*models.Link, error) { + span := sentry.StartSpan(ctx, "GetLink") + defer span.Finish() + + span.Data = map[string]interface{}{ + "accountId": accountId, + "linkId": linkId, + } + + var link models.Link + err := r.txn.ModelContext(span.Context(), &link). + Relation("PlaidLink"). + Relation("BankAccounts"). + Where(`"link"."account_id" = ?`, accountId). + Where(`"link"."link_id" = ?`, linkId). + Limit(1). + Select(&link) + if err != nil { + return nil, errors.Wrap(err, "failed to retrieve link") + } + + return &link, nil +} diff --git a/pkg/repository/plaid_link_test.go b/pkg/repository/plaid_link_test.go new file mode 100644 index 00000000..86246695 --- /dev/null +++ b/pkg/repository/plaid_link_test.go @@ -0,0 +1,111 @@ +package repository + +import ( + "context" + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestPlaidRepositoryBase_GetLink(t *testing.T) { + repo := GetTestAuthenticatedRepository(t) + txn := repo.(*repositoryBase).txn + require.NotNil(t, txn, "must be able to pull the transaction for test") + + plaidLink := &models.PlaidLink{ + ItemId: gofakeit.UUID(), + Products: []string{ + "transactions", + }, + WebhookUrl: "https://monetr.test/webhook", + } + + link := &models.Link{ + AccountId: repo.AccountId(), + LinkType: models.PlaidLinkType, + PlaidLinkId: nil, + LinkStatus: models.LinkStatusSetup, + InstitutionName: "Institution " + t.Name(), + CustomInstitutionName: "Institution " + t.Name(), + CreatedAt: time.Now(), + CreatedByUserId: repo.UserId(), + UpdatedAt: time.Now(), + UpdatedByUserId: nil, + LastSuccessfulUpdate: nil, + } + + { // Create the links. + require.NoError(t, repo.CreatePlaidLink(context.Background(), plaidLink), "must create plaid link") + link.PlaidLinkId = &plaidLink.PlaidLinkID + require.NoError(t, repo.CreateLink(context.Background(), link), "must create link") + } + + plaidRepo := NewPlaidRepository(txn) + + t.Run("simple", func(t *testing.T) { + readLink, err := plaidRepo.GetLink(context.Background(), link.AccountId, link.LinkId) + assert.NoError(t, err, "failed to retrieve link") + assert.NotNil(t, readLink.PlaidLink, "must include plaid link child") + assert.EqualValues(t, link.LinkId, readLink.LinkId, "link Id must match") + assert.EqualValues(t, plaidLink.PlaidLinkID, readLink.PlaidLink.PlaidLinkID, "plaid link Id must match") + }) + + t.Run("not found", func(t *testing.T) { + readLink, err := plaidRepo.GetLink(context.Background(), link.AccountId, link.LinkId + 100) + assert.EqualError(t, err, "failed to retrieve link: pg: no rows in result set") + assert.Nil(t, readLink, "link must be nil") + }) +} + +func TestPlaidRepositoryBase_GetLinkByItemId(t *testing.T) { + repo := GetTestAuthenticatedRepository(t) + txn := repo.(*repositoryBase).txn + require.NotNil(t, txn, "must be able to pull the transaction for test") + + plaidLink := &models.PlaidLink{ + ItemId: gofakeit.UUID(), + Products: []string{ + "transactions", + }, + WebhookUrl: "https://monetr.test/webhook", + } + + link := &models.Link{ + AccountId: repo.AccountId(), + LinkType: models.PlaidLinkType, + PlaidLinkId: nil, + LinkStatus: models.LinkStatusSetup, + InstitutionName: "Institution " + t.Name(), + CustomInstitutionName: "Institution " + t.Name(), + CreatedAt: time.Now(), + CreatedByUserId: repo.UserId(), + UpdatedAt: time.Now(), + UpdatedByUserId: nil, + LastSuccessfulUpdate: nil, + } + + { // Create the links. + require.NoError(t, repo.CreatePlaidLink(context.Background(), plaidLink), "must create plaid link") + link.PlaidLinkId = &plaidLink.PlaidLinkID + require.NoError(t, repo.CreateLink(context.Background(), link), "must create link") + } + + plaidRepo := NewPlaidRepository(txn) + + t.Run("simple", func(t *testing.T) { + readLink, err := plaidRepo.GetLinkByItemId(context.Background(), plaidLink.ItemId) + assert.NoError(t, err, "failed to retrieve link") + assert.NotNil(t, readLink.PlaidLink, "must include plaid link child") + assert.EqualValues(t, link.LinkId, readLink.LinkId, "link Id must match") + assert.EqualValues(t, plaidLink.PlaidLinkID, readLink.PlaidLink.PlaidLinkID, "plaid link Id must match") + }) + + t.Run("not found", func(t *testing.T) { + readLink, err := plaidRepo.GetLinkByItemId(context.Background(), "not a real item id") + assert.EqualError(t, err, "failed to retrieve link by item Id: pg: no rows in result set") + assert.Nil(t, readLink, "link must be nil") + }) +} diff --git a/templates/api-config.yaml b/templates/api-config.yaml index 89a1be0f..ff00b1bb 100644 --- a/templates/api-config.yaml +++ b/templates/api-config.yaml @@ -19,6 +19,7 @@ data: MONETR_PLAID_RETURNING_EXPERIENCE: {{ quote .Values.api.plaid.enableReturningUserExperience }} MONETR_PLAID_WEBHOOKS_ENABLED: {{ quote .Values.api.plaid.webhooksEnabled }} MONETR_PLAID_WEBHOOKS_DOMAIN: {{ quote .Values.api.plaid.webhooksDomain }} + MONETR_PLAID_OAUTH_DOMAIN: {{ quote .Values.api.plaid.oauthDomain }} MONETR_PG_ADDRESS: {{ quote .Values.api.postgreSql.address }} MONETR_PG_PORT: {{ quote .Values.api.postgreSql.port }} MONETR_PG_DATABASE: {{ quote .Values.api.postgreSql.database }} diff --git a/tests/pg/foreign_keys.sql b/tests/pg/foreign_keys.sql new file mode 100644 index 00000000..3c38d08c --- /dev/null +++ b/tests/pg/foreign_keys.sql @@ -0,0 +1,9 @@ +BEGIN; +SELECT plan(2); + +SELECT has_fk('transactions'); -- Make sure the transactions table has a foreign key. +SELECT col_is_fk('transactions', 'account_id'); -- Account ID should always be a foreign key. + +SELECT * +FROM finish(); +ROLLBACK; diff --git a/tests/pg/indexes.sql b/tests/pg/indexes.sql new file mode 100644 index 00000000..52df9c7b --- /dev/null +++ b/tests/pg/indexes.sql @@ -0,0 +1,12 @@ +BEGIN; +SELECT plan(2); + +-- Make sure that the index we specify does get renamed. I don't want to change the migration script as it's old. So I +-- want to make sure that it is properly applied. If this fails then that means the PostgreSQL behavior has changed in +-- whatever version that is being used. And that this index (when applied from scratch) is no longer being renamed. +SELECT hasnt_index('plaid_links', 'ix_uq_plaid_links_item_id'); +SELECT has_index('plaid_links', 'uq_plaid_links_item_id'); + +SELECT * +FROM finish(); +ROLLBACK; diff --git a/values.acceptance.yaml b/values.acceptance.yaml index 88950687..c5f40f1b 100644 --- a/values.acceptance.yaml +++ b/values.acceptance.yaml @@ -129,7 +129,8 @@ api: environment: "https://development.plaid.com" enableReturningUserExperience: true webhooksEnabled: true - webhooksDomain: https://api.acceptance.monetr.dev + webhooksDomain: api.acceptance.monetr.dev + oauthDomain: app.acceptance.monetr.dev cors: allowedOrigins: - "https://acceptance.monetr.dev" diff --git a/values.dog.yaml b/values.dog.yaml index ca218613..794ff397 100644 --- a/values.dog.yaml +++ b/values.dog.yaml @@ -5,7 +5,7 @@ image: pullPolicy: Always tag: "" # Will be overwritten with the SHA for the commit of this deploy -imagePullSecrets: [] +imagePullSecrets: [ ] podAnnotations: monetr.dev/branch: "" # Branch of the deployment will be put here @@ -108,7 +108,7 @@ api: beta: enableBetaCodes: true postgreSql: - address: newfoundland.monetr-acceptance.svc.cluster.local + address: postgres.google.acceptance.monetr.in port: 5432 database: monetr smtp: @@ -127,14 +127,15 @@ api: environment: "https://development.plaid.com" enableReturningUserExperience: true webhooksEnabled: true - webhooksDomain: https://api.monetr.dog + webhooksDomain: api.monetr.dog + oauthDomain: ui.monetr.dog cors: allowedOrigins: - "https://monetr.dog" debug: false redis: enabled: true - address: redis.monetr-acceptance.svc.cluster.local + address: redis.redis.kluster.monetr.in port: 6379 logging: level: debug diff --git a/values.staging.yaml b/values.staging.yaml index 0fc0420e..b466db1f 100644 --- a/values.staging.yaml +++ b/values.staging.yaml @@ -127,7 +127,8 @@ api: environment: "https://sandbox.plaid.com" enableReturningUserExperience: true webhooksEnabled: true - webhooksDomain: https://api.staging.monetr.dev + webhooksDomain: api.staging.monetr.dev + oauthDomain: app.staging.monetr.dev cors: allowedOrigins: - "https://app.staging.monetr.dev" diff --git a/values.yaml b/values.yaml index e22c5075..91addf44 100644 --- a/values.yaml +++ b/values.yaml @@ -135,6 +135,7 @@ api: enableReturningUserExperience: false webhooksEnabled: false webhooksDomain: "" + oauthDomain: "" cors: allowedOrigins: - "*" From 73c2c5a7a08f0d57e392638f60d20aa9331ef590 Mon Sep 17 00:00:00 2001 From: Elliot Courant Date: Fri, 10 Sep 2021 19:01:33 -0500 Subject: [PATCH 3/3] Code review changes for Plaid V1. --- go.mod | 3 +- go.sum | 26 +++++++++++- minikube/vault/auth.tf | 4 ++ minikube/vault/rest-api-service.tf | 5 ++- pkg/config/configuration.go | 6 ++- pkg/controller/links.go | 8 ++-- pkg/controller/plaid.go | 14 +++---- .../mock_http_helper/responder_test.go | 7 +++- pkg/internal/mock_plaid/accounts.go | 13 +++--- pkg/internal/mock_plaid/accounts_test.go | 10 ++++- pkg/internal/mock_plaid/link.go | 11 +++-- pkg/internal/mock_plaid/plaid_test.go | 16 ++++++++ pkg/internal/mock_plaid/verification.go | 19 +++++---- pkg/internal/myownsanity/number_test.go | 23 ++++++++++- pkg/internal/myownsanity/strings_test.go | 40 +++++++++++++++++++ pkg/internal/myownsanity/times_test.go | 30 ++++++++++++++ pkg/internal/platypus/account.go | 16 ++++++++ pkg/internal/platypus/account_test.go | 2 + pkg/internal/platypus/platypus.go | 27 +++++++++---- 19 files changed, 235 insertions(+), 45 deletions(-) create mode 100644 pkg/internal/mock_plaid/plaid_test.go create mode 100644 pkg/internal/myownsanity/strings_test.go create mode 100644 pkg/internal/myownsanity/times_test.go diff --git a/go.mod b/go.mod index 093b795a..57439d6c 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 github.com/stripe/stripe-go/v72 v72.64.1 + github.com/swaggo/swag v1.7.1 github.com/teambition/rrule-go v1.7.2 github.com/vmihailenco/msgpack/v5 v5.3.4 github.com/xlzd/gotp v0.0.0-20181030022105-c8557ba2c119 @@ -83,7 +84,6 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.1 // indirect github.com/google/go-cmp v0.5.6 // indirect - github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.1.2 // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/gorilla/css v1.0.0 // indirect @@ -154,7 +154,6 @@ require ( github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 // indirect diff --git a/go.sum b/go.sum index de0cb47d..d43de400 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Joker/hpp v1.0.0 h1:65+iuJYdRXv/XyN62C1uEmmOx3432rNG/rKlX6V7Kkc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/MicahParks/keyfunc v0.4.0 h1:+4Gj1EJXy09j6e+S+O9jNNdAOxc6Sra6KWCgbLSkL6E= github.com/MicahParks/keyfunc v0.4.0/go.mod h1:zLNyBGSzTMF3hq4XLLsZsKvxKe0tqHYSfXoFmv9w9g4= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -66,6 +68,10 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398 h1:WDC6ySpJzbxGWFh4aMxFFC28wwGp5pEuoTtvA4q/qQ4= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= @@ -73,6 +79,7 @@ github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -148,6 +155,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -229,6 +237,16 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/spec v0.20.3 h1:uH9RQ6vdyPSs2pSy9fL8QPspDF2AMIMPtmK5coSSjtQ= +github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-pg/migrations/v8 v8.1.0 h1:bc1wQwFoWRKvLdluXCRFRkeaw9xDU4qJ63uCAagh66w= github.com/go-pg/migrations/v8 v8.1.0/go.mod h1:o+CN1u572XHphEHZyK6tqyg2GDkRvL2bIoLNyGIewus= github.com/go-pg/pg/v10 v10.4.0/go.mod h1:BfgPoQnD2wXNd986RYEHzikqv9iE875PrFaZ9vXvtNM= @@ -428,6 +446,7 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -490,6 +509,8 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -692,6 +713,8 @@ github.com/stripe/stripe-go/v72 v72.64.1 h1:LsT6QVC8xF4X/Kp8xsNYqvubE3vuXn4/dhOF github.com/stripe/stripe-go/v72 v72.64.1/go.mod h1:QwqJQtduHubZht9mek5sds9CtQcKFdsykV9ZepRWwo0= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/swaggo/swag v1.7.1 h1:gY9ZakXlNWg/i/v5bQBic7VMZ4teq4m89lpiao74p/s= +github.com/swaggo/swag v1.7.1/go.mod h1:gAiHxNTb9cIpNmA/VEGUP+CyZMCP/EW7mdtc8Bny+p8= github.com/tdewolff/minify/v2 v2.9.10 h1:p+ifTTl+JMFFLDYNAm7nxQ9XuCG10HTW00wlPAZ7aoE= github.com/tdewolff/minify/v2 v2.9.10/go.mod h1:U1Nc+/YBSB0FPEarqcgkYH3Ep4DNyyIbOyl5P4eWMuo= github.com/tdewolff/parse/v2 v2.5.5 h1:b7ICJa4I/54JQGEGgTte8DiyJPKcC5g8V773QMzkeUM= @@ -710,6 +733,7 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -1151,8 +1175,6 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 h1:3V2dxSZpz4zozWWUq36vUxXEKnSYitEH2LdsAx+RUmg= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af h1:aLMMXFYqw01RA6XJim5uaN+afqNNjc9P8HPAbnpnc5s= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= diff --git a/minikube/vault/auth.tf b/minikube/vault/auth.tf index ec69a308..d9102320 100644 --- a/minikube/vault/auth.tf +++ b/minikube/vault/auth.tf @@ -15,6 +15,10 @@ resource "vault_auth_backend" "userpass" { type = "userpass" } +// monetr-user is just a basic user-password authentication for the API. This is to be used when trying to debug +// permissions or trying to dignose issues outside of Kubernetes. As it is easier to provision access using a +// username and password than it is to try to use the Kubernetes authentication, outside kube. +// The user is "monetr", the password is "password". resource "vault_generic_endpoint" "monetr-user" { depends_on = [vault_auth_backend.userpass] path = "auth/userpass/users/monetr" diff --git a/minikube/vault/rest-api-service.tf b/minikube/vault/rest-api-service.tf index 550f1def..af8bc1ee 100644 --- a/minikube/vault/rest-api-service.tf +++ b/minikube/vault/rest-api-service.tf @@ -1,4 +1,7 @@ data "vault_policy_document" "rest-api-policy" { + // There are two rules here because I still have no idea what I'm doing when it comes to trying to provision this + // stuff. I'm getting permissions errors for both paths so this is likely to change in the future. But again this + // is only used for local development against vault. rule { path = "${vault_mount.plaid-client-secrets.path}/*" capabilities = [ @@ -40,4 +43,4 @@ resource "vault_kubernetes_auth_backend_role" "rest-api" { token_policies = [ vault_policy.rest-api-service-policy.name, ] -} \ No newline at end of file +} diff --git a/pkg/config/configuration.go b/pkg/config/configuration.go index 59462272..c66d93ff 100644 --- a/pkg/config/configuration.go +++ b/pkg/config/configuration.go @@ -127,7 +127,11 @@ type Plaid struct { WebhooksEnabled bool WebhooksDomain string - OAuthDomain string + // OAuthDomain is used to specify the domain name that the user will be brought to upon returning to monetr after + // authenticating to a bank that requires OAuth. This will typically be a UI domain name and should not include a + // protocol or a path. The protocol is auto inserted as `https` as it is the only protocol supported. The path is + // currently hard coded until a need for different paths arises? + OAuthDomain string } type CORS struct { diff --git a/pkg/controller/links.go b/pkg/controller/links.go index ffa01486..921dc188 100644 --- a/pkg/controller/links.go +++ b/pkg/controller/links.go @@ -2,15 +2,16 @@ package controller import ( "fmt" + "net/http" + "strings" + "time" + "github.com/getsentry/sentry-go" "github.com/kataras/iris/v12" "github.com/monetr/rest-api/pkg/crumbs" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/swag" "github.com/sirupsen/logrus" - "net/http" - "strings" - "time" ) func (c *Controller) linksController(p iris.Party) { @@ -271,7 +272,6 @@ func (c *Controller) deleteLink(ctx iris.Context) { client, err := c.plaid.NewClient(c.getContext(ctx), link, accessToken) if err != nil { - // ERROR THINGS c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create plaid client") return } diff --git a/pkg/controller/plaid.go b/pkg/controller/plaid.go index 66829469..b840d843 100644 --- a/pkg/controller/plaid.go +++ b/pkg/controller/plaid.go @@ -3,6 +3,10 @@ package controller import ( "context" "fmt" + "net/http" + "strconv" + "time" + "github.com/getsentry/sentry-go" "github.com/kataras/iris/v12" "github.com/monetr/rest-api/pkg/crumbs" @@ -11,9 +15,6 @@ import ( "github.com/monetr/rest-api/pkg/models" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "net/http" - "strconv" - "time" "github.com/kataras/iris/v12/core/router" ) @@ -428,10 +429,9 @@ func (c *Controller) plaidTokenCallback(ctx iris.Context) { Mask: plaidAccount.GetMask(), PlaidName: plaidAccount.GetName(), PlaidOfficialName: plaidAccount.GetOfficialName(), - // THIS MIGHT BREAK SOMETHING. - //Type: models.BankAccountType(plaidAccount.Type), - //SubType: models.BankAccountSubType(plaidAccount.Subtype), - LastUpdated: now, + Type: models.BankAccountType(plaidAccount.GetType()), + SubType: models.BankAccountSubType(plaidAccount.GetSubType()), + LastUpdated: now, } } if err = repo.CreateBankAccounts(c.getContext(ctx), accounts...); err != nil { diff --git a/pkg/internal/mock_http_helper/responder_test.go b/pkg/internal/mock_http_helper/responder_test.go index fe80c6c7..4b00ad88 100644 --- a/pkg/internal/mock_http_helper/responder_test.go +++ b/pkg/internal/mock_http_helper/responder_test.go @@ -2,11 +2,12 @@ package mock_http_helper import ( "encoding/json" - "github.com/jarcoal/httpmock" - "github.com/stretchr/testify/assert" "io/ioutil" "net/http" "testing" + + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" ) func TestNewHttpMockJsonResponder(t *testing.T) { @@ -25,6 +26,7 @@ func TestNewHttpMockJsonResponder(t *testing.T) { response, err := http.Get(url) assert.NoError(t, err, "http get request must succeed") assert.Equal(t, http.StatusOK, response.StatusCode, "status code must be 200") + assert.Equal(t, "application/json", response.Header.Get("Content-Type"), "content type should always be json") body, err := ioutil.ReadAll(response.Body) assert.NoError(t, err, "must be able to read the response body") @@ -36,4 +38,5 @@ func TestNewHttpMockJsonResponder(t *testing.T) { assert.EqualValues(t, 123, result["value"], "value must match") assert.Len(t, result, 1, "must only have one key") + } diff --git a/pkg/internal/mock_plaid/accounts.go b/pkg/internal/mock_plaid/accounts.go index 29f3f22c..b3397f63 100644 --- a/pkg/internal/mock_plaid/accounts.go +++ b/pkg/internal/mock_plaid/accounts.go @@ -3,15 +3,16 @@ package mock_plaid import ( "encoding/json" "fmt" + "net/http" + "strings" + "testing" + "github.com/brianvoe/gofakeit/v6" "github.com/monetr/rest-api/pkg/internal/mock_http_helper" "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/internal/testutils" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" - "net/http" - "strings" - "testing" ) func BankAccountFixture(t *testing.T) plaid.AccountBase { @@ -82,8 +83,8 @@ func MockGetAccountsExtended(t *testing.T, plaidData *testutils.MockPlaidData) { func(t *testing.T, request *http.Request) (interface{}, int) { accessToken := ValidatePlaidAuthentication(t, request, RequireAccessToken) var getAccountsRequest struct { - Options struct { - AccountIds []string `json:"account_ids"` + Options struct { + AccountIds []string `json:"account_ids"` } `json:"options"` } require.NoError(t, json.NewDecoder(request.Body).Decode(&getAccountsRequest), "must decode request") @@ -99,7 +100,7 @@ func MockGetAccountsExtended(t *testing.T, plaidData *testutils.MockPlaidData) { for _, accountId := range getAccountsRequest.Options.AccountIds { account, ok := accounts[accountId] if !ok { - panic("bad account id handling not yet implemented") + panic("bad account id handling not yet implemented") } response.Accounts = append(response.Accounts, account) diff --git a/pkg/internal/mock_plaid/accounts_test.go b/pkg/internal/mock_plaid/accounts_test.go index a86635e8..17021211 100644 --- a/pkg/internal/mock_plaid/accounts_test.go +++ b/pkg/internal/mock_plaid/accounts_test.go @@ -1,11 +1,19 @@ package mock_plaid import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestBankAccountFixture(t *testing.T) { account := BankAccountFixture(t) assert.NotEmpty(t, account, "account must not be empty") + assert.NotEmpty(t, account.GetAccountId(), "bank account ID must not be empty") + + balances := account.GetBalances() + assert.NotEmpty(t, balances, "balances must not be empty") + assert.NotZero(t, balances.GetAvailable(), "available must not be zero") + assert.NotZero(t, balances.GetCurrent(), "available must not be zero") + assert.LessOrEqual(t, balances.GetAvailable(), balances.GetCurrent(), "available must always be less than or equal to current") } diff --git a/pkg/internal/mock_plaid/link.go b/pkg/internal/mock_plaid/link.go index ec12de75..d6908357 100644 --- a/pkg/internal/mock_plaid/link.go +++ b/pkg/internal/mock_plaid/link.go @@ -2,13 +2,14 @@ package mock_plaid import ( "encoding/json" + "net/http" + "testing" + "time" + "github.com/brianvoe/gofakeit/v6" "github.com/monetr/rest-api/pkg/internal/mock_http_helper" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" - "net/http" - "testing" - "time" ) func MockCreateLinkToken(t *testing.T) { @@ -22,6 +23,10 @@ func MockCreateLinkToken(t *testing.T) { require.NotEmpty(t, createLinkTokenRequest.ClientName, "client name is required") require.NotEmpty(t, createLinkTokenRequest.Language, "language is required") + if createLinkTokenRequest.AccessToken != nil { + require.Empty(t, createLinkTokenRequest.Products, "products array must be empty when updating a link") + } + return plaid.LinkTokenCreateResponse{ LinkToken: gofakeit.UUID(), Expiration: time.Now().Add(30 * time.Second), diff --git a/pkg/internal/mock_plaid/plaid_test.go b/pkg/internal/mock_plaid/plaid_test.go new file mode 100644 index 00000000..a226406c --- /dev/null +++ b/pkg/internal/mock_plaid/plaid_test.go @@ -0,0 +1,16 @@ +package mock_plaid + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPlaidHeaders(t *testing.T) { + assert.NotPanics(t, func() { + headers := PlaidHeaders(t, nil, nil, http.StatusOK) + assert.NotEmpty(t, headers, "headers should not be empty") + assert.Contains(t, headers, "X-Request-Id", "should contain the X-Request-Id header") + }, "this method must not panic if we pass a nil request and response") +} diff --git a/pkg/internal/mock_plaid/verification.go b/pkg/internal/mock_plaid/verification.go index bbf2b4a1..9d4c8cf7 100644 --- a/pkg/internal/mock_plaid/verification.go +++ b/pkg/internal/mock_plaid/verification.go @@ -2,14 +2,15 @@ package mock_plaid import ( "encoding/json" + "net/http" + "testing" + "time" + "github.com/brianvoe/gofakeit/v6" "github.com/monetr/rest-api/pkg/internal/mock_http_helper" "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" - "net/http" - "testing" - "time" ) func MockGetWebhookVerificationKey(t *testing.T) { @@ -17,16 +18,18 @@ func MockGetWebhookVerificationKey(t *testing.T) { "POST", Path(t, "/webhook_verification_key/get"), func(t *testing.T, request *http.Request) (interface{}, int) { ValidatePlaidAuthentication(t, request, DoNotRequireAccessToken) - var getWebhookVerificationKeyRequest struct { - KeyId string `json:"kid"` - } - require.NoError(t, json.NewDecoder(request.Body).Decode(&getWebhookVerificationKeyRequest), "must decode request") + var requestBody plaid.WebhookVerificationKeyGetRequest + require.NoError(t, json.NewDecoder(request.Body).Decode(&requestBody), "must decode request") + // TODO webhooks: Properly implement testing for webhook verification keys. + // I have little to know idea how to actually implement the key issuing side of this code. I will need to learn + // how it actually works and then build a mock system off of it. In the mean time I will leave this blank for + // now. return plaid.WebhookVerificationKeyGetResponse{ Key: plaid.JWKPublicKey{ Alg: "", Crv: "", - Kid: "", + Kid: requestBody.KeyId, Kty: "", Use: "", X: "", diff --git a/pkg/internal/myownsanity/number_test.go b/pkg/internal/myownsanity/number_test.go index 8e166385..5afe8123 100644 --- a/pkg/internal/myownsanity/number_test.go +++ b/pkg/internal/myownsanity/number_test.go @@ -1,12 +1,33 @@ package myownsanity import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) +func TestFloat32P(t *testing.T) { + var input float32 = 1.2345 + result := Float32P(input) + assert.NotNil(t, result, "resulting pointer should never be nil") + assert.Equal(t, input, *result, "and the underlying value should match the input") +} + +func TestInt32P(t *testing.T) { + var input int32 = 12345 + result := Int32P(input) + assert.NotNil(t, result, "resulting pointer should never be nil") + assert.Equal(t, input, *result, "and the underlying value should match the input") +} + func TestMax(t *testing.T) { assert.Equal(t, 2, Max(1, 2)) assert.Equal(t, 1000, Max(1000, 100)) assert.Equal(t, 500, Max(500, 500)) } + +func TestMin(t *testing.T) { + assert.Equal(t, 1, Min(1, 2)) + assert.Equal(t, 100, Min(1000, 100)) + assert.Equal(t, 500, Min(500, 500)) +} diff --git a/pkg/internal/myownsanity/strings_test.go b/pkg/internal/myownsanity/strings_test.go new file mode 100644 index 00000000..6e8a69a6 --- /dev/null +++ b/pkg/internal/myownsanity/strings_test.go @@ -0,0 +1,40 @@ +package myownsanity + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringP(t *testing.T) { + var input string = "12345" + result := StringP(input) + assert.NotNil(t, result, "resulting pointer should never be nil") + assert.Equal(t, input, *result, "and the underlying value should match the input") +} + +func TestStringDefault(t *testing.T) { + t.Run("value", func(t *testing.T) { + inputString := "I am a string" + defaultString := "I am the default string" + result := StringDefault(&inputString, defaultString) + assert.Equal(t, inputString, result, "result should match the input string") + assert.NotEqual(t, defaultString, result, "and should definitely not match the default string") + }) + + t.Run("nil", func(t *testing.T) { + defaultString := "I am stil the default string" + result := StringDefault(nil, defaultString) + assert.Equal(t, defaultString, result, "when the input is nil, the default should be returned") + }) +} + +func TestSliceContains(t *testing.T) { + data := []string{ + "Item #1", + "Item #2", + } + + assert.True(t, SliceContains(data, "Item #1"), "should contain item #1") + assert.False(t, SliceContains(data, "Item #3"), "should contain item #3") +} diff --git a/pkg/internal/myownsanity/times_test.go b/pkg/internal/myownsanity/times_test.go new file mode 100644 index 00000000..6d20cb4e --- /dev/null +++ b/pkg/internal/myownsanity/times_test.go @@ -0,0 +1,30 @@ +package myownsanity + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestTimeP(t *testing.T) { + input := time.Now() + result := TimeP(input) + assert.NotNil(t, result, "output must not be nil") + assert.EqualValues(t, input, *result, "output's value must match input") +} + +func TestTimesPEqual(t *testing.T) { + t.Run("equal", func(t *testing.T) { + input := time.Now() + a, b := &input, &input + assert.True(t, TimesPEqual(a, b), "should be equal") + }) + + t.Run("nils", func(t *testing.T) { + input := time.Now() + assert.False(t, TimesPEqual(nil, &input), "should not be equal when one is nil") + assert.False(t, TimesPEqual(&input, nil), "should not be equal when one is nil") + assert.True(t, TimesPEqual(nil, nil), "but should be equal if they are both nil") + }) +} diff --git a/pkg/internal/platypus/account.go b/pkg/internal/platypus/account.go index f5de0e99..f7e62748 100644 --- a/pkg/internal/platypus/account.go +++ b/pkg/internal/platypus/account.go @@ -16,6 +16,10 @@ type ( GetName() string // GetOfficialName will return the name of the account specified by the financial institution itself. GetOfficialName() string + // GetType will return the plaid type of the account. For our use case this is typically "depository". + GetType() string + // GetSubType will return the sub-type of the account. This can be something like "checking" or "savings". + GetSubType() string } BankAccountBalances interface { @@ -91,6 +95,8 @@ func NewPlaidBankAccount(bankAccount plaid.AccountBase) (PlaidBankAccount, error Mask: bankAccount.GetMask(), Name: bankAccount.GetName(), OfficialName: bankAccount.GetOfficialName(), + Type: string(bankAccount.GetType()), + SubType: string(bankAccount.GetSubtype()), }, nil } @@ -100,6 +106,8 @@ type PlaidBankAccount struct { Mask string Name string OfficialName string + Type string + SubType string } func (p PlaidBankAccount) GetAccountId() string { @@ -121,3 +129,11 @@ func (p PlaidBankAccount) GetName() string { func (p PlaidBankAccount) GetOfficialName() string { return p.OfficialName } + +func (p PlaidBankAccount) GetType() string { + return p.Type +} + +func (p PlaidBankAccount) GetSubType() string { + return p.SubType +} diff --git a/pkg/internal/platypus/account_test.go b/pkg/internal/platypus/account_test.go index 588a3f1d..bf95d5c3 100644 --- a/pkg/internal/platypus/account_test.go +++ b/pkg/internal/platypus/account_test.go @@ -75,6 +75,8 @@ func TestNewPlaidBankAccount(t *testing.T) { assert.EqualValues(t, "1234", bank.GetMask(), "mask must match") assert.EqualValues(t, "Checking Account", bank.GetName(), "name must match") assert.EqualValues(t, "CHECKING - 1234", bank.GetOfficialName(), "official name must match") + assert.EqualValues(t, "depository", bank.GetType(), "account type must match") + assert.EqualValues(t, "checking", bank.GetSubType(), "account sub-type must match") assert.IsType(t, PlaidBankAccountBalances{}, bank.GetBalances(), "must return plaid bank account balances") }) diff --git a/pkg/internal/platypus/platypus.go b/pkg/internal/platypus/platypus.go index 8e998aa1..e61ee953 100644 --- a/pkg/internal/platypus/platypus.go +++ b/pkg/internal/platypus/platypus.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" "fmt" + "net/http" + "time" + "github.com/getsentry/sentry-go" "github.com/monetr/rest-api/pkg/config" "github.com/monetr/rest-api/pkg/crumbs" @@ -14,8 +17,6 @@ import ( "github.com/pkg/errors" "github.com/plaid/plaid-go/plaid" "github.com/sirupsen/logrus" - "net/http" - "time" ) var ( @@ -52,6 +53,11 @@ func after(span *sentry.Span, response *http.Response, err error, message, error data := map[string]interface{}{} + // With plaid responses we can actually still use the body of the response :tada:. The request Id is also stored on + // the response body itself in most of my testing. I could have sworn the documentation cited X-Request-Id as being + // a possible source for it, but I have not seen that yet. This bit of code extracts the body into a map. I know to + // some degree of certainty that the response will always be an object and not an array. So a map with a string key + // is safe. I can then extract the request Id and store that with my logging and diagnostic data. { var extractedResponseBody map[string]interface{} if e := json.NewDecoder(response.Body).Decode(&extractedResponseBody); e == nil { @@ -59,16 +65,21 @@ func after(span *sentry.Span, response *http.Response, err error, message, error requestId = extractedResponseBody["request_id"].(string) } + // But if our request was not successful, then I also want to yoink that body and throw it into my diagnostic + // data as well. This will help me if I ever need to track down bugs with Plaid's API or problems with requests + // that I am making incorrectly. if response.StatusCode != http.StatusOK { data["body"] = extractedResponseBody } } } - data["X-RequestId"] = requestId + { // Make sure we put the request ID everywhere, this is easily the most important diagnostic data we need. + data["X-RequestId"] = requestId + span.Data["plaidRequestId"] = requestId + span.SetTag("plaidRequestId", requestId) + } - span.Data["plaidRequestId"] = requestId - span.SetTag("plaidRequestId", requestId) crumbs.HTTP( span.Context(), message, @@ -79,12 +90,14 @@ func after(span *sentry.Span, response *http.Response, err error, message, error data, ) } + + // Properly set the span status for this request. if err != nil { span.Status = sentry.SpanStatusInternalError + } else { + span.Status = sentry.SpanStatusOK } - span.Status = sentry.SpanStatusOK - return errors.Wrap(err, errorMessage) }