diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml deleted file mode 100644 index 525a921b..00000000 --- a/.github/workflows/docker-build.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Build - -on: - push: - branches: - - '!renovate/*' - - staging - - trying -# pull_request: -# branches: -# - main - -jobs: - docker: - name: Docker - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Build Image - id: docker_build - uses: docker/build-push-action@v2 - with: - push: false - tags: containers.monetr.dev/rest-api diff --git a/.github/workflows/docs-deploy.yaml b/.github/workflows/docs-deploy.yaml deleted file mode 100644 index 359a0f0e..00000000 --- a/.github/workflows/docs-deploy.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: Documentation - -on: - push: - branches: - - main - -jobs: - build: - name: Generate - runs-on: ubuntu-latest - container: ghcr.io/monetr/build-containers/ubuntu:20.04-2021.05.01 - steps: - - uses: actions/checkout@v2 - - run: yarn install - - run: make dependencies - - run: make docs - - run: $GITHUB_WORKSPACE/node_modules/.bin/redoc-cli bundle $GITHUB_WORKSPACE/docs/swagger.yaml -o $GITHUB_WORKSPACE/docs/index.html - - name: Deploy - uses: JamesIves/github-pages-deploy-action@4.1.5 - with: - branch: gh-pages - folder: docs diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 83d3b54e..00000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Documentation - -on: - pull_request: - branches: [ staging, trying ] - -jobs: - build: - name: Generate - runs-on: ubuntu-latest - container: ghcr.io/monetr/build-containers/ubuntu:20.04-2021.05.01 - steps: - - uses: actions/checkout@v2 - - run: yarn install - - run: make dependencies - - run: make docs - - run: $GITHUB_WORKSPACE/node_modules/.bin/redoc-cli bundle $GITHUB_WORKSPACE/docs/swagger.yaml -o $GITHUB_WORKSPACE/docs/index.html diff --git a/.github/workflows/environments.yaml b/.github/workflows/environments.yaml index 8eb951d2..50cfaf62 100644 --- a/.github/workflows/environments.yaml +++ b/.github/workflows/environments.yaml @@ -6,10 +6,22 @@ on: workflow_dispatch: { } jobs: + release-acceptance: + name: Acceptance + runs-on: ubuntu-latest + container: golang:1.17.1 + steps: + - uses: actions/checkout@v2 + with: + ref: 'main' + - name: fetch + run: git fetch --prune --unshallow + - name: Push To Dog Food + run: git push origin $(go run github.com/monetr/rest-api/tools/releaser --since=-24h):acceptance release-dog: name: Dog Food runs-on: ubuntu-latest - container: golang:1.16.3 + container: golang:1.17.1 steps: - uses: actions/checkout@v2 with: @@ -21,7 +33,7 @@ jobs: release-production: name: Production runs-on: ubuntu-latest - container: golang:1.16.3 + container: golang:1.17.1 steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 5425009d..00000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Go - -on: - push: - branches: [ main, staging, trying ] -# pull_request: -# branches: [ main ] - -jobs: - build: - name: Test - runs-on: ubuntu-latest - container: golang:1.17.0 - env: - POSTGRES_HOST: postgres - POSTGRES_PASSWORD: "" - POSTGRES_USER: api-testing - POSTGRES_DB: test-db - services: - postgres: - image: postgres:13 - env: - POSTGRES_HOST_AUTH_METHOD: trust - POSTGRES_USER: api-testing - POSTGRES_DB: test-db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - uses: actions/checkout@v2 - - name: Dependencies - run: make dependencies - - name: Build - run: make build - - name: Setup Schema - run: make apply-schema-ci - - name: Test - run: make test -# license: -# name: License -# runs-on: ubuntu-latest -# container: ghcr.io/monetr/build-containers/ubuntu:20.04-2021.08.17 -# steps: -# - uses: actions/checkout@v2 -# - name: Licenses -# run: make license diff --git a/.github/workflows/incoming.yml b/.github/workflows/incoming.yml new file mode 100644 index 00000000..be9384e5 --- /dev/null +++ b/.github/workflows/incoming.yml @@ -0,0 +1,102 @@ +name: GitHub + +on: + push: + branches: + - staging + - trying + +jobs: + test: + name: Test + runs-on: ubuntu-latest + container: golang:1.17.0 + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: api-testing + POSTGRES_DB: test-db + services: + postgres: + image: postgres:13 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: api-testing + POSTGRES_DB: test-db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + - name: Dependencies + run: make dependencies + - name: Build + run: make build + - name: Setup Schema + run: make apply-schema-ci + - name: Test + run: make test + pg-test: + name: PostgreSQL Test + runs-on: ubuntu-latest + container: ghcr.io/monetr/pgtest:latest + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: postgres + POSTGRES_DB: test-db + POSTGRES_HOST_AUTH_METHOD: trust + services: + postgres: + image: ghcr.io/monetr/pgtest:latest + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: postgres + POSTGRES_DB: test-db + POSTGRES_HOST_AUTH_METHOD: trust + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + - name: Test + run: make pg_test + - name: Publish Test Report + uses: mikepenz/action-junit-report@v2 + if: always() # always run even if the previous step fails + with: + report_paths: '/junit.xml' + check_name: 'PostgreSQL Test Summary' + docs-generate: + name: Generate Documentation + runs-on: ubuntu-latest + container: ghcr.io/monetr/build-containers/ubuntu:20.04 + steps: + - uses: actions/checkout@v2 + - run: yarn install + - run: make dependencies + - run: make docs + - run: $GITHUB_WORKSPACE/node_modules/.bin/redoc-cli bundle $GITHUB_WORKSPACE/docs/swagger.yaml -o $GITHUB_WORKSPACE/docs/index.html + docker: + needs: + - test + name: Docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build Image + id: docker_build + uses: docker/build-push-action@v2 + with: + push: false + tags: containers.monetr.dev/rest-api + diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 00000000..99fbc2b1 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,90 @@ +name: Main + +on: + push: + branches: + - main + +jobs: + test: + name: Test + runs-on: ubuntu-latest + container: golang:1.17.0 + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: api-testing + POSTGRES_DB: test-db + services: + postgres: + image: postgres:13 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: api-testing + POSTGRES_DB: test-db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + - name: Dependencies + run: make dependencies + - name: Build + run: make build + - name: Setup Schema + run: make apply-schema-ci + - name: Test + run: make test + pg-test: + name: PostgreSQL Test + runs-on: ubuntu-latest + container: ghcr.io/monetr/pgtest:latest + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: postgres + POSTGRES_DB: test-db + POSTGRES_HOST_AUTH_METHOD: trust + services: + postgres: + image: ghcr.io/monetr/pgtest:latest + env: + POSTGRES_HOST: postgres + POSTGRES_PASSWORD: "" + POSTGRES_USER: postgres + POSTGRES_DB: test-db + POSTGRES_HOST_AUTH_METHOD: trust + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + - name: Test + run: make pg_test + - name: Publish Test Report + uses: mikepenz/action-junit-report@v2 + if: always() # always run even if the previous step fails + with: + report_paths: '/junit.xml' + check_name: 'PostgreSQL Test Summary' + docs-deploy: + needs: + - test + name: Deploy Documentation + runs-on: ubuntu-latest + container: ghcr.io/monetr/build-containers/ubuntu:20.04 + steps: + - uses: actions/checkout@v2 + - run: yarn install + - run: make dependencies + - run: make docs + - run: $GITHUB_WORKSPACE/node_modules/.bin/redoc-cli bundle $GITHUB_WORKSPACE/docs/swagger.yaml -o $GITHUB_WORKSPACE/docs/index.html + - name: Deploy + uses: JamesIves/github-pages-deploy-action@4.1.5 + with: + branch: gh-pages + folder: docs diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d61556b9..9bda5435 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,8 +1,9 @@ name: Daily Release on: - schedule: - - cron: 0 23 * * * + push: + branches: + - acceptance workflow_dispatch: { } jobs: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e0bec041..918c006b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ # exception for staging branches that bors creates. workflow: rules: - - if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "staging" || $CI_COMMIT_BRANCH == "trying"' + - if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "staging" || $CI_COMMIT_BRANCH == "trying" || $CI_COMMIT_BRANCH == "dog" || $CI_COMMIT_BRANCH == "acceptance"' when: always - if: '$CI_COMMIT_BRANCH == "staging" || $CI_COMMIT_BRANCH == "trying"' when: never @@ -12,8 +12,8 @@ workflow: when: never - if: $CI_COMMIT_TAG when: always - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != "main"' - when: never +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != "main"' +# when: never stages: - Release @@ -47,7 +47,7 @@ Release Testing: Go Dependencies: needs: [ ] stage: Dependencies - image: containers.monetr.dev/golang:1.16.3 + image: containers.monetr.dev/golang:1.17.0 tags: - k8s:shared - arch:amd64 @@ -63,7 +63,7 @@ Binary: needs: - "Go Dependencies" stage: Build - image: containers.monetr.dev/golang:1.16.3 + image: containers.monetr.dev/golang:1.17.0 tags: - k8s:shared - arch:amd64 @@ -89,7 +89,7 @@ Go Tests: POSTGRES_DB: test-db POSTGRES_HOST_AUTH_METHOD: trust stage: Test - image: containers.monetr.dev/golang:1.16.3 + image: containers.monetr.dev/golang:1.17.0 tags: - k8s:shared - arch:amd64 @@ -154,6 +154,7 @@ Yeet Coverage: refs: - main - tags + - acceptance image: proxy.monetr.dev/docker:19.03.15 variables: DOCKER_HOST: tcp://docker:2375 @@ -241,6 +242,9 @@ Generate - Acceptance: expire_in: 7 days Generate - Dog: + only: + refs: + - dog needs: [ ] stage: Prepare image: containers.monetr.dev/ubuntu:20.04 @@ -301,7 +305,7 @@ Dry - Staging: Dry - Acceptance: only: refs: - - tags + - accceptance environment: name: Acceptance url: https://api.acceptance.monetr.dev @@ -356,7 +360,7 @@ Deploy - Staging: Deploy - Acceptance: only: refs: - - tags + - accceptance environment: name: Acceptance url: https://api.acceptance.monetr.dev diff --git a/Makefile b/Makefile index 9560e2e6..0f1e4a09 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,19 @@ ifdef GITHUB_ACTION include Makefile.github-actions endif +# PostgreSQL tests currently only work in CI pipelines. +ifdef CI +PG_TEST_EXTENSION_QUERY = "CREATE EXTENSION pgtap;" +JUNIT_OUTPUT_FILE=/junit.xml +pg_test: + @for FILE in $(PWD)/schema/*.up.sql; do \ + echo "Applying $$FILE"; \ + psql -q -d $(POSTGRES_DB) -U $(POSTGRES_USER) -h $(POSTGRES_HOST) -f $$FILE || exit 1; \ + done; + psql -q -d $(POSTGRES_DB) -U $(POSTGRES_USER) -h $(POSTGRES_HOST) -c $(PG_TEST_EXTENSION_QUERY) + -JUNIT_OUTPUT_FILE=$(JUNIT_OUTPUT_FILE) pg_prove -h $(POSTGRES_HOST) -U $(POSTGRES_USER) -d $(POSTGRES_DB) -f -c $(PWD)/tests/pg/*.sql --verbose --harness TAP::Harness::JUnit +endif + include Makefile.release include Makefile.docker diff --git a/Makefile.gitlab-ci b/Makefile.gitlab-ci index 363544a9..3dd0c110 100644 --- a/Makefile.gitlab-ci +++ b/Makefile.gitlab-ci @@ -15,19 +15,3 @@ ifdef GOPATH apply-schema-ci: go run $(MONETR_CLI_PACKAGE) database migrate -d $(POSTGRES_DB) -U $(POSTGRES_USER) -H $(POSTGRES_HOST) endif - -apply-schema-ci-psql: - @for FILE in $(CI_PROJECT_DIR)/schema/*.up.sql; do \ - echo "Applying $$FILE"; \ - psql -q -d $(POSTGRES_DB) -U $(POSTGRES_USER) -h $(POSTGRES_HOST) -f $$FILE || exit 1; \ - done; - -PG_PROVE_TESTS = "$(PWD)/tests/pg/*.sql" -PG_TEST_EXTENSION_QUERY = "CREATE EXTENSION pgtap;" - -pg_test: apply-schema-ci-psql - psql -q -d $(POSTGRES_DB) -U $(POSTGRES_USER) -h $(POSTGRES_HOST) -c $(PG_TEST_EXTENSION_QUERY) - make pg_prove_tests - -pg_prove_tests: - pg_prove -h $(POSTGRES_HOST) -U $(POSTGRES_USER) -d $(POSTGRES_DB) -f ./tests/pg/*.sql \ No newline at end of file diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 26ea5e00..21cca300 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -7,7 +7,3 @@ steps: waitFor: [ 'build-docker' ] name: 'gcr.io/cloud-builders/docker' args: [ 'push', 'gcr.io/$PROJECT_ID/github.com/monetr/rest-api:$COMMIT_SHA' ] - - id: 'test-gcloud' - waitFor: [ 'build-docker' ] - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' - args: [ 'gcloud', 'container', 'clusters', 'list' ] diff --git a/go.mod b/go.mod index c24c2c77..57439d6c 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ module github.com/monetr/rest-api -go 1.16 +go 1.17 require ( cloud.google.com/go/logging v1.4.2 github.com/MicahParks/keyfunc v0.4.0 + github.com/OneOfOne/xxhash v1.2.2 + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/alicebob/miniredis/v2 v2.15.1 github.com/brianvoe/gofakeit/v6 v6.7.1 github.com/form3tech-oss/jwt-go v3.2.5+incompatible @@ -26,7 +28,7 @@ require ( github.com/nleeper/goment v1.4.2 github.com/nyaruka/phonenumbers v1.0.71 github.com/pkg/errors v0.9.2-0.20201214064552-5dd12d0cfe7f - github.com/plaid/plaid-go v0.0.0-20210804185432-0d3f02cddaa8 + github.com/plaid/plaid-go v1.0.0 github.com/prometheus/client_golang v1.11.0 github.com/robfig/cron v1.2.0 // indirect github.com/sirupsen/logrus v1.8.1 @@ -34,6 +36,7 @@ require ( github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 github.com/stripe/stripe-go/v72 v72.64.1 + github.com/swaggo/swag v1.7.1 github.com/teambition/rrule-go v1.7.2 github.com/vmihailenco/msgpack/v5 v5.3.4 github.com/xlzd/gotp v0.0.0-20181030022105-c8557ba2c119 @@ -42,3 +45,133 @@ require ( google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af gopkg.in/ezzarghili/recaptcha-go.v4 v4.3.0 ) + +require ( + cloud.google.com/go v0.81.0 // indirect + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect + github.com/CloudyKit/jet/v5 v5.1.1 // indirect + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/ajg/form v1.5.1 // indirect + github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect + github.com/andybalholm/brotli v1.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v3 v3.0.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/chris-ramon/douceur v0.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect + github.com/emirpasic/gods v1.12.0 // indirect + github.com/fatih/structs v1.1.0 // indirect + github.com/flosch/pongo2/v4 v4.0.1 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/spec v0.20.3 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-pg/zerochecker v0.2.0 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/gorilla/css v1.0.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.1 // indirect + github.com/hashicorp/go-multierror v1.1.0 // indirect + github.com/hashicorp/go-retryablehttp v0.6.6 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/vault/sdk v0.2.1 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/imkira/go-interpol v1.1.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/iris-contrib/httpexpect/v2 v2.0.5 // indirect + github.com/iris-contrib/jade v1.1.4 // indirect + github.com/iris-contrib/schema v0.0.6 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/kataras/blocks v0.0.4 // indirect + github.com/kataras/golog v0.1.6 // indirect + github.com/kataras/pio v0.0.10 // indirect + github.com/kataras/sitemap v0.0.5 // indirect + github.com/kataras/tunnel v0.0.2 // indirect + github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect + github.com/klauspost/compress v1.11.3 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/microcosm-cc/bluemonday v1.0.4 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pierrec/lz4 v2.5.2+incompatible // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/schollz/closestmatch v2.1.0+incompatible // indirect + github.com/sergi/go-diff v1.1.0 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/tdewolff/minify/v2 v2.9.10 // indirect + github.com/tdewolff/parse/v2 v2.5.5 // indirect + github.com/tkuchiki/go-timezone v0.2.0 // indirect + github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/vmihailenco/bufpool v0.1.11 // indirect + github.com/vmihailenco/tagparser v0.1.2 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/xanzy/ssh-agent v0.3.0 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect + github.com/yosssi/ace v0.0.5 // indirect + github.com/yudai/gojsondiff v1.0.0 // indirect + github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect + github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect + golang.org/x/tools v0.1.5 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/api v0.46.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/grpc v1.40.0 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/square/go-jose.v2 v2.5.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + mellium.im/sasl v0.2.1 // indirect + moul.io/http2curl v1.0.0 // indirect +) diff --git a/go.sum b/go.sum index 50e8002c..d43de400 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Joker/hpp v1.0.0 h1:65+iuJYdRXv/XyN62C1uEmmOx3432rNG/rKlX6V7Kkc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/MicahParks/keyfunc v0.4.0 h1:+4Gj1EJXy09j6e+S+O9jNNdAOxc6Sra6KWCgbLSkL6E= github.com/MicahParks/keyfunc v0.4.0/go.mod h1:zLNyBGSzTMF3hq4XLLsZsKvxKe0tqHYSfXoFmv9w9g4= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -62,9 +64,14 @@ github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tT github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398 h1:WDC6ySpJzbxGWFh4aMxFFC28wwGp5pEuoTtvA4q/qQ4= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= @@ -72,6 +79,7 @@ github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -147,6 +155,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -228,6 +237,16 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/spec v0.20.3 h1:uH9RQ6vdyPSs2pSy9fL8QPspDF2AMIMPtmK5coSSjtQ= +github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-pg/migrations/v8 v8.1.0 h1:bc1wQwFoWRKvLdluXCRFRkeaw9xDU4qJ63uCAagh66w= github.com/go-pg/migrations/v8 v8.1.0/go.mod h1:o+CN1u572XHphEHZyK6tqyg2GDkRvL2bIoLNyGIewus= github.com/go-pg/pg/v10 v10.4.0/go.mod h1:BfgPoQnD2wXNd986RYEHzikqv9iE875PrFaZ9vXvtNM= @@ -427,6 +446,7 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -489,6 +509,8 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -594,8 +616,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.2-0.20201214064552-5dd12d0cfe7f h1:lJqhwddJVYAkyp72a4pwzMClI20xTwL7miDdm2W/KBM= github.com/pkg/errors v0.9.2-0.20201214064552-5dd12d0cfe7f/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/plaid/plaid-go v0.0.0-20210804185432-0d3f02cddaa8 h1:+fef2TcO+bG6vkiYljmYpBxpTRXg5jllX5Ku0w/kg8E= -github.com/plaid/plaid-go v0.0.0-20210804185432-0d3f02cddaa8/go.mod h1:c7cDT1Lkcr0AgKJGVIG+oCa07jOrrg4Um8nduQ1eQN0= +github.com/plaid/plaid-go v1.0.0 h1:JQEDXHt0K7wNs/WaL/E9DLH1RNYIybO98GiIjekMxYo= +github.com/plaid/plaid-go v1.0.0/go.mod h1:jsPs/+TSYwDPNxMhY2uwlpDUJBnqppGg+pNXNgdITc0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -691,6 +713,8 @@ github.com/stripe/stripe-go/v72 v72.64.1 h1:LsT6QVC8xF4X/Kp8xsNYqvubE3vuXn4/dhOF github.com/stripe/stripe-go/v72 v72.64.1/go.mod h1:QwqJQtduHubZht9mek5sds9CtQcKFdsykV9ZepRWwo0= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/swaggo/swag v1.7.1 h1:gY9ZakXlNWg/i/v5bQBic7VMZ4teq4m89lpiao74p/s= +github.com/swaggo/swag v1.7.1/go.mod h1:gAiHxNTb9cIpNmA/VEGUP+CyZMCP/EW7mdtc8Bny+p8= github.com/tdewolff/minify/v2 v2.9.10 h1:p+ifTTl+JMFFLDYNAm7nxQ9XuCG10HTW00wlPAZ7aoE= github.com/tdewolff/minify/v2 v2.9.10/go.mod h1:U1Nc+/YBSB0FPEarqcgkYH3Ep4DNyyIbOyl5P4eWMuo= github.com/tdewolff/parse/v2 v2.5.5 h1:b7ICJa4I/54JQGEGgTte8DiyJPKcC5g8V773QMzkeUM= @@ -709,6 +733,7 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -1150,8 +1175,6 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 h1:3V2dxSZpz4zozWWUq36vUxXEKnSYitEH2LdsAx+RUmg= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af h1:aLMMXFYqw01RA6XJim5uaN+afqNNjc9P8HPAbnpnc5s= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= diff --git a/minikube/vault.yaml b/minikube/vault.yaml index e8dee26f..93adeba1 100644 --- a/minikube/vault.yaml +++ b/minikube/vault.yaml @@ -268,7 +268,7 @@ metadata: spec: type: ClusterIP ports: - - port: 443 + - port: 80 targetPort: http protocol: TCP name: http @@ -300,7 +300,7 @@ spec: - path: / backend: serviceName: vault - servicePort: 443 + servicePort: 80 --- apiVersion: v1 diff --git a/minikube/vault/auth.tf b/minikube/vault/auth.tf index d6a93a18..d9102320 100644 --- a/minikube/vault/auth.tf +++ b/minikube/vault/auth.tf @@ -9,4 +9,25 @@ resource "vault_kubernetes_auth_backend_config" "example" { token_reviewer_jwt = var.kubernetes_reviewer_jwt issuer = "kubernetes.io/serviceaccount" disable_iss_validation = true -} \ No newline at end of file +} + +resource "vault_auth_backend" "userpass" { + type = "userpass" +} + +// monetr-user is just a basic user-password authentication for the API. This is to be used when trying to debug +// permissions or trying to dignose issues outside of Kubernetes. As it is easier to provision access using a +// username and password than it is to try to use the Kubernetes authentication, outside kube. +// The user is "monetr", the password is "password". +resource "vault_generic_endpoint" "monetr-user" { + depends_on = [vault_auth_backend.userpass] + path = "auth/userpass/users/monetr" + ignore_absent_fields = true + + data_json = jsonencode({ + policies = [ + vault_policy.rest-api-service-policy.name, + ] + password = "password" + }) +} diff --git a/minikube/vault/mounts.tf b/minikube/vault/mounts.tf index 138c9f6b..08cb5c71 100644 --- a/minikube/vault/mounts.tf +++ b/minikube/vault/mounts.tf @@ -1,5 +1,5 @@ resource "vault_mount" "plaid-client-secrets" { - path = "plaid/clients" + path = "customers/plaid" type = "kv-v2" description = "KV store used for Plaid client credentials" } \ No newline at end of file diff --git a/minikube/vault/rest-api-service.tf b/minikube/vault/rest-api-service.tf index 779ff7a3..af8bc1ee 100644 --- a/minikube/vault/rest-api-service.tf +++ b/minikube/vault/rest-api-service.tf @@ -1,4 +1,18 @@ data "vault_policy_document" "rest-api-policy" { + // There are two rules here because I still have no idea what I'm doing when it comes to trying to provision this + // stuff. I'm getting permissions errors for both paths so this is likely to change in the future. But again this + // is only used for local development against vault. + rule { + path = "${vault_mount.plaid-client-secrets.path}/*" + capabilities = [ + "create", + "read", + "update", + "delete", + ] + description = "Allow the REST API to manage client secrets." + } + rule { path = "${vault_mount.plaid-client-secrets.path}/data/*" capabilities = [ @@ -29,4 +43,4 @@ resource "vault_kubernetes_auth_backend_role" "rest-api" { token_policies = [ vault_policy.rest-api-service-policy.name, ] -} \ No newline at end of file +} diff --git a/pkg/config/configuration.go b/pkg/config/configuration.go index 04340a05..c66d93ff 100644 --- a/pkg/config/configuration.go +++ b/pkg/config/configuration.go @@ -127,6 +127,11 @@ type Plaid struct { WebhooksEnabled bool WebhooksDomain string + // OAuthDomain is used to specify the domain name that the user will be brought to upon returning to monetr after + // authenticating to a bank that requires OAuth. This will typically be a UI domain name and should not include a + // protocol or a path. The protocol is auto inserted as `https` as it is the only protocol supported. The path is + // currently hard coded until a need for different paths arises? + OAuthDomain string } type CORS struct { @@ -185,6 +190,7 @@ type Vault struct { Auth string Token string TokenFile string + Username, Password string Role string CertificatePath string KeyPath string @@ -262,6 +268,7 @@ func setupEnv(v *viper.Viper) { v.BindEnv("Plaid.EnableReturningUserExperience", "MONETR_PLAID_RETURNING_EXPERIENCE") v.BindEnv("Plaid.WebhooksEnabled", "MONETR_PLAID_WEBHOOKS_ENABLED") v.BindEnv("Plaid.WebhooksDomain", "MONETR_PLAID_WEBHOOKS_DOMAIN") + v.BindEnv("Plaid.OAuthDomain", "MONETR_PLAID_OAUTH_DOMAIN") v.BindEnv("PostgreSQL.Address", "MONETR_PG_ADDRESS") v.BindEnv("PostgreSQL.Port", "MONETR_PG_PORT") v.BindEnv("PostgreSQL.Username", "MONETR_PG_USERNAME") diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 855b1016..a3670c94 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/mail" "net/http" "net/smtp" @@ -19,7 +20,6 @@ import ( "github.com/monetr/rest-api/pkg/build" "github.com/monetr/rest-api/pkg/cache" "github.com/monetr/rest-api/pkg/config" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/jobs" "github.com/monetr/rest-api/pkg/metrics" @@ -38,8 +38,8 @@ type Controller struct { db *pg.DB configuration config.Configuration captcha *recaptcha.ReCAPTCHA - plaid plaid_helper.Client - plaidWebhookVerification plaid_helper.WebhookVerification + plaid platypus.Platypus + plaidWebhookVerification platypus.WebhookVerification plaidSecrets secrets.PlaidSecretsProvider smtp *smtp.Client mailVerifyCode *gotp.HOTP @@ -61,7 +61,7 @@ func NewController( configuration config.Configuration, db *pg.DB, job jobs.JobManager, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, stripe stripe_helper.Stripe, cachePool *redis.Pool, @@ -85,12 +85,14 @@ func NewController( pubSub := pubsub.NewPostgresPubSub(log, db) basicBilling := billing.NewBasicBilling(log, accountsRepo, pubSub) + plaidWebhookVerification := platypus.NewInMemoryWebhookVerification(log, plaidClient, 5*time.Minute) + return &Controller{ captcha: &captcha, configuration: configuration, db: db, plaid: plaidClient, - plaidWebhookVerification: plaid_helper.NewMemoryWebhookVerificationCache(log, plaidClient), + plaidWebhookVerification: plaidWebhookVerification, plaidSecrets: plaidSecrets, log: log, job: job, diff --git a/pkg/controller/links.go b/pkg/controller/links.go index 0ef61f12..921dc188 100644 --- a/pkg/controller/links.go +++ b/pkg/controller/links.go @@ -2,15 +2,16 @@ package controller import ( "fmt" + "net/http" + "strings" + "time" + "github.com/getsentry/sentry-go" "github.com/kataras/iris/v12" "github.com/monetr/rest-api/pkg/crumbs" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/swag" "github.com/sirupsen/logrus" - "net/http" - "strings" - "time" ) func (c *Controller) linksController(p iris.Party) { @@ -269,7 +270,13 @@ func (c *Controller) deleteLink(ctx iris.Context) { return } - if err = c.plaid.RemoveItem(c.getContext(ctx), accessToken); err != nil { + client, err := c.plaid.NewClient(c.getContext(ctx), link, accessToken) + if err != nil { + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create plaid client") + return + } + + if err = client.RemoveItem(c.getContext(ctx)); err != nil { crumbs.Error(c.getContext(ctx), "Failed to remove item", "plaid", map[string]interface{}{ "linkId": link.LinkId, "itemId": link.PlaidLink.ItemId, diff --git a/pkg/controller/main_test.go b/pkg/controller/main_test.go index 6e8cbfa5..a9347df4 100644 --- a/pkg/controller/main_test.go +++ b/pkg/controller/main_test.go @@ -11,10 +11,12 @@ import ( "github.com/monetr/rest-api/pkg/config" "github.com/monetr/rest-api/pkg/controller" "github.com/monetr/rest-api/pkg/internal/mock_secrets" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/internal/testutils" "github.com/monetr/rest-api/pkg/jobs" + "github.com/monetr/rest-api/pkg/repository" + "github.com/monetr/rest-api/pkg/secrets" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" "net/http" @@ -35,7 +37,9 @@ func NewTestApplicationConfig(t *testing.T) config.Configuration { SMTP: config.SMTPClient{}, ReCAPTCHA: config.ReCAPTCHA{}, Plaid: config.Plaid{ - Environment: plaid.Sandbox, + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, }, CORS: config.CORS{ Debug: false, @@ -54,12 +58,9 @@ func NewTestApplication(t *testing.T) *httptest.Expect { func NewTestApplicationWithConfig(t *testing.T, configuration config.Configuration) *httptest.Expect { log := testutils.GetLog(t) db := testutils.GetPgDatabase(t) - plaidClient := plaid_helper.NewPlaidClient(log, plaid.ClientOptions{ - ClientID: configuration.Plaid.ClientID, - Secret: configuration.Plaid.ClientSecret, - Environment: configuration.Plaid.Environment, - HTTPClient: http.DefaultClient, - }) + secretProvider := secrets.NewPostgresPlaidSecretsProvider(log, db) + plaidRepo := repository.NewPlaidRepository(db) + plaidClient := platypus.NewPlaid(log, secretProvider, plaidRepo, configuration.Plaid) miniRedis := miniredis.NewMiniRedis() require.NoError(t, miniRedis.Start()) diff --git a/pkg/controller/plaid.go b/pkg/controller/plaid.go index e98b6a66..b840d843 100644 --- a/pkg/controller/plaid.go +++ b/pkg/controller/plaid.go @@ -3,18 +3,20 @@ package controller import ( "context" "fmt" + "net/http" + "strconv" + "time" + "github.com/getsentry/sentry-go" "github.com/kataras/iris/v12" "github.com/monetr/rest-api/pkg/crumbs" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/models" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "net/http" - "strconv" - "time" "github.com/kataras/iris/v12/core/router" - "github.com/plaid/plaid-go/plaid" ) func (c *Controller) handlePlaidLinkEndpoints(p router.Party) { @@ -125,67 +127,35 @@ func (c *Controller) newPlaidToken(ctx iris.Context) { } } - plaidProducts := []string{ - "transactions", - } - legalName := "" if len(me.LastName) > 0 { legalName = fmt.Sprintf("%s %s", me.FirstName, me.LastName) } - var phoneNumber string + var phoneNumber *string if me.Login.PhoneNumber != nil { - phoneNumber = me.Login.PhoneNumber.E164() + phoneNumber = myownsanity.StringP(me.Login.PhoneNumber.E164()) } - var webhook string - if c.configuration.Plaid.WebhooksEnabled { - domain := c.configuration.Plaid.WebhooksDomain - if domain != "" { - webhook = fmt.Sprintf("%s/plaid/webhook", c.configuration.Plaid.WebhooksDomain) - } else { - c.log.Errorf("plaid webhooks are enabled, but they cannot be registered with without a domain") - } - } - - redirectUri := fmt.Sprintf("https://%s/plaid/oauth-return", c.configuration.UIDomainName) - - token, err := c.plaid.CreateLinkToken(c.getContext(ctx), plaid.LinkTokenConfigs{ - User: &plaid.LinkTokenUser{ - ClientUserID: strconv.FormatUint(userId, 10), - LegalName: legalName, - PhoneNumber: phoneNumber, - EmailAddress: me.Login.Email, - // TODO (elliotcourant) I'm going to leave these be for now but we need - // to loop back and add this once email/phone verification is working. - PhoneNumberVerifiedTime: time.Time{}, - EmailAddressVerifiedTime: time.Time{}, - }, - ClientName: "monetr", - Products: plaidProducts, - CountryCodes: []string{ - "US", - }, - Webhook: webhook, - AccountFilters: nil, - CrossAppItemAdd: nil, - PaymentInitiation: nil, - Language: "en", - LinkCustomizationName: "", - RedirectUri: redirectUri, + token, err := c.plaid.CreateLinkToken(c.getContext(ctx), platypus.LinkTokenOptions{ + ClientUserID: strconv.FormatUint(userId, 10), + LegalName: legalName, + PhoneNumber: phoneNumber, + PhoneNumberVerifiedTime: nil, + EmailAddress: me.Login.Email, + EmailAddressVerifiedTime: nil, }) if err != nil { c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create link token") return } - if err = c.storeLinkTokenInCache(c.getContext(ctx), log, me.UserId, token.LinkToken, token.Expiration); err != nil { + if err = c.storeLinkTokenInCache(c.getContext(ctx), log, me.UserId, token.Token(), token.Expiration()); err != nil { log.WithError(err).Warn("failed to cache link token") } ctx.JSON(map[string]interface{}{ - "linkToken": token.LinkToken, + "linkToken": token.Token(), }) } @@ -235,72 +205,24 @@ func (c *Controller) updatePlaidLink(ctx iris.Context) { return } - legalName := "" - if len(me.LastName) > 0 { - legalName = fmt.Sprintf("%s %s", me.FirstName, me.LastName) - } else { - // TODO Handle a missing last name, we need a legal name Plaid. - // Should this be considered an error state? - } - - var phoneNumber string - if me.Login.PhoneNumber != nil { - phoneNumber = me.Login.PhoneNumber.E164() - } - - var webhook string - if c.configuration.Plaid.WebhooksEnabled { - domain := c.configuration.Plaid.WebhooksDomain - if domain != "" { - webhook = fmt.Sprintf("%s/plaid/webhook", c.configuration.Plaid.WebhooksDomain) - } else { - log.Errorf("plaid webhooks are enabled, but they cannot be registered with without a domain") - } - } - - accessToken, err := c.plaidSecrets.GetAccessTokenForPlaidLinkId(c.getContext(ctx), repo.AccountId(), link.PlaidLink.ItemId) + client, err := c.plaid.NewClientFromLink(c.getContext(ctx), me.AccountId, linkId) if err != nil { - log.WithError(err).Errorf("failed to retrieve current access token") - c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to retrieve current access token") + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create Plaid client for link") return } - redirectUri := fmt.Sprintf("https://%s/plaid/oauth-return", c.configuration.UIDomainName) - - token, err := c.plaid.CreateLinkToken(c.getContext(ctx), plaid.LinkTokenConfigs{ - User: &plaid.LinkTokenUser{ - ClientUserID: strconv.FormatUint(me.UserId, 10), - LegalName: legalName, - PhoneNumber: phoneNumber, - EmailAddress: me.Login.Email, - // TODO Add in email/phone verification. - PhoneNumberVerifiedTime: time.Time{}, - EmailAddressVerifiedTime: time.Time{}, - }, - ClientName: "monetr", - CountryCodes: []string{ - "US", - }, - Webhook: webhook, - AccountFilters: nil, - CrossAppItemAdd: nil, - PaymentInitiation: nil, - Language: "en", - LinkCustomizationName: "", - RedirectUri: redirectUri, - AccessToken: accessToken, - }) + token, err := client.UpdateItem(c.getContext(ctx)) if err != nil { - c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create link token") + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create link token to update Plaid link") return } - if err = c.storeLinkTokenInCache(c.getContext(ctx), log, me.UserId, token.LinkToken, token.Expiration); err != nil { + if err = c.storeLinkTokenInCache(c.getContext(ctx), log, me.UserId, token.Token(), token.Expiration()); err != nil { log.WithError(err).Warn("failed to cache link token") } ctx.JSON(map[string]interface{}{ - "linkToken": token.LinkToken, + "linkToken": token.Token(), }) } @@ -424,19 +346,6 @@ func (c *Controller) plaidTokenCallback(ctx iris.Context) { return } - plaidAccounts, err := c.plaid.GetAccounts(c.getContext(ctx), result.AccessToken, plaid.GetAccountsOptions{ - AccountIDs: callbackRequest.AccountIds, - }) - if err != nil { - c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to retrieve accounts") - return - } - - if len(plaidAccounts) == 0 { - c.returnError(ctx, http.StatusInternalServerError, "could not retrieve details for any accounts") - return - } - repo := c.mustGetAuthenticatedRepository(ctx) var webhook string @@ -452,7 +361,7 @@ func (c *Controller) plaidTokenCallback(ctx iris.Context) { if err = c.plaidSecrets.UpdateAccessTokenForPlaidLinkId( c.getContext(ctx), repo.AccountId(), - result.ItemID, + result.ItemId, result.AccessToken, ); err != nil { log.WithError(err).Errorf("failed to store access token") @@ -461,7 +370,7 @@ func (c *Controller) plaidTokenCallback(ctx iris.Context) { } plaidLink := models.PlaidLink{ - ItemId: result.ItemID, + ItemId: result.ItemId, Products: []string{ // TODO (elliotcourant) Make this based on what product's we sent in the create link token request. "transactions", @@ -488,21 +397,40 @@ func (c *Controller) plaidTokenCallback(ctx iris.Context) { return } + // Create a plaid client for the new link. + client, err := c.plaid.NewClient(c.getContext(ctx), &link, result.AccessToken) + if err != nil { + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to create Plaid client") + return + } + + // Then use that client to retrieve that link's bank accounts. + plaidAccounts, err := client.GetAccounts(c.getContext(ctx), callbackRequest.AccountIds...) + if err != nil { + c.wrapAndReturnError(ctx, err, http.StatusInternalServerError, "failed to retrieve accounts") + return + } + + if len(plaidAccounts) == 0 { + c.returnError(ctx, http.StatusInternalServerError, "could not retrieve details for any accounts") + return + } + now := time.Now().UTC() accounts := make([]models.BankAccount, len(plaidAccounts)) for i, plaidAccount := range plaidAccounts { accounts[i] = models.BankAccount{ AccountId: repo.AccountId(), LinkId: link.LinkId, - PlaidAccountId: plaidAccount.AccountID, - AvailableBalance: int64(plaidAccount.Balances.Available * 100), - CurrentBalance: int64(plaidAccount.Balances.Current * 100), - Name: plaidAccount.Name, - Mask: plaidAccount.Mask, - PlaidName: plaidAccount.Name, - PlaidOfficialName: plaidAccount.OfficialName, - Type: models.BankAccountType(plaidAccount.Type), - SubType: models.BankAccountSubType(plaidAccount.Subtype), + PlaidAccountId: plaidAccount.GetAccountId(), + AvailableBalance: plaidAccount.GetBalances().GetAvailable(), + CurrentBalance: plaidAccount.GetBalances().GetCurrent(), + Name: plaidAccount.GetName(), + Mask: plaidAccount.GetMask(), + PlaidName: plaidAccount.GetName(), + PlaidOfficialName: plaidAccount.GetOfficialName(), + Type: models.BankAccountType(plaidAccount.GetType()), + SubType: models.BankAccountSubType(plaidAccount.GetSubType()), LastUpdated: now, } } diff --git a/pkg/internal/cmd/controllers.go b/pkg/internal/cmd/controllers.go index b03ce59d..1f1323f7 100644 --- a/pkg/internal/cmd/controllers.go +++ b/pkg/internal/cmd/controllers.go @@ -9,7 +9,7 @@ import ( "github.com/monetr/rest-api/pkg/billing" "github.com/monetr/rest-api/pkg/config" "github.com/monetr/rest-api/pkg/controller" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/jobs" "github.com/monetr/rest-api/pkg/metrics" @@ -22,7 +22,7 @@ func getControllers( configuration config.Configuration, db *pg.DB, job jobs.JobManager, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, stripe stripe_helper.Stripe, cache *redis.Pool, diff --git a/pkg/internal/cmd/controllers_ui.go b/pkg/internal/cmd/controllers_ui.go index 561f23d1..da3c5f55 100644 --- a/pkg/internal/cmd/controllers_ui.go +++ b/pkg/internal/cmd/controllers_ui.go @@ -9,7 +9,7 @@ import ( "github.com/monetr/rest-api/pkg/billing" "github.com/monetr/rest-api/pkg/config" "github.com/monetr/rest-api/pkg/controller" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/jobs" "github.com/monetr/rest-api/pkg/metrics" @@ -23,7 +23,7 @@ func getControllers( configuration config.Configuration, db *pg.DB, job jobs.JobManager, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, stripe stripe_helper.Stripe, cache *redis.Pool, diff --git a/pkg/internal/cmd/serve.go b/pkg/internal/cmd/serve.go index f84d6a23..4ca325af 100644 --- a/pkg/internal/cmd/serve.go +++ b/pkg/internal/cmd/serve.go @@ -6,13 +6,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" - "net" - "net/http" - "os" - "path/filepath" - "time" - "github.com/getsentry/sentry-go" "github.com/go-pg/pg/v10" "github.com/kataras/iris/v12" @@ -24,16 +17,21 @@ import ( "github.com/monetr/rest-api/pkg/internal/certhelper" "github.com/monetr/rest-api/pkg/internal/migrations" "github.com/monetr/rest-api/pkg/internal/myownsanity" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/stripe_helper" "github.com/monetr/rest-api/pkg/internal/vault_helper" "github.com/monetr/rest-api/pkg/jobs" "github.com/monetr/rest-api/pkg/logging" "github.com/monetr/rest-api/pkg/metrics" + "github.com/monetr/rest-api/pkg/repository" "github.com/monetr/rest-api/pkg/secrets" "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" "github.com/spf13/cobra" + "io/ioutil" + "net" + "os" + "path/filepath" + "time" ) func init() { @@ -73,11 +71,13 @@ func RunServer() error { var vault vault_helper.VaultHelper if configuration.Vault.Enabled { client, err := vault_helper.NewVaultHelper(log, vault_helper.Config{ - Address: "http://vault.monetr.in:443", - Role: "rest-api", - Auth: "kubernetes", + Address: configuration.Vault.Address, + Role: configuration.Vault.Role, + Auth: configuration.Vault.Auth, Timeout: 30 * time.Second, IdleConnTimeout: 9 * time.Minute, + Username: configuration.Vault.Username, + Password: configuration.Vault.Password, }) if err != nil { log.WithError(err).Fatalf("failed to create vault helper") @@ -315,15 +315,6 @@ func RunServer() error { basicPaywall = billing.NewBasicPaywall(log, accountRepo) } - plaidHelper := plaid_helper.NewPlaidClient(log, plaid.ClientOptions{ - ClientID: configuration.Plaid.ClientID, - Secret: configuration.Plaid.ClientSecret, - Environment: configuration.Plaid.Environment, - HTTPClient: &http.Client{ - Timeout: 30 * time.Second, - }, - }) - if configuration.Plaid.WebhooksEnabled { log.Debugf("plaid webhooks are enabled and will be sent to: %s", configuration.Plaid.WebhooksDomain) } @@ -339,11 +330,13 @@ func RunServer() error { plaidSecrets = secrets.NewPostgresPlaidSecretsProvider(log, db) } + plaidClient := platypus.NewPlaid(log, plaidSecrets, repository.NewPlaidRepository(db), configuration.Plaid) + jobManager := jobs.NewJobManager( log, redisController.Pool(), db, - plaidHelper, + plaidClient, stats, plaidSecrets, ) @@ -354,7 +347,7 @@ func RunServer() error { configuration, db, jobManager, - plaidHelper, + plaidClient, stats, stripe, redisController.Pool(), diff --git a/pkg/internal/mock_http_helper/responder.go b/pkg/internal/mock_http_helper/responder.go index 1bff2325..e2386fad 100644 --- a/pkg/internal/mock_http_helper/responder.go +++ b/pkg/internal/mock_http_helper/responder.go @@ -17,7 +17,12 @@ func NewHttpMockJsonResponder( headersFn func(t *testing.T, request *http.Request, response interface{}, status int) map[string][]string, ) { httpmock.RegisterResponder(method, path, func(request *http.Request) (*http.Response, error) { - result, status := handler(t, request) + var result interface{} + var status int + require.NotPanics(t, func() { + result, status = handler(t, request) + }, "handle must not panic") + body := bytes.NewBuffer(nil) require.NoError(t, json.NewEncoder(body).Encode(result), "must encode response body") response := &http.Response{ @@ -28,6 +33,11 @@ func NewHttpMockJsonResponder( Body: ioutil.NopCloser(body), ContentLength: int64(body.Len()), Request: request, + Header: map[string][]string{ + "Content-Type": { + "application/json", + }, + }, } headers := headersFn(t, request, result, status) diff --git a/pkg/internal/mock_http_helper/responder_test.go b/pkg/internal/mock_http_helper/responder_test.go new file mode 100644 index 00000000..4b00ad88 --- /dev/null +++ b/pkg/internal/mock_http_helper/responder_test.go @@ -0,0 +1,42 @@ +package mock_http_helper + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "testing" + + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" +) + +func TestNewHttpMockJsonResponder(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + url := "https://monetr.test/thing" + NewHttpMockJsonResponder(t, "GET", url, func(t *testing.T, request *http.Request) (interface{}, int) { + return map[string]interface{}{ + "value": 123, + }, http.StatusOK + }, func(t *testing.T, request *http.Request, response interface{}, status int) map[string][]string { + return nil + }) + + response, err := http.Get(url) + assert.NoError(t, err, "http get request must succeed") + assert.Equal(t, http.StatusOK, response.StatusCode, "status code must be 200") + assert.Equal(t, "application/json", response.Header.Get("Content-Type"), "content type should always be json") + + body, err := ioutil.ReadAll(response.Body) + assert.NoError(t, err, "must be able to read the response body") + assert.NotEmpty(t, body, "body must not be empty") + + var result map[string]interface{} + err = json.Unmarshal(body, &result) + assert.NoError(t, err, "must be able to unmarshal response") + + assert.EqualValues(t, 123, result["value"], "value must match") + assert.Len(t, result, 1, "must only have one key") + +} diff --git a/pkg/internal/mock_mail/mail_test.go b/pkg/internal/mock_mail/mail_test.go index 1d5b8919..2cea9921 100644 --- a/pkg/internal/mock_mail/mail_test.go +++ b/pkg/internal/mock_mail/mail_test.go @@ -25,7 +25,7 @@ func TestMockMailCommunication_Send(t *testing.T) { assert.Len(t, mockMail.Sent, 1, "should have added request to list") }) - t.Run("simple", func(t *testing.T) { + t.Run("none sent", func(t *testing.T) { mockMail := NewMockMail() mockMail.ShouldFail = true diff --git a/pkg/internal/mock_plaid/accounts.go b/pkg/internal/mock_plaid/accounts.go index 9d47e09b..b3397f63 100644 --- a/pkg/internal/mock_plaid/accounts.go +++ b/pkg/internal/mock_plaid/accounts.go @@ -2,37 +2,100 @@ package mock_plaid import ( "encoding/json" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/brianvoe/gofakeit/v6" "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/internal/testutils" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" - "net/http" - "testing" ) +func BankAccountFixture(t *testing.T) plaid.AccountBase { + accountNumber := gofakeit.AchAccount() + require.NotEmpty(t, accountNumber, "account number cannot be empty") + + accountType := gofakeit.RandomString([]string{ + string(plaid.ACCOUNTTYPE_DEPOSITORY), + string(plaid.ACCOUNTTYPE_CREDIT), + string(plaid.ACCOUNTTYPE_INVESTMENT), + string(plaid.ACCOUNTTYPE_LOAN), + }) + + var accountSubType plaid.AccountSubtype + switch plaid.AccountType(accountType) { + case plaid.ACCOUNTTYPE_DEPOSITORY: + accountSubType = plaid.AccountSubtype(gofakeit.RandomString([]string{ + string(plaid.ACCOUNTSUBTYPE_CHECKING), + string(plaid.ACCOUNTSUBTYPE_SAVINGS), + string(plaid.ACCOUNTSUBTYPE_PAYPAL), + })) + case plaid.ACCOUNTTYPE_CREDIT: + accountSubType = plaid.AccountSubtype(gofakeit.RandomString([]string{ + string(plaid.ACCOUNTSUBTYPE_CREDIT_CARD), + string(plaid.ACCOUNTSUBTYPE_PAYPAL), + })) + case plaid.ACCOUNTTYPE_INVESTMENT: + accountSubType = plaid.AccountSubtype(gofakeit.RandomString([]string{ + string(plaid.ACCOUNTSUBTYPE_IRA), + string(plaid.ACCOUNTSUBTYPE_ROTH), + })) + case plaid.ACCOUNTTYPE_LOAN: + accountSubType = plaid.AccountSubtype(gofakeit.RandomString([]string{ + string(plaid.ACCOUNTSUBTYPE_AUTO), + string(plaid.ACCOUNTSUBTYPE_HOME), + })) + } + + mask := accountNumber[len(accountNumber)-4:] + + currencyCode := "USD" + + current := gofakeit.Float32Range(100, 500) + available := gofakeit.Float32Range(current-10, current) + limit := gofakeit.Float32Range(current, current+100) + + return plaid.AccountBase{ + AccountId: gofakeit.Generate("????????????????"), + Balances: plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(myownsanity.Float32P(available)), + Current: *plaid.NewNullableFloat32(myownsanity.Float32P(current)), + Limit: *plaid.NewNullableFloat32(myownsanity.Float32P(limit)), + IsoCurrencyCode: *plaid.NewNullableString(myownsanity.StringP(currencyCode)), + UnofficialCurrencyCode: *plaid.NewNullableString(myownsanity.StringP(currencyCode)), + }, + Mask: *plaid.NewNullableString(myownsanity.StringP(mask)), + Name: fmt.Sprintf("Personal Account - %s", mask), + OfficialName: *plaid.NewNullableString(myownsanity.StringP(fmt.Sprintf("%s - %s", strings.ToUpper(accountType), mask))), + Type: plaid.AccountType(accountType), + Subtype: *plaid.NewNullableAccountSubtype(&accountSubType), + } +} + func MockGetAccountsExtended(t *testing.T, plaidData *testutils.MockPlaidData) { mock_http_helper.NewHttpMockJsonResponder( t, "POST", Path(t, "/accounts/get"), func(t *testing.T, request *http.Request) (interface{}, int) { + accessToken := ValidatePlaidAuthentication(t, request, RequireAccessToken) var getAccountsRequest struct { - ClientId string `json:"client_id"` - Secret string `json:"secret"` - AccessToken string `json:"access_token"` - Options struct { + Options struct { AccountIds []string `json:"account_ids"` } `json:"options"` } require.NoError(t, json.NewDecoder(request.Body).Decode(&getAccountsRequest), "must decode request") - accounts, ok := plaidData.BankAccounts[getAccountsRequest.AccessToken] - if !ok { - panic("invalid access token mocking not implemented") - } + accounts, ok := plaidData.BankAccounts[accessToken] + require.True(t, ok, "invalid access token mocking not implemented") - response := plaid.GetAccountsResponse{ - Accounts: make([]plaid.Account, 0), - Item: plaid.Item{}, // Not yet populating this. + response := plaid.AccountsGetResponse{ + RequestId: gofakeit.UUID(), + Accounts: make([]plaid.AccountBase, 0), + Item: plaid.Item{}, // Not yet populating this. } for _, accountId := range getAccountsRequest.Options.AccountIds { account, ok := accounts[accountId] @@ -49,7 +112,7 @@ func MockGetAccountsExtended(t *testing.T, plaidData *testutils.MockPlaidData) { ) } -func MockGetAccounts(t *testing.T, accounts []plaid.Account) { +func MockGetAccounts(t *testing.T, accounts []plaid.AccountBase) { mock_http_helper.NewHttpMockJsonResponder( t, "POST", Path(t, "/accounts/get"), @@ -87,7 +150,14 @@ func MockGetAccountsError(t *testing.T, plaidError plaid.Error) { } require.NoError(t, json.NewDecoder(request.Body).Decode(&getAccountsRequest), "must decode request") - return plaidError, plaidError.StatusCode + var status int + if s := plaidError.Status.Get(); s != nil { + status = int(*s) + } else { + status = http.StatusInternalServerError + } + + return plaidError, status }, PlaidHeaders, ) diff --git a/pkg/internal/mock_plaid/accounts_test.go b/pkg/internal/mock_plaid/accounts_test.go new file mode 100644 index 00000000..17021211 --- /dev/null +++ b/pkg/internal/mock_plaid/accounts_test.go @@ -0,0 +1,19 @@ +package mock_plaid + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBankAccountFixture(t *testing.T) { + account := BankAccountFixture(t) + assert.NotEmpty(t, account, "account must not be empty") + assert.NotEmpty(t, account.GetAccountId(), "bank account ID must not be empty") + + balances := account.GetBalances() + assert.NotEmpty(t, balances, "balances must not be empty") + assert.NotZero(t, balances.GetAvailable(), "available must not be zero") + assert.NotZero(t, balances.GetCurrent(), "available must not be zero") + assert.LessOrEqual(t, balances.GetAvailable(), balances.GetCurrent(), "available must always be less than or equal to current") +} diff --git a/pkg/internal/mock_plaid/authentication.go b/pkg/internal/mock_plaid/authentication.go new file mode 100644 index 00000000..55e8677b --- /dev/null +++ b/pkg/internal/mock_plaid/authentication.go @@ -0,0 +1,43 @@ +package mock_plaid + +import ( + "bytes" + "encoding/json" + "github.com/stretchr/testify/require" + "io" + "net/http" + "strings" + "testing" +) + +const ( + RequireAccessToken = true + DoNotRequireAccessToken = false +) + +func ValidatePlaidAuthentication(t *testing.T, request *http.Request, requireAccessToken bool) (accessToken string) { + split := bytes.NewBuffer(nil) + bodyReader := io.TeeReader(request.Body, split) + request.Body = io.NopCloser(split) + + var authenticationInBody struct { + ClientId string `json:"client_id"` + Secret string `json:"secret"` + AccessToken string `json:"access_token"` + } + require.NoError(t, json.NewDecoder(bodyReader).Decode(&authenticationInBody), "must decode request") + + if strings.TrimSpace(authenticationInBody.ClientId) == "" { + require.NotEmpty(t, request.Header.Get("Plaid-Client-Id"), "client Id cannot be missing") + } + + if strings.TrimSpace(authenticationInBody.Secret) == "" { + require.NotEmpty(t, request.Header.Get("Plaid-Secret"), "secret cannot be missing") + } + + if requireAccessToken { + require.NotEmpty(t, authenticationInBody.AccessToken, "access token is required") + } + + return authenticationInBody.AccessToken +} diff --git a/pkg/internal/mock_plaid/link.go b/pkg/internal/mock_plaid/link.go new file mode 100644 index 00000000..d6908357 --- /dev/null +++ b/pkg/internal/mock_plaid/link.go @@ -0,0 +1,38 @@ +package mock_plaid + +import ( + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/require" +) + +func MockCreateLinkToken(t *testing.T) { + mock_http_helper.NewHttpMockJsonResponder( + t, + "POST", Path(t, "/link/token/create"), + func(t *testing.T, request *http.Request) (interface{}, int) { + ValidatePlaidAuthentication(t, request, DoNotRequireAccessToken) + var createLinkTokenRequest plaid.LinkTokenCreateRequest + require.NoError(t, json.NewDecoder(request.Body).Decode(&createLinkTokenRequest), "must decode request") + require.NotEmpty(t, createLinkTokenRequest.ClientName, "client name is required") + require.NotEmpty(t, createLinkTokenRequest.Language, "language is required") + + if createLinkTokenRequest.AccessToken != nil { + require.Empty(t, createLinkTokenRequest.Products, "products array must be empty when updating a link") + } + + return plaid.LinkTokenCreateResponse{ + LinkToken: gofakeit.UUID(), + Expiration: time.Now().Add(30 * time.Second), + RequestId: gofakeit.UUID(), + }, http.StatusOK + }, + PlaidHeaders, + ) +} diff --git a/pkg/internal/mock_plaid/mock_exchange.go b/pkg/internal/mock_plaid/mock_exchange.go index e29cde35..b6f5b2e5 100644 --- a/pkg/internal/mock_plaid/mock_exchange.go +++ b/pkg/internal/mock_plaid/mock_exchange.go @@ -4,6 +4,7 @@ import ( "encoding/json" "github.com/brianvoe/gofakeit/v6" "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" "net/http" @@ -20,26 +21,28 @@ func MockExchangePublicToken(t *testing.T) string { t, "POST", Path(t, "/item/public_token/exchange"), func(t *testing.T, request *http.Request) (interface{}, int) { + ValidatePlaidAuthentication(t, request, DoNotRequireAccessToken) var exchangeRequest struct { - ClientID string `json:"client_id"` - Secret string `json:"secret"` PublicToken string `json:"public_token"` } require.NoError(t, json.NewDecoder(request.Body).Decode(&exchangeRequest), "must decode request") + requestId := gofakeit.UUID() if exchangeRequest.PublicToken != publicToken { return plaid.Error{ - ErrorType: "INVALID_REQUEST", - ErrorCode: "1234", - ErrorMessage: "public_token is not valid", - DisplayMessage: "public_token is not valid", - StatusCode: http.StatusBadRequest, + RequestId: &requestId, + ErrorType: "INVALID_REQUEST", + ErrorCode: "1234", + ErrorMessage: "public_token is not valid", + DisplayMessage: *plaid.NewNullableString(myownsanity.StringP("public_token is not valid")), + Status: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(http.StatusBadRequest))), }, http.StatusBadRequest } - return plaid.ExchangePublicTokenResponse{ + return plaid.ItemPublicTokenExchangeResponse{ + RequestId: requestId, AccessToken: gofakeit.UUID(), - ItemID: gofakeit.UUID(), + ItemId: gofakeit.UUID(), }, http.StatusOK }, PlaidHeaders, diff --git a/pkg/internal/mock_plaid/plaid_test.go b/pkg/internal/mock_plaid/plaid_test.go new file mode 100644 index 00000000..a226406c --- /dev/null +++ b/pkg/internal/mock_plaid/plaid_test.go @@ -0,0 +1,16 @@ +package mock_plaid + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPlaidHeaders(t *testing.T) { + assert.NotPanics(t, func() { + headers := PlaidHeaders(t, nil, nil, http.StatusOK) + assert.NotEmpty(t, headers, "headers should not be empty") + assert.Contains(t, headers, "X-Request-Id", "should contain the X-Request-Id header") + }, "this method must not panic if we pass a nil request and response") +} diff --git a/pkg/internal/mock_plaid/transactions.go b/pkg/internal/mock_plaid/transactions.go new file mode 100644 index 00000000..dc14b415 --- /dev/null +++ b/pkg/internal/mock_plaid/transactions.go @@ -0,0 +1,87 @@ +package mock_plaid + +import ( + "encoding/json" + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/require" + "net/http" + "testing" + "time" +) + +func GenerateTransactions(t *testing.T, start, end time.Time, numberOfTransactions int, bankAccountIds []string) []plaid.Transaction { + transactions := make([]plaid.Transaction, numberOfTransactions*len(bankAccountIds)) + for i := 0; i < len(transactions); i++ { + bankAccountId := bankAccountIds[i%len(bankAccountIds)] + + transaction := plaid.Transaction{} + transaction.SetAmount(gofakeit.Float32Range(0.99, 100)) + transaction.SetCategory([]string{ + "Bank Fees", + }) + transaction.SetCategoryId("10000000") + transaction.SetAccountId(bankAccountId) + transaction.SetIsoCurrencyCode("USD") + transaction.SetUnofficialCurrencyCode("USD") + + // Should break down transaction dates evenly over the provided range. + down := end.Add(-(end.Sub(start) / time.Duration(numberOfTransactions*len(bankAccountIds))) * time.Duration(i)) + + transaction.SetDate(down.Format("2006-01-02")) + transaction.SetName(gofakeit.Company()) + transaction.SetTransactionId(gofakeit.Generate("?????????????????????")) + + transactions[i] = transaction + } + + return transactions +} + +func MockGetRandomTransactions(t *testing.T, start, end time.Time, numberOfTransactions int, bankAccountIds []string) { + transactions := GenerateTransactions(t, start, end, numberOfTransactions, bankAccountIds) + mock_http_helper.NewHttpMockJsonResponder( + t, + "POST", Path(t, "/transactions/get"), + func(t *testing.T, request *http.Request) (interface{}, int) { + ValidatePlaidAuthentication(t, request, RequireAccessToken) + var getTransactionsRequest struct { + Options struct { + AccountIds []string `json:"account_ids"` + Count int `json:"count"` + Offset int `json:"offset"` + } `json:"options"` + StartDate string `json:"start_date"` + EndDate string `json:"end_date"` + } + require.NoError(t, json.NewDecoder(request.Body).Decode(&getTransactionsRequest), "must decode request") + + // Make sure our request dates are valid. + _, err := time.Parse("2006-01-02", getTransactionsRequest.StartDate) + require.NoError(t, err, "must provide a valid start date") + _, err = time.Parse("2006-01-02", getTransactionsRequest.EndDate) + require.NoError(t, err, "must provide a valid end date") + + if getTransactionsRequest.Options.Offset > len(transactions) { + return plaid.TransactionsGetResponse{}, http.StatusOK + } + + offset := getTransactionsRequest.Options.Offset + count := getTransactionsRequest.Options.Count + endingOffset := myownsanity.Min(len(transactions), offset+count) + data := transactions[offset:endingOffset] + + return plaid.TransactionsGetResponse{ + Accounts: nil, // Add some basic reporting here too + Transactions: data, + TotalTransactions: int32(len(transactions)), + Item: plaid.Item{}, + RequestId: gofakeit.UUID(), + AdditionalProperties: nil, + }, http.StatusOK + }, + PlaidHeaders, + ) +} diff --git a/pkg/internal/mock_plaid/transactions_test.go b/pkg/internal/mock_plaid/transactions_test.go new file mode 100644 index 00000000..ccce9dec --- /dev/null +++ b/pkg/internal/mock_plaid/transactions_test.go @@ -0,0 +1,37 @@ +package mock_plaid + +import ( + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestGenerateTransactions(t *testing.T) { + t.Run("2 days", func(t *testing.T) { + numberOfTransactions := 600 + bankAccounts := []string{ + "1234", + "5678", + } + end := time.Now() + start := time.Now().Add(-25 * time.Hour) + transactions := GenerateTransactions(t, start, end, numberOfTransactions, bankAccounts) + assert.Len(t, transactions, len(bankAccounts) * numberOfTransactions) + assert.Equal(t, end.Format("2006-01-02"), transactions[0].GetDate(), "date of first transaction should be end") + assert.Equal(t, start.Format("2006-01-02"), transactions[len(transactions)-1].GetDate(), "date of last transaction should be start") + }) + + t.Run("30 days", func(t *testing.T) { + numberOfTransactions := 3000 + bankAccounts := []string{ + "1234", + "5678", + } + end := time.Now() + start := time.Now().Add((-30 * 24 * time.Hour) + (-1 * time.Hour)) + transactions := GenerateTransactions(t, start, end, numberOfTransactions, bankAccounts) + assert.Len(t, transactions, len(bankAccounts) * numberOfTransactions) + assert.Equal(t, end.Format("2006-01-02"), transactions[0].GetDate(), "date of first transaction should be end") + assert.Equal(t, start.Format("2006-01-02"), transactions[len(transactions)-1].GetDate(), "date of last transaction should be start") + }) +} diff --git a/pkg/internal/mock_plaid/verification.go b/pkg/internal/mock_plaid/verification.go new file mode 100644 index 00000000..9d4c8cf7 --- /dev/null +++ b/pkg/internal/mock_plaid/verification.go @@ -0,0 +1,46 @@ +package mock_plaid + +import ( + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/internal/mock_http_helper" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/require" +) + +func MockGetWebhookVerificationKey(t *testing.T) { + mock_http_helper.NewHttpMockJsonResponder(t, + "POST", Path(t, "/webhook_verification_key/get"), + func(t *testing.T, request *http.Request) (interface{}, int) { + ValidatePlaidAuthentication(t, request, DoNotRequireAccessToken) + var requestBody plaid.WebhookVerificationKeyGetRequest + require.NoError(t, json.NewDecoder(request.Body).Decode(&requestBody), "must decode request") + + // TODO webhooks: Properly implement testing for webhook verification keys. + // I have little to know idea how to actually implement the key issuing side of this code. I will need to learn + // how it actually works and then build a mock system off of it. In the mean time I will leave this blank for + // now. + return plaid.WebhookVerificationKeyGetResponse{ + Key: plaid.JWKPublicKey{ + Alg: "", + Crv: "", + Kid: requestBody.KeyId, + Kty: "", + Use: "", + X: "", + Y: "", + CreatedAt: int32(time.Now().Unix()), + ExpiredAt: *plaid.NewNullableInt32(myownsanity.Int32P(int32(time.Now().Add(10 * time.Second).Unix()))), + }, + RequestId: gofakeit.UUID(), + AdditionalProperties: nil, + }, http.StatusOK + }, + PlaidHeaders, + ) +} diff --git a/pkg/internal/myownsanity/number_test.go b/pkg/internal/myownsanity/number_test.go new file mode 100644 index 00000000..5afe8123 --- /dev/null +++ b/pkg/internal/myownsanity/number_test.go @@ -0,0 +1,33 @@ +package myownsanity + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFloat32P(t *testing.T) { + var input float32 = 1.2345 + result := Float32P(input) + assert.NotNil(t, result, "resulting pointer should never be nil") + assert.Equal(t, input, *result, "and the underlying value should match the input") +} + +func TestInt32P(t *testing.T) { + var input int32 = 12345 + result := Int32P(input) + assert.NotNil(t, result, "resulting pointer should never be nil") + assert.Equal(t, input, *result, "and the underlying value should match the input") +} + +func TestMax(t *testing.T) { + assert.Equal(t, 2, Max(1, 2)) + assert.Equal(t, 1000, Max(1000, 100)) + assert.Equal(t, 500, Max(500, 500)) +} + +func TestMin(t *testing.T) { + assert.Equal(t, 1, Min(1, 2)) + assert.Equal(t, 100, Min(1000, 100)) + assert.Equal(t, 500, Min(500, 500)) +} diff --git a/pkg/internal/myownsanity/numbers.go b/pkg/internal/myownsanity/numbers.go new file mode 100644 index 00000000..e0e68cb4 --- /dev/null +++ b/pkg/internal/myownsanity/numbers.go @@ -0,0 +1,25 @@ +package myownsanity + +func Float32P(value float32) *float32 { + return &value +} + +func Int32P(value int32) *int32 { + return &value +} + +func Max(a, b int) int { + if a > b { + return a + } + + return b +} + +func Min(a, b int) int { + if a < b { + return a + } + + return b +} \ No newline at end of file diff --git a/pkg/internal/myownsanity/strings_test.go b/pkg/internal/myownsanity/strings_test.go new file mode 100644 index 00000000..6e8a69a6 --- /dev/null +++ b/pkg/internal/myownsanity/strings_test.go @@ -0,0 +1,40 @@ +package myownsanity + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringP(t *testing.T) { + var input string = "12345" + result := StringP(input) + assert.NotNil(t, result, "resulting pointer should never be nil") + assert.Equal(t, input, *result, "and the underlying value should match the input") +} + +func TestStringDefault(t *testing.T) { + t.Run("value", func(t *testing.T) { + inputString := "I am a string" + defaultString := "I am the default string" + result := StringDefault(&inputString, defaultString) + assert.Equal(t, inputString, result, "result should match the input string") + assert.NotEqual(t, defaultString, result, "and should definitely not match the default string") + }) + + t.Run("nil", func(t *testing.T) { + defaultString := "I am stil the default string" + result := StringDefault(nil, defaultString) + assert.Equal(t, defaultString, result, "when the input is nil, the default should be returned") + }) +} + +func TestSliceContains(t *testing.T) { + data := []string{ + "Item #1", + "Item #2", + } + + assert.True(t, SliceContains(data, "Item #1"), "should contain item #1") + assert.False(t, SliceContains(data, "Item #3"), "should contain item #3") +} diff --git a/pkg/internal/myownsanity/times_test.go b/pkg/internal/myownsanity/times_test.go new file mode 100644 index 00000000..6d20cb4e --- /dev/null +++ b/pkg/internal/myownsanity/times_test.go @@ -0,0 +1,30 @@ +package myownsanity + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestTimeP(t *testing.T) { + input := time.Now() + result := TimeP(input) + assert.NotNil(t, result, "output must not be nil") + assert.EqualValues(t, input, *result, "output's value must match input") +} + +func TestTimesPEqual(t *testing.T) { + t.Run("equal", func(t *testing.T) { + input := time.Now() + a, b := &input, &input + assert.True(t, TimesPEqual(a, b), "should be equal") + }) + + t.Run("nils", func(t *testing.T) { + input := time.Now() + assert.False(t, TimesPEqual(nil, &input), "should not be equal when one is nil") + assert.False(t, TimesPEqual(&input, nil), "should not be equal when one is nil") + assert.True(t, TimesPEqual(nil, nil), "but should be equal if they are both nil") + }) +} diff --git a/pkg/internal/plaid_helper/client.go b/pkg/internal/plaid_helper/client.go deleted file mode 100644 index 4deecad2..00000000 --- a/pkg/internal/plaid_helper/client.go +++ /dev/null @@ -1,303 +0,0 @@ -package plaid_helper - -import ( - "context" - "github.com/getsentry/sentry-go" - "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" - "github.com/sirupsen/logrus" - "strings" - "time" -) - -type Client interface { - CreateLinkToken(ctx context.Context, config plaid.LinkTokenConfigs) (*plaid.CreateLinkTokenResponse, error) - ExchangePublicToken(ctx context.Context, publicToken string) (*plaid.ExchangePublicTokenResponse, error) - GetAccounts(ctx context.Context, accessToken string, options plaid.GetAccountsOptions) ([]plaid.Account, error) - GetAllTransactions(ctx context.Context, accessToken string, start, end time.Time, accountIds []string) ([]plaid.Transaction, error) - GetAllInstitutions(ctx context.Context, countryCodes []string, options plaid.GetInstitutionsOptions) ([]plaid.Institution, error) - GetInstitutions(ctx context.Context, count, offset int, countryCodes []string, options plaid.GetInstitutionsOptions) (total int, _ []plaid.Institution, _ error) - GetInstitution(ctx context.Context, institutionId string, includeMetadata bool, countryCodes []string) (*plaid.Institution, error) - GetWebhookVerificationKey(ctx context.Context, keyId string) (plaid.GetWebhookVerificationKeyResponse, error) - RemoveItem(ctx context.Context, accessToken string) error - Close() error -} - -var ( - _ Client = &plaidClient{} -) - -func NewPlaidClient(log *logrus.Entry, options plaid.ClientOptions) Client { - client, err := plaid.NewClient(options) - if err != nil { - // There currently isn't a code path that actually returns an error from the client. So if something happens - // then its new. - panic(err) - } - - return &plaidClient{ - log: log, - client: client, - institutionTicker: time.NewTicker(2400 * time.Millisecond), // Limit our institution API calls to 25 per minute. - } -} - -type plaidClient struct { - log *logrus.Entry - client *plaid.Client - institutionTicker *time.Ticker -} - -func (p *plaidClient) CreateLinkToken(ctx context.Context, config plaid.LinkTokenConfigs) (*plaid.CreateLinkTokenResponse, error) { - span := sentry.StartSpan(ctx, "Plaid - CreateLinkToken") - defer span.Finish() - - span.Data = map[string]interface{}{} - - result, err := p.client.CreateLinkToken(config) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - err = errors.Wrap(err, "failed to create link token") - } else { - span.Status = sentry.SpanStatusOK - } - - return &result, err -} - -func (p *plaidClient) ExchangePublicToken(ctx context.Context, publicToken string) (*plaid.ExchangePublicTokenResponse, error) { - span := sentry.StartSpan(ctx, "Plaid - ExchangePublicToken") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - - result, err := p.client.ExchangePublicToken(publicToken) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - return nil, errors.Wrap(err, "failed to exchange public token") - } - - span.Status = sentry.SpanStatusOK - - return &result, nil -} - -func (p *plaidClient) GetAccounts(ctx context.Context, accessToken string, options plaid.GetAccountsOptions) ([]plaid.Account, error) { - span := sentry.StartSpan(ctx, "Plaid - GetAccounts") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - span.Data["options"] = options - - result, err := p.client.GetAccountsWithOptions(accessToken, options) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - return nil, errors.Wrap(err, "failed to retrieve plaid accounts") - } - - return result.Accounts, nil -} - -func (p *plaidClient) GetAllTransactions(ctx context.Context, accessToken string, start, end time.Time, accountIds []string) ([]plaid.Transaction, error) { - span := sentry.StartSpan(ctx, "Plaid - GetAllTransactions") - defer span.Finish() - span.Data = map[string]interface{}{ - "start": start, - "end": end, - } - if len(accountIds) > 0 { - span.Data["accountIds"] = accountIds - } - - perPage := 100 - - options := plaid.GetTransactionsOptions{ - StartDate: start.Format("2006-01-02"), - EndDate: end.Format("2006-01-02"), - AccountIDs: accountIds, - Count: perPage, - Offset: 0, - } - - transactions := make([]plaid.Transaction, 0) - for { - options.Offset = len(transactions) - total, items, err := p.GetTransactions(span.Context(), accessToken, options) - if err != nil { - return nil, err - } - - transactions = append(transactions, items...) - - if len(items) < perPage { - break - } - - if len(transactions) >= total { - break - } - } - - return transactions, nil -} - -func (p *plaidClient) GetTransactions(ctx context.Context, accessToken string, options plaid.GetTransactionsOptions) (total int, _ []plaid.Transaction, _ error) { - span := sentry.StartSpan(ctx, "Plaid - GetTransactions") - defer span.Finish() - span.Data = map[string]interface{}{ - "options": options, - } - - result, err := p.client.GetTransactionsWithOptions(accessToken, options) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - return 0, nil, errors.Wrap(err, "failed to retrieve plaid transactions") - } - - return result.TotalTransactions, result.Transactions, nil -} - -func (p *plaidClient) GetInstitution(ctx context.Context, institutionId string, includeMetadata bool, countryCodes []string) (*plaid.Institution, error) { - span := sentry.StartSpan(ctx, "Plaid - GetInstitution") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - - span.Data["institutionId"] = institutionId - span.Data["includeMetadata"] = includeMetadata - span.Data["countryCodes"] = countryCodes - - result, err := p.client.GetInstitutionByIDWithOptions(institutionId, countryCodes, plaid.GetInstitutionByIDOptions{ - IncludeOptionalMetadata: true, - IncludePaymentInitiationMetadata: false, - IncludeStatus: false, - }) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - return nil, errors.Wrap(err, "failed to retrieve plaid institution") - } - - return &result.Institution, nil -} - -func (p *plaidClient) GetAllInstitutions(ctx context.Context, countryCodes []string, options plaid.GetInstitutionsOptions) ([]plaid.Institution, error) { - span := sentry.StartSpan(ctx, "Plaid - GetAllInstitutions") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - span.Data["countryCodes"] = countryCodes - span.Data["options"] = options - - perPage := 500 - institutions := make([]plaid.Institution, 0) - for { - total, items, err := p.GetInstitutions(span.Context(), perPage, len(institutions), countryCodes, options) - if err != nil { - span.Status = sentry.SpanStatusInternalError - return nil, err - } - - institutions = append(institutions, items...) - - // If we received fewer items than we requested, then we have reached the end. - if len(items) < perPage { - break - } - - // If we have received at least what we expect to be the total amount, then we are also done. - if len(institutions) >= total { - break - } - } - - return institutions, nil -} - -func (p *plaidClient) GetInstitutions(ctx context.Context, count, offset int, countryCodes []string, options plaid.GetInstitutionsOptions) (total int, _ []plaid.Institution, _ error) { - span := sentry.StartSpan(ctx, "Plaid - GetInstitutions") - defer span.Finish() - if span.Data == nil { - span.Data = map[string]interface{}{} - } - - span.Data["count"] = count - span.Data["offset"] = offset - span.Data["countryCodes"] = countryCodes - span.Data["options"] = options - - log := p.log.WithFields(logrus.Fields{ - "count": count, - "offset": offset, - "countryCodes": strings.Join(countryCodes, ","), - }) - - log.Debug("retrieving plaid institutions") - - rateLimitTimeout := time.NewTimer(30 * time.Second) - select { - // The institution ticker handles rate limiting for the get institutions endpoint. It makes sure that even - // concurrently, we should not be able to exceed our request limit. At least on a single replica. - case <-p.institutionTicker.C: - result, err := p.client.GetInstitutionsWithOptions(count, offset, countryCodes, options) - span.Data["plaidRequestId"] = result.RequestID - log = log.WithField("plaidRequestId", result.RequestID) - if err != nil { - span.Status = sentry.SpanStatusInternalError - log.WithError(err).Errorf("failed to retrieve plaid institutions") - return 0, nil, errors.Wrap(err, "failed to retrieve plaid institutions") - } - - log.Debugf("successfully retrieved %d institutions", len(result.Institutions)) - - span.Status = sentry.SpanStatusOK - - return result.Total, result.Institutions, nil - case <-rateLimitTimeout.C: - return 0, nil, errors.Errorf("timed out waiting for rate limit") - } -} - -func (p *plaidClient) GetWebhookVerificationKey(ctx context.Context, keyId string) (plaid.GetWebhookVerificationKeyResponse, error) { - span := sentry.StartSpan(ctx, "Plaid - GetWebhookVerificationKey") - defer span.Finish() - span.Data = map[string]interface{}{} - - result, err := p.client.GetWebhookVerificationKey(keyId) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - } else { - span.Status = sentry.SpanStatusOK - } - - return result, errors.Wrap(err, "failed to retrieve webhook verification key") -} - -func (p *plaidClient) RemoveItem(ctx context.Context, accessToken string) error { - span := sentry.StartSpan(ctx, "Plaid - RemoveItem") - defer span.Finish() - span.Data = map[string]interface{}{} - - result, err := p.client.RemoveItem(accessToken) - span.Data["plaidRequestId"] = result.RequestID - if err != nil { - span.Status = sentry.SpanStatusInternalError - } else { - span.Status = sentry.SpanStatusOK - } - - return errors.Wrap(err, "failed to remove Plaid item") -} - -func (p *plaidClient) Close() error { - p.client = nil - p.institutionTicker.Stop() - return nil -} diff --git a/pkg/internal/plaid_helper/client_test.go b/pkg/internal/plaid_helper/client_test.go deleted file mode 100644 index 89b0c03d..00000000 --- a/pkg/internal/plaid_helper/client_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package plaid_helper - -import ( - "context" - "github.com/brianvoe/gofakeit/v6" - "github.com/jarcoal/httpmock" - "github.com/monetr/rest-api/pkg/internal/mock_plaid" - "github.com/monetr/rest-api/pkg/internal/testutils" - "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" - "github.com/stretchr/testify/assert" - "net/http" - "testing" -) - -func TestPlaidClient_GetAccounts(t *testing.T) { - t.Run("simple", func(t *testing.T) { - httpmock.Activate() - defer httpmock.Deactivate() - - account := plaid.Account{ - AccountID: gofakeit.UUID(), - Balances: plaid.AccountBalances{ - Available: 10.00, - Current: 10.00, - Limit: 10.00, - ISOCurrencyCode: "USD", - UnofficialCurrencyCode: "USD", - }, - Mask: "1234", - Name: "Checking account", - OfficialName: "Super duper checking", - Subtype: "checking", - Type: "depository", - VerificationStatus: "verified?", - } - - mock_plaid.MockGetAccounts(t, []plaid.Account{ - account, - }) - - client := NewPlaidClient(testutils.GetLog(t), plaid.ClientOptions{ - ClientID: gofakeit.UUID(), - Secret: gofakeit.UUID(), - Environment: plaid.Sandbox, - HTTPClient: &http.Client{ - Transport: httpmock.DefaultTransport, - }, - }) - - result, err := client.GetAccounts(context.Background(), gofakeit.UUID(), plaid.GetAccountsOptions{ - AccountIDs: []string{ - account.AccountID, - }, - }) - assert.NoError(t, err, "should succeed") - assert.NotEmpty(t, result) - }) - - t.Run("error NO_ACCOUNTS", func(t *testing.T) { - httpmock.Activate() - defer httpmock.Deactivate() - - mock_plaid.MockGetAccountsError(t, plaid.Error{ - APIResponse: plaid.APIResponse{ - RequestID: gofakeit.UUID(), - }, - ErrorType: "ITEM_ERROR", - ErrorCode: "NO_ACCOUNTS", - ErrorMessage: "no valid accounts were found for this item", - DisplayMessage: "No valid accounts were found at the financial institution. Please visit your financial institution's website to confirm accounts are available.", - StatusCode: http.StatusBadRequest, - }) - - client := NewPlaidClient(testutils.GetLog(t), plaid.ClientOptions{ - ClientID: gofakeit.UUID(), - Secret: gofakeit.UUID(), - Environment: plaid.Sandbox, - HTTPClient: &http.Client{ - Transport: httpmock.DefaultTransport, - }, - }) - - result, err := client.GetAccounts(context.Background(), gofakeit.UUID(), plaid.GetAccountsOptions{ - AccountIDs: []string{ - gofakeit.UUID(), - }, - }) - assert.Error(t, err, "should fail with NO_ACCOUNTS error") - assert.Empty(t, result) - - cause := errors.Cause(err) - assert.IsType(t, plaid.Error{}, cause, "should be a plaid error") - }) -} diff --git a/pkg/internal/plaid_helper/verification.go b/pkg/internal/plaid_helper/verification.go deleted file mode 100644 index c2dcf166..00000000 --- a/pkg/internal/plaid_helper/verification.go +++ /dev/null @@ -1,125 +0,0 @@ -package plaid_helper - -import ( - "context" - "encoding/json" - "github.com/MicahParks/keyfunc" - "github.com/getsentry/sentry-go" - "github.com/gomodule/redigo/redis" - "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" - "github.com/sirupsen/logrus" - "sync" -) - -type WebhookVerification interface { - GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) - Close() error -} - -var ( - _ WebhookVerification = &redisWebhookVerification{} - _ WebhookVerification = &memoryWebhookVerification{} -) - -func NewMemoryWebhookVerificationCache(log *logrus.Entry, client Client) WebhookVerification { - return &memoryWebhookVerification{ - log: log, - plaidClient: client, - lock: sync.Mutex{}, - cache: map[string]*keyfunc.JWKS{}, - } -} - -type memoryWebhookVerification struct { - log *logrus.Entry - plaidClient Client - lock sync.Mutex - cache map[string]*keyfunc.JWKS -} - -func (m *memoryWebhookVerification) GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) { - span := sentry.StartSpan(ctx, "Cache - GetVerificationKey") - defer span.Finish() - - log := m.log.WithField("keyId", keyId).WithContext(ctx) - - m.lock.Lock() - defer m.lock.Unlock() - - jwksFunc, ok := m.cache[keyId] - if ok { - log.Trace("jwk function already present in cache, returning") - return jwksFunc, nil - } - - log.Trace("jwk function missing in cache, retrieving from plaid") - - verificationResponse, err := m.plaidClient.GetWebhookVerificationKey(span.Context(), keyId) - if err != nil { - return nil, errors.Wrap(err, "failed to retrieve public verification key") - } - - var keys = struct { - Keys []plaid.WebhookVerificationKey `json:"keys"` - }{ - Keys: []plaid.WebhookVerificationKey{ - verificationResponse.Key, - }, - } - - encodedKeys, err := json.Marshal(keys) - if err != nil { - return nil, errors.Wrap(err, "failed to convert plaid verification key to json") - } - - var jwksJSON json.RawMessage = encodedKeys - - jwksFunc, err = keyfunc.New(jwksJSON) - if err != nil { - return nil, errors.Wrap(err, "failed to create key function") - } - - m.cache[keyId] = jwksFunc - - return jwksFunc, nil -} - -func (m *memoryWebhookVerification) Close() error { - return m.plaidClient.Close() -} - -type redisWebhookVerification struct { - log *logrus.Entry - plaidClient Client - redisClient redis.Conn -} - -// TODO Finish building out caching of public verification keys. -func (r *redisWebhookVerification) GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) { - result, err := redis.String(r.redisClient.Do("GET", keyId)) - if err != nil { - return nil, err - } - - var jwksJSON json.RawMessage = []byte(result) - jwkKeyFunc, err := keyfunc.New(jwksJSON) - if err != nil { - return nil, errors.Wrap(err, "failed to create key function") - } - - return jwkKeyFunc, nil -} - -func (r *redisWebhookVerification) Close() error { - if err := r.plaidClient.Close(); err != nil { - r.log.WithError(err).Errorf("failed to close plaid client gracefully") - return errors.Wrap(err, "failed to close plaid client") - } - if err := r.redisClient.Close(); err != nil { - r.log.WithError(err).Errorf("failed to close redis client gracefully") - return errors.Wrap(err, "failed to close redis client") - } - - return nil -} diff --git a/pkg/internal/platypus/account.go b/pkg/internal/platypus/account.go new file mode 100644 index 00000000..f7e62748 --- /dev/null +++ b/pkg/internal/platypus/account.go @@ -0,0 +1,139 @@ +package platypus + +import "github.com/plaid/plaid-go/plaid" + +type ( + BankAccount interface { + // GetAccountId will return Plaid's unique identifier for the bank account. + GetAccountId() string + // GetBalances will return the bank account's balances. + GetBalances() BankAccountBalances + // GetMask typically returns the last 4 of the bank account number. This could technically return anything that + // represents a small portion of the bank account's identification. I don't currently know enough about this to know + // what other values this might have. + GetMask() string + // GetName will return the name of the account specified by the user or the financial institution itself. + GetName() string + // GetOfficialName will return the name of the account specified by the financial institution itself. + GetOfficialName() string + // GetType will return the plaid type of the account. For our use case this is typically "depository". + GetType() string + // GetSubType will return the sub-type of the account. This can be something like "checking" or "savings". + GetSubType() string + } + + BankAccountBalances interface { + // GetAvailable returns the total amount available for the bank account in cents. + GetAvailable() int64 + // GetCurrent returns the current bank account balance in cents. This is typically the total account value excluding + // pending transactions. + GetCurrent() int64 + // GetLimit returns the limit of the account (this applies for credit accounts) in cents. + GetLimit() int64 + GetIsoCurrencyCode() string + GetUnofficialCurrencyCode() string + } +) + +var ( + _ BankAccountBalances = PlaidBankAccountBalances{} +) + +func NewPlaidBankAccountBalances(balances plaid.AccountBalance) (PlaidBankAccountBalances, error) { + return PlaidBankAccountBalances{ + // We work with all amounts in cents. So we need to convert all balances to cents in order to make them whole + // integers rather than floats. + Available: int64(balances.GetAvailable() * 100), + Current: int64(balances.GetCurrent() * 100), + Limit: int64(balances.GetLimit() * 100), + IsoCurrencyCode: balances.GetIsoCurrencyCode(), + UnofficialCurrencyCode: balances.GetUnofficialCurrencyCode(), + }, nil +} + +type PlaidBankAccountBalances struct { + Available int64 + Current int64 + Limit int64 + IsoCurrencyCode string + UnofficialCurrencyCode string +} + +func (p PlaidBankAccountBalances) GetAvailable() int64 { + return p.Available +} + +func (p PlaidBankAccountBalances) GetCurrent() int64 { + return p.Current +} + +func (p PlaidBankAccountBalances) GetLimit() int64 { + return p.Limit +} + +func (p PlaidBankAccountBalances) GetIsoCurrencyCode() string { + return p.IsoCurrencyCode +} + +func (p PlaidBankAccountBalances) GetUnofficialCurrencyCode() string { + return p.UnofficialCurrencyCode +} + +var ( + _ BankAccount = PlaidBankAccount{} +) + +func NewPlaidBankAccount(bankAccount plaid.AccountBase) (PlaidBankAccount, error) { + balances, err := NewPlaidBankAccountBalances(bankAccount.GetBalances()) + if err != nil { + return PlaidBankAccount{}, err + } + + return PlaidBankAccount{ + AccountId: bankAccount.GetAccountId(), + Balances: balances, + Mask: bankAccount.GetMask(), + Name: bankAccount.GetName(), + OfficialName: bankAccount.GetOfficialName(), + Type: string(bankAccount.GetType()), + SubType: string(bankAccount.GetSubtype()), + }, nil +} + +type PlaidBankAccount struct { + AccountId string + Balances PlaidBankAccountBalances + Mask string + Name string + OfficialName string + Type string + SubType string +} + +func (p PlaidBankAccount) GetAccountId() string { + return p.AccountId +} + +func (p PlaidBankAccount) GetBalances() BankAccountBalances { + return p.Balances +} + +func (p PlaidBankAccount) GetMask() string { + return p.Mask +} + +func (p PlaidBankAccount) GetName() string { + return p.Name +} + +func (p PlaidBankAccount) GetOfficialName() string { + return p.OfficialName +} + +func (p PlaidBankAccount) GetType() string { + return p.Type +} + +func (p PlaidBankAccount) GetSubType() string { + return p.SubType +} diff --git a/pkg/internal/platypus/account_test.go b/pkg/internal/platypus/account_test.go new file mode 100644 index 00000000..bf95d5c3 --- /dev/null +++ b/pkg/internal/platypus/account_test.go @@ -0,0 +1,83 @@ +package platypus + +import ( + "testing" + + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/assert" +) + +func TestNewPlaidBankAccountBalances(t *testing.T) { + t.Run("simple", func(t *testing.T) { + var available, current, limit float32 + available = 101.34 + current = 110.25 + limit = 500.13 + + plaidBalances := plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(&available), + Current: *plaid.NewNullableFloat32(¤t), + Limit: *plaid.NewNullableFloat32(&limit), + IsoCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), + UnofficialCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), + LastUpdatedDatetime: plaid.NullableTime{}, // Leave this be so that is has no value. + AdditionalProperties: map[string]interface{}{}, + } + + balances, err := NewPlaidBankAccountBalances(plaidBalances) + assert.NoError(t, err, "must be able to convert balances") + assert.NotEmpty(t, balances, "balances must not be empty") + assert.EqualValues(t, available*100, balances.GetAvailable(), "available should be converted to cents") + assert.EqualValues(t, current*100, balances.GetCurrent(), "current should be converted to cents") + assert.EqualValues(t, limit*100, balances.GetLimit(), "limit should be converted to cents") + assert.EqualValues(t, "USD", balances.GetIsoCurrencyCode(), "ISO currency code should match USD") + assert.EqualValues(t, "USD", balances.GetUnofficialCurrencyCode(), "unofficial currency code should match USD") + }) + + t.Run("missing value", func(t *testing.T) { + plaidBalances := plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(nil), + Current: *plaid.NewNullableFloat32(nil), + Limit: *plaid.NewNullableFloat32(nil), + IsoCurrencyCode: *plaid.NewNullableString(nil), + UnofficialCurrencyCode: *plaid.NewNullableString(nil), + } + + balances, err := NewPlaidBankAccountBalances(plaidBalances) + assert.NoError(t, err, "must be able to convert balances") + assert.EqualValues(t, 0, balances.GetAvailable(), "available should be 0 when no value is present") + assert.EqualValues(t, 0, balances.GetCurrent(), "current should be 0 when no value is present") + assert.EqualValues(t, 0, balances.GetLimit(), "limit should be 0 when no value is present") + assert.EqualValues(t, "", balances.GetIsoCurrencyCode(), "ISO currency code should be empty if no value is present") + assert.EqualValues(t, "", balances.GetUnofficialCurrencyCode(), "unofficial currency code should be empty if no value is present") + }) +} + +func TestNewPlaidBankAccount(t *testing.T) { + t.Run("simple", func(t *testing.T) { + subType := plaid.ACCOUNTSUBTYPE_CHECKING + plaidBank := plaid.AccountBase{ + AccountId: gofakeit.UUID(), + Balances: plaid.AccountBalance{}, + Mask: *plaid.NewNullableString(myownsanity.StringP("1234")), + Name: "Checking Account", + OfficialName: *plaid.NewNullableString(myownsanity.StringP("CHECKING - 1234")), + Type: plaid.ACCOUNTTYPE_DEPOSITORY, + Subtype: *plaid.NewNullableAccountSubtype(&subType), + } + + bank, err := NewPlaidBankAccount(plaidBank) + assert.NoError(t, err, "must be able to convert bank account") + assert.NotEmpty(t, bank, "bank account must not be empty") + assert.EqualValues(t, plaidBank.GetAccountId(), bank.GetAccountId(), "account Id must match") + assert.EqualValues(t, "1234", bank.GetMask(), "mask must match") + assert.EqualValues(t, "Checking Account", bank.GetName(), "name must match") + assert.EqualValues(t, "CHECKING - 1234", bank.GetOfficialName(), "official name must match") + assert.EqualValues(t, "depository", bank.GetType(), "account type must match") + assert.EqualValues(t, "checking", bank.GetSubType(), "account sub-type must match") + + assert.IsType(t, PlaidBankAccountBalances{}, bank.GetBalances(), "must return plaid bank account balances") + }) +} diff --git a/pkg/internal/platypus/client.go b/pkg/internal/platypus/client.go new file mode 100644 index 00000000..fc30dcd1 --- /dev/null +++ b/pkg/internal/platypus/client.go @@ -0,0 +1,266 @@ +package platypus + +import ( + "context" + "fmt" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/crumbs" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "strconv" + "time" + + "github.com/getsentry/sentry-go" + "github.com/plaid/plaid-go/plaid" + "github.com/sirupsen/logrus" +) + +type ( + Client interface { + GetAccounts(ctx context.Context, accountIds ...string) ([]BankAccount, error) + GetAllTransactions(ctx context.Context, start, end time.Time, accountIds []string) ([]Transaction, error) + UpdateItem(ctx context.Context) (LinkToken, error) + RemoveItem(ctx context.Context) error + } +) + +var ( + _ Client = &PlaidClient{} +) + +type PlaidClient struct { + accountId uint64 + linkId uint64 + accessToken string + log *logrus.Entry + client *plaid.APIClient + config config.Plaid +} + +func (p *PlaidClient) getLog(span *sentry.Span) *logrus.Entry { + return p.log.WithContext(span.Context()).WithField("plaid", span.Op) +} + +func (p *PlaidClient) GetAccounts(ctx context.Context, accountIds ...string) ([]BankAccount, error) { + span := sentry.StartSpan(ctx, "Plaid - GetAccount") + defer span.Finish() + + log := p.getLog(span) + + // By default report the accountIds as "all accounts" to sentry. This way we know that if we are not requesting + // specific accounts then we are requesting all of them. + span.Data = map[string]interface{}{ + "accountIds": "ALL_BANK_ACCOUNTS", + } + + // If however we are requesting specific accounts, overwrite the value. + if len(accountIds) > 0 { + span.Data["accountIds"] = accountIds + } + + log.Trace("retrieving bank accounts from plaid") + + // Build the get accounts request. + request := p.client.PlaidApi. + AccountsGet(span.Context()). + AccountsGetRequest(plaid.AccountsGetRequest{ + AccessToken: p.accessToken, + Options: &plaid.AccountsGetRequestOptions{ + // This might not work, if it does not we should just add a nil check somehow here. + AccountIds: &accountIds, + }, + }) + + // Send the request. + result, response, err := request.Execute() + // And handle the response. + if err = after( + span, + response, + err, + "Retrieving bank accounts from Plaid", + "failed to retrieve bank accounts from plaid", + ); err != nil { + log.WithError(err).Errorf("failed to retrieve bank accounts from plaid") + return nil, err + } + + plaidAccounts := result.GetAccounts() + accounts := make([]BankAccount, len(plaidAccounts)) + + // Once we have our data, convert all of the results from our request to our own bank account interface. + for i, plaidAccount := range plaidAccounts { + accounts[i], err = NewPlaidBankAccount(plaidAccount) + if err != nil { + log.WithError(err). + WithField("bankAccountId", plaidAccount.GetAccountId()). + Errorf("failed to convert bank account") + crumbs.Error(span.Context(), "failed to convert bank account", "debug", map[string]interface{}{ + // Maybe we don't want to report the entire account object here, but it'll sure save us a ton of time + // if there is ever a problem with actually converting the account. This way we can actually see the + // account object that caused the problem -> when it caused the problem. + "bankAccount": plaidAccount, + }) + return nil, err + } + } + + return accounts, nil +} + +func (p *PlaidClient) GetAllTransactions(ctx context.Context, start, end time.Time, accountIds []string) ([]Transaction, error) { + span := sentry.StartSpan(ctx, "Plaid - GetAllTransactions") + defer span.Finish() + + transactions := make([]Transaction, 0) + + var perPage int32 = 500 + var offset int32 = 0 + for { + someTransactions, err := p.GetTransactions(span.Context(), start, end, perPage, offset, accountIds) + if err != nil { + return nil, err + } + + transactions = append(transactions, someTransactions...) + if retrieved := int32(len(someTransactions)); retrieved == perPage { + offset += retrieved + continue + } + + break + } + + return transactions, nil +} + +func (p *PlaidClient) GetTransactions(ctx context.Context, start, end time.Time, count, offset int32, bankAccountIds []string) ([]Transaction, error) { + span := sentry.StartSpan(ctx, "Plaid - GetTransactions") + defer span.Finish() + + log := p.getLog(span) + + log.Trace("retrieving transactions") + + request := p.client.PlaidApi. + TransactionsGet(span.Context()). + TransactionsGetRequest(plaid.TransactionsGetRequest{ + Options: &plaid.TransactionsGetRequestOptions{ + AccountIds: &bankAccountIds, + Count: &count, + Offset: &offset, + IncludeOriginalDescription: plaid.NullableBool{}, + }, + AccessToken: p.accessToken, + Secret: nil, + StartDate: start.Format("2006-01-02"), + EndDate: end.Format("2006-01-02"), + }) + + // Send the request. + result, response, err := request.Execute() + // And handle the response. + if err = after( + span, + response, + err, + "Retrieving transactions from Plaid", + "failed to retrieve transactions from plaid", + ); err != nil { + log.WithError(err).Errorf("failed to retrieve transactions from plaid") + return nil, err + } + + transactions := make([]Transaction, len(result.Transactions)) + for i, transaction := range result.Transactions { + transactions[i], err = NewTransactionFromPlaid(transaction) + if err != nil { + return nil, err + } + } + + return transactions, nil +} + +func (p *PlaidClient) UpdateItem(ctx context.Context) (LinkToken, error) { + span := sentry.StartSpan(ctx, "Plaid - UpdateItem") + defer span.Finish() + + log := p.getLog(span) + + log.Trace("creating link token for update") + + redirectUri := fmt.Sprintf("https://%s/plaid/oauth-return", p.config.OAuthDomain) + + var webhooksUrl *string + if p.config.WebhooksEnabled { + if p.config.WebhooksDomain == "" { + crumbs.Warn(span.Context(), "BUG: Plaid webhook domain is not present but webhooks are enabled.", "bug", nil) + } else { + webhooksUrl = myownsanity.StringP(fmt.Sprintf("https://%s/plaid/webhook", p.config.WebhooksDomain)) + } + } + + request := p.client.PlaidApi. + LinkTokenCreate(span.Context()). + LinkTokenCreateRequest(plaid.LinkTokenCreateRequest{ + ClientName: "monetr", + Language: PlaidLanguage, + CountryCodes: PlaidCountries, + User: plaid.LinkTokenCreateRequestUser{ + ClientUserId: strconv.FormatUint(p.accountId, 10), + }, + Webhook: webhooksUrl, + AccessToken: &p.accessToken, + LinkCustomizationName: nil, + RedirectUri: &redirectUri, + }) + + result, response, err := request.Execute() + if err = after( + span, + response, + err, + "Updating Plaid link token", + "failed to update Plaid link token", + ); err != nil { + log.WithError(err).Errorf("failed to create link token") + return nil, err + } + + return PlaidLinkToken{ + LinkToken: result.LinkToken, + Expires: result.Expiration, + }, nil +} + +func (p *PlaidClient) RemoveItem(ctx context.Context) error { + span := sentry.StartSpan(ctx, "Plaid - RemoveItem") + defer span.Finish() + + log := p.getLog(span) + + log.Trace("removing item") + + // Build the get accounts request. + request := p.client.PlaidApi. + ItemRemove(span.Context()). + ItemRemoveRequest(plaid.ItemRemoveRequest{ + AccessToken: p.accessToken, + }) + + // Send the request. + _, response, err := request.Execute() + // And handle the response. + if err = after( + span, + response, + err, + "Removing Plaid item", + "failed to remove Plaid item", + ); err != nil { + log.WithError(err).Errorf("failed to remove Plaid item") + return err + } + + return nil +} diff --git a/pkg/internal/platypus/client_test.go b/pkg/internal/platypus/client_test.go new file mode 100644 index 00000000..af4247e3 --- /dev/null +++ b/pkg/internal/platypus/client_test.go @@ -0,0 +1,130 @@ +package platypus + +import ( + "context" + "github.com/brianvoe/gofakeit/v6" + "github.com/jarcoal/httpmock" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/internal/mock_plaid" + "github.com/monetr/rest-api/pkg/internal/testutils" + "github.com/monetr/rest-api/pkg/models" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestPlaidClient_GetAccount(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + log := testutils.GetLog(t) + accountId := testutils.GetAccountIdForTest(t) + + accessToken := gofakeit.UUID() + + account := mock_plaid.BankAccountFixture(t) + + mock_plaid.MockGetAccounts(t, []plaid.AccountBase{ + account, + }) + + client := NewPlaid(log, nil, nil, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + }) + + link := &models.Link{ + LinkId: 1234, + AccountId: accountId, + } + + platypus, err := client.NewClient(context.Background(), link, accessToken) + assert.NoError(t, err, "should create platypus") + assert.NotNil(t, platypus, "should not be nil") + + accounts, err := platypus.GetAccounts(context.Background(), account.GetAccountId()) + assert.NoError(t, err, "should not return an error retrieving accounts") + assert.NotEmpty(t, accounts, "should return some accounts") + }) +} + +func TestPlaidClient_GetAllTransactions(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + log := testutils.GetLog(t) + accountId := testutils.GetAccountIdForTest(t) + + accessToken := gofakeit.UUID() + + account := mock_plaid.BankAccountFixture(t) + + end := time.Now() + start := end.Add(-7 * 24 * time.Hour) + mock_plaid.MockGetRandomTransactions(t, start, end, 5000, []string{ + account.GetAccountId(), + }) + + platypus := NewPlaid(log, nil, nil, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + }) + + link := &models.Link{ + LinkId: 1234, + AccountId: accountId, + } + + client, err := platypus.NewClient(context.Background(), link, accessToken) + assert.NoError(t, err, "should create platypus") + assert.NotNil(t, client, "should not be nil") + + transactions, err := client.GetAllTransactions(context.Background(), start, end, []string{ + account.GetAccountId(), + }) + assert.NoError(t, err, "should not return an error") + assert.NotEmpty(t, transactions, "should return a few transactions") + assert.Equal(t, map[string]int{ + "POST https://sandbox.plaid.com/transactions/get": 11, + }, httpmock.GetCallCountInfo(), "API calls should match") + }) +} + +func TestPlaidClient_UpdateItem(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + log := testutils.GetLog(t) + accountId := testutils.GetAccountIdForTest(t) + + accessToken := gofakeit.UUID() + + mock_plaid.MockCreateLinkToken(t) + + platypus := NewPlaid(log, nil, nil, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + OAuthDomain: "localhost", + }) + + link := &models.Link{ + LinkId: 1234, + AccountId: accountId, + } + + client, err := platypus.NewClient(context.Background(), link, accessToken) + assert.NoError(t, err, "should create client") + assert.NotNil(t, client, "should not be nil") + + linkToken, err := client.UpdateItem(context.Background()) + assert.NoError(t, err, "should not return an error creating an update link token") + assert.NotEmpty(t, linkToken.Token(), "must not be empty") + }) +} diff --git a/pkg/internal/platypus/platypus.go b/pkg/internal/platypus/platypus.go new file mode 100644 index 00000000..e61ee953 --- /dev/null +++ b/pkg/internal/platypus/platypus.go @@ -0,0 +1,328 @@ +package platypus + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/getsentry/sentry-go" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/crumbs" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/monetr/rest-api/pkg/models" + "github.com/monetr/rest-api/pkg/repository" + "github.com/monetr/rest-api/pkg/secrets" + "github.com/pkg/errors" + "github.com/plaid/plaid-go/plaid" + "github.com/sirupsen/logrus" +) + +var ( + PlaidLanguage = "en" + PlaidCountries = []plaid.CountryCode{ + plaid.COUNTRYCODE_US, + } + PlaidProducts = []plaid.Products{ + plaid.PRODUCTS_TRANSACTIONS, + } +) + +type ( + Platypus interface { + CreateLinkToken(ctx context.Context, options LinkTokenOptions) (LinkToken, error) + ExchangePublicToken(ctx context.Context, publicToken string) (*ItemToken, error) + GetWebhookVerificationKey(ctx context.Context, keyId string) (*WebhookVerificationKey, error) + NewClientFromItemId(ctx context.Context, itemId string) (Client, error) + NewClientFromLink(ctx context.Context, accountId uint64, linkId uint64) (Client, error) + NewClient(ctx context.Context, link *models.Link, accessToken string) (Client, error) + Close() error + } +) + +// after is a wrapper around some of the basic operations we would want to perform after each request. Mainly that we +// want to keep track of things like the request Id and some information about the request itself. It also handles error +// wrapping. +func after(span *sentry.Span, response *http.Response, err error, message, errorMessage string) error { + if response != nil { + requestId := response.Header.Get("X-Request-Id") + if span.Data == nil { + span.Data = map[string]interface{}{} + } + + data := map[string]interface{}{} + + // With plaid responses we can actually still use the body of the response :tada:. The request Id is also stored on + // the response body itself in most of my testing. I could have sworn the documentation cited X-Request-Id as being + // a possible source for it, but I have not seen that yet. This bit of code extracts the body into a map. I know to + // some degree of certainty that the response will always be an object and not an array. So a map with a string key + // is safe. I can then extract the request Id and store that with my logging and diagnostic data. + { + var extractedResponseBody map[string]interface{} + if e := json.NewDecoder(response.Body).Decode(&extractedResponseBody); e == nil { + if requestId == "" { + requestId = extractedResponseBody["request_id"].(string) + } + + // But if our request was not successful, then I also want to yoink that body and throw it into my diagnostic + // data as well. This will help me if I ever need to track down bugs with Plaid's API or problems with requests + // that I am making incorrectly. + if response.StatusCode != http.StatusOK { + data["body"] = extractedResponseBody + } + } + } + + { // Make sure we put the request ID everywhere, this is easily the most important diagnostic data we need. + data["X-RequestId"] = requestId + span.Data["plaidRequestId"] = requestId + span.SetTag("plaidRequestId", requestId) + } + + crumbs.HTTP( + span.Context(), + message, + "plaid", + response.Request.URL.String(), + response.Request.Method, + response.StatusCode, + data, + ) + } + + // Properly set the span status for this request. + if err != nil { + span.Status = sentry.SpanStatusInternalError + } else { + span.Status = sentry.SpanStatusOK + } + + return errors.Wrap(err, errorMessage) +} + +var ( + _ Platypus = &Plaid{} +) + +func NewPlaid(log *logrus.Entry, secret secrets.PlaidSecretsProvider, repo repository.PlaidRepository, options config.Plaid) *Plaid { + httpClient := &http.Client{ + Timeout: 60 * time.Second, + } + + conf := plaid.NewConfiguration() + conf.HTTPClient = httpClient + conf.UseEnvironment(options.Environment) + conf.AddDefaultHeader("PLAID-CLIENT-ID", options.ClientID) + conf.AddDefaultHeader("PLAID-SECRET", options.ClientSecret) + + client := plaid.NewAPIClient(conf) + + return &Plaid{ + client: client, + log: log, + secret: secret, + repo: repo, + config: options, + } +} + +type Plaid struct { + client *plaid.APIClient + log *logrus.Entry + secret secrets.PlaidSecretsProvider + repo repository.PlaidRepository + config config.Plaid +} + +func (p *Plaid) CreateLinkToken(ctx context.Context, options LinkTokenOptions) (LinkToken, error) { + span := sentry.StartSpan(ctx, "Plaid - CreateLinkToken") + defer span.Finish() + + log := p.log + + redirectUri := fmt.Sprintf("https://%s/plaid/oauth-return", p.config.OAuthDomain) + + var webhooksUrl *string + if p.config.WebhooksEnabled { + if p.config.WebhooksDomain == "" { + crumbs.Warn(span.Context(), "BUG: Plaid webhook domain is not present but webhooks are enabled.", "bug", nil) + } else { + webhooksUrl = myownsanity.StringP(fmt.Sprintf("https://%s/plaid/webhook", p.config.WebhooksDomain)) + } + } + + request := p.client.PlaidApi. + LinkTokenCreate(span.Context()). + LinkTokenCreateRequest(plaid.LinkTokenCreateRequest{ + ClientName: "monetr", + Language: PlaidLanguage, + CountryCodes: PlaidCountries, + User: plaid.LinkTokenCreateRequestUser{ + ClientUserId: options.ClientUserID, + LegalName: &options.LegalName, + PhoneNumber: options.PhoneNumber, + PhoneNumberVerifiedTime: options.PhoneNumberVerifiedTime, + EmailAddress: &options.EmailAddress, + EmailAddressVerifiedTime: options.EmailAddressVerifiedTime, + Ssn: nil, + DateOfBirth: nil, + }, + Products: &PlaidProducts, + Webhook: webhooksUrl, + AccessToken: nil, + LinkCustomizationName: nil, + RedirectUri: &redirectUri, + AndroidPackageName: nil, + AccountFilters: nil, + EuConfig: nil, + InstitutionId: nil, + PaymentInitiation: nil, + DepositSwitch: nil, + IncomeVerification: nil, + Auth: nil, + }) + + result, response, err := request.Execute() + if err = after( + span, + response, + err, + "Creating link token with Plaid", + "failed to create link token", + ); err != nil { + log.WithError(err).Errorf("failed to create link token") + return nil, err + } + + return PlaidLinkToken{ + LinkToken: result.LinkToken, + Expires: result.Expiration, + }, nil +} + +func (p *Plaid) ExchangePublicToken(ctx context.Context, publicToken string) (*ItemToken, error) { + span := sentry.StartSpan(ctx, "Plaid - ExchangePublicToken") + defer span.Finish() + + log := p.log + + request := p.client.PlaidApi. + ItemPublicTokenExchange(span.Context()). + ItemPublicTokenExchangeRequest(plaid.ItemPublicTokenExchangeRequest{ + PublicToken: publicToken, + }) + + result, response, err := request.Execute() + if err = after( + span, + response, + err, + "Exchanging public token with Plaid", + "failed to exchange public token with Plaid", + ); err != nil { + log.WithError(err).Errorf("failed to exchange public token with Plaid") + return nil, err + } + + token, err := NewItemTokenFromPlaid(result) + if err != nil { + return nil, err + } + + return &token, nil +} + +func (p *Plaid) GetWebhookVerificationKey(ctx context.Context, keyId string) (*WebhookVerificationKey, error) { + span := sentry.StartSpan(ctx, "Plaid - GetWebhookVerificationKey") + defer span.Finish() + + log := p.log + + request := p.client.PlaidApi. + WebhookVerificationKeyGet(span.Context()). + WebhookVerificationKeyGetRequest(plaid.WebhookVerificationKeyGetRequest{ + KeyId: keyId, + }) + + result, response, err := request.Execute() + if err = after( + span, + response, + err, + "Retrieving webhook verification key", + "failed to retrieve webhook verification key from Plaid", + ); err != nil { + log.WithError(err).Errorf("failed to retrieve webhook verification key from Plaid") + return nil, err + } + + webhook, err := NewWebhookVerificationKeyFromPlaid(result.Key) + if err != nil { + return nil, err + } + + return &webhook, nil +} + +func (p *Plaid) NewClientFromItemId(ctx context.Context, itemId string) (Client, error) { + span := sentry.StartSpan(ctx, "Plaid - NewClientFromItemId") + defer span.Finish() + + link, err := p.repo.GetLinkByItemId(span.Context(), itemId) + if err != nil { + return nil, errors.Wrap(err, "cannot create client without link") + } + + return p.newClient(span.Context(), link) +} + +func (p *Plaid) NewClientFromLink(ctx context.Context, accountId uint64, linkId uint64) (Client, error) { + span := sentry.StartSpan(ctx, "Plaid - NewClientFromLink") + defer span.Finish() + + link, err := p.repo.GetLink(span.Context(), accountId, linkId) + if err != nil { + return nil, errors.Wrap(err, "cannot create Plaid client from link") + } + + return p.newClient(span.Context(), link) +} + +func (p *Plaid) NewClient(ctx context.Context, link *models.Link, accessToken string) (Client, error) { + return &PlaidClient{ + accountId: link.AccountId, + linkId: link.LinkId, + accessToken: accessToken, + log: p.log.WithFields(logrus.Fields{ + "accountId": link.AccountId, + "linkId": link.LinkId, + }), + client: p.client, + config: p.config, + }, nil +} + +func (p *Plaid) newClient(ctx context.Context, link *models.Link) (Client, error) { + span := sentry.StartSpan(ctx, "Plaid - newClient") + defer span.Finish() + + if link == nil { + return nil, errors.New("cannot create client without link") + } + + if link.PlaidLink == nil { + return nil, errors.New("cannot create client without link") + } + + accessToken, err := p.secret.GetAccessTokenForPlaidLinkId(span.Context(), link.AccountId, link.PlaidLink.ItemId) + if err != nil { + return nil, err + } + + return p.NewClient(span.Context(), link, accessToken) +} + +func (p *Plaid) Close() error { + panic("implement me") +} diff --git a/pkg/internal/platypus/platypus_test.go b/pkg/internal/platypus/platypus_test.go new file mode 100644 index 00000000..8bcb00c6 --- /dev/null +++ b/pkg/internal/platypus/platypus_test.go @@ -0,0 +1,43 @@ +package platypus + +import ( + "context" + "github.com/brianvoe/gofakeit/v6" + "github.com/jarcoal/httpmock" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/internal/mock_plaid" + "github.com/monetr/rest-api/pkg/internal/testutils" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestPlaid_CreateLinkToken(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + log := testutils.GetLog(t) + mock_plaid.MockCreateLinkToken(t) + + platypus := NewPlaid(log, nil, nil, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + OAuthDomain: "localhost", + }) + + linkToken, err := platypus.CreateLinkToken(context.Background(), LinkTokenOptions{ + ClientUserID: "1234", + LegalName: gofakeit.Name(), + PhoneNumber: nil, + PhoneNumberVerifiedTime: nil, + EmailAddress: gofakeit.Email(), + EmailAddressVerifiedTime: nil, + RedirectURI: "", + UpdateMode: false, + }) + assert.NoError(t, err, "should not return an error creating a link token") + assert.NotEmpty(t, linkToken.Token(), "must not be empty") + }) +} diff --git a/pkg/internal/platypus/token.go b/pkg/internal/platypus/token.go new file mode 100644 index 00000000..fddda585 --- /dev/null +++ b/pkg/internal/platypus/token.go @@ -0,0 +1,52 @@ +package platypus + +import ( + "github.com/plaid/plaid-go/plaid" + "time" +) + +type ItemToken struct { + AccessToken string + ItemId string +} + +func NewItemTokenFromPlaid(input plaid.ItemPublicTokenExchangeResponse) (ItemToken, error) { + return ItemToken{ + AccessToken: input.GetAccessToken(), + ItemId: input.GetItemId(), + }, nil +} + +type LinkTokenOptions struct { + ClientUserID string + LegalName string + PhoneNumber *string + PhoneNumberVerifiedTime *time.Time + EmailAddress string + EmailAddressVerifiedTime *time.Time + RedirectURI string + UpdateMode bool +} + +type LinkToken interface { + Token() string + Expiration() time.Time +} + +var ( + _ LinkToken = PlaidLinkToken{} +) + +type PlaidLinkToken struct { + LinkToken string + Expires time.Time +} + +func (p PlaidLinkToken) Token() string { + return p.LinkToken +} + +func (p PlaidLinkToken) Expiration() time.Time { + return p.Expires +} + diff --git a/pkg/internal/platypus/transaction.go b/pkg/internal/platypus/transaction.go new file mode 100644 index 00000000..79b3802c --- /dev/null +++ b/pkg/internal/platypus/transaction.go @@ -0,0 +1,119 @@ +package platypus + +import ( + "github.com/pkg/errors" + "github.com/plaid/plaid-go/plaid" + "time" +) + +type Transaction interface { + GetAmount() int64 + GetBankAccountId() string + GetCategory() []string + GetDate() time.Time + GetDateLocal(timezone *time.Location) time.Time + GetISOCurrencyCode() string + GetIsPending() bool + GetMerchantName() string + GetName() string + GetOriginalDescription() string + GetPendingTransactionId() *string + GetTransactionId() string + GetUnofficialCurrencyCode() string +} + +var ( + _ Transaction = PlaidTransaction{} +) + +type PlaidTransaction struct { + Amount int64 + BankAccountId string + Category []string + Date time.Time + ISOCurrencyCode string + UnofficialCurrencyCode string + IsPending bool + MerchantName string + Name string + OriginalDescription string + PendingTransactionId *string + TransactionId string +} + +func NewTransactionFromPlaid(input plaid.Transaction) (Transaction, error) { + date, err := time.Parse("2006-01-02", input.GetDate()) + if err != nil { + return nil, errors.Wrap(err, "failed to parse transaction date") + } + pendingTransactionId, _ := input.GetPendingTransactionIdOk() + + transaction := PlaidTransaction{ + Amount: int64(input.GetAmount() * 100), + BankAccountId: input.GetAccountId(), + Category: input.GetCategory(), + Date: date, + ISOCurrencyCode: input.GetIsoCurrencyCode(), + UnofficialCurrencyCode: input.GetUnofficialCurrencyCode(), + IsPending: input.GetPending(), + MerchantName: input.GetMerchantName(), + Name: input.GetName(), + OriginalDescription: input.GetOriginalDescription(), + PendingTransactionId: pendingTransactionId, + TransactionId: input.GetTransactionId(), + } + + return transaction, nil +} + +func (p PlaidTransaction) GetAmount() int64 { + return p.Amount +} + +func (p PlaidTransaction) GetBankAccountId() string { + return p.BankAccountId +} + +func (p PlaidTransaction) GetCategory() []string { + return p.Category +} + +func (p PlaidTransaction) GetDate() time.Time { + return p.Date +} + +func (p PlaidTransaction) GetDateLocal(timezone *time.Location) time.Time { + return p.Date.In(timezone) +} + +func (p PlaidTransaction) GetISOCurrencyCode() string { + return p.ISOCurrencyCode +} + +func (p PlaidTransaction) GetIsPending() bool { + return p.IsPending +} + +func (p PlaidTransaction) GetMerchantName() string { + return p.MerchantName +} + +func (p PlaidTransaction) GetName() string { + return p.Name +} + +func (p PlaidTransaction) GetOriginalDescription() string { + return p.OriginalDescription +} + +func (p PlaidTransaction) GetPendingTransactionId() *string { + return p.PendingTransactionId +} + +func (p PlaidTransaction) GetTransactionId() string { + return p.TransactionId +} + +func (p PlaidTransaction) GetUnofficialCurrencyCode() string { + return p.UnofficialCurrencyCode +} diff --git a/pkg/internal/platypus/webhook.go b/pkg/internal/platypus/webhook.go new file mode 100644 index 00000000..082e1fae --- /dev/null +++ b/pkg/internal/platypus/webhook.go @@ -0,0 +1,212 @@ +package platypus + +import ( + "context" + "encoding/json" + "github.com/MicahParks/keyfunc" + "github.com/getsentry/sentry-go" + "github.com/monetr/rest-api/pkg/internal/myownsanity" + "github.com/pkg/errors" + "github.com/plaid/plaid-go/plaid" + "github.com/sirupsen/logrus" + "sync" + "sync/atomic" + "time" +) + +type WebhookVerificationKey struct { + // The alg member identifies the cryptographic algorithm family used with the key. + Alg string `json:"alg"` + // The crv member identifies the cryptographic curve used with the key. + Crv string `json:"crv"` + // The kid (Key ID) member can be used to match a specific key. This can be used, for instance, to choose among a set of keys within the JWK during key rollover. + Kid string `json:"kid"` + // The kty (key type) parameter identifies the cryptographic algorithm family used with the key, such as RSA or EC. + Kty string `json:"kty"` + // The use (public key use) parameter identifies the intended use of the public key. + Use string `json:"use"` + // The x member contains the x coordinate for the elliptic curve point. + X string `json:"x"` + // The y member contains the y coordinate for the elliptic curve point. + Y string `json:"y"` + CreatedAt int32 `json:"created_at"` + ExpiredAt *int32 `json:"expired_at"` +} + +func NewWebhookVerificationKeyFromPlaid(input plaid.JWKPublicKey) (WebhookVerificationKey, error) { + return WebhookVerificationKey{ + Alg: input.GetAlg(), + Crv: input.GetCrv(), + Kid: input.GetKid(), + Kty: input.GetKty(), + Use: input.GetUse(), + X: input.GetX(), + Y: input.GetY(), + CreatedAt: input.GetCreatedAt(), + ExpiredAt: myownsanity.Int32P(input.GetExpiredAt()), + }, nil +} + +type WebhookVerification interface { + GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) + Close() error +} + +var ( + _ WebhookVerification = &memoryWebhookVerification{} +) + +func NewInMemoryWebhookVerification(log *logrus.Entry, plaid Platypus, cleanupInterval time.Duration) WebhookVerification { + verification := &memoryWebhookVerification{ + closed: 0, + log: log, + plaid: plaid, + lock: sync.Mutex{}, + cache: map[string]*keyCacheItem{}, + cleanupTicker: time.NewTicker(cleanupInterval), + closer: make(chan chan error, 1), + } + go verification.cacheWorker() // Start the background worker. + + return verification +} + +type keyCacheItem struct { + expiration time.Time + keyFunction *keyfunc.JWKS +} + +type memoryWebhookVerification struct { + closed uint32 + log *logrus.Entry + plaid Platypus + lock sync.Mutex + cache map[string]*keyCacheItem + cleanupTicker *time.Ticker + closer chan chan error +} + +func (m *memoryWebhookVerification) GetVerificationKey(ctx context.Context, keyId string) (*keyfunc.JWKS, error) { + if atomic.LoadUint32(&m.closed) > 0 { + return nil, errors.New("webhook verification is closed") + } + + span := sentry.StartSpan(ctx, "GetVerificationKey [InMemory]") + defer span.Finish() + + log := m.log.WithField("keyId", keyId).WithContext(ctx) + + m.lock.Lock() + defer m.lock.Unlock() + + item, ok := m.cache[keyId] + if ok { + if item.expiration.After(time.Now()) { + log.Trace("jwk function already present in cache, returning") + return item.keyFunction, nil + } + + log.Trace("jwk function present in cache, but is expired; the cached function will be removed and a new one will be retrieved") + delete(m.cache, keyId) + } + + log.Trace("retrieving jwk from plaid") + + result, err := m.plaid.GetWebhookVerificationKey(span.Context(), keyId) + if err != nil { + return nil, err + } + + var expiration time.Time + if result.ExpiredAt != nil { + expiration = time.Unix(int64(*result.ExpiredAt), 0) + } else { + // Making a huge assumption here, and this might end up causing problems later on. Maybe we should also add a + // check here to make sure that items that are close to expiration even here should not be cached? + expiration = time.Unix(int64(result.CreatedAt), 0).Add(30 * time.Minute) + } + + var keys = struct { + Keys []WebhookVerificationKey `json:"keys"` + }{ + Keys: []WebhookVerificationKey{ + *result, + }, + } + + encodedKeys, err := json.Marshal(keys) + if err != nil { + return nil, errors.Wrap(err, "failed to convert plaid verification key to json") + } + + var jwksJSON json.RawMessage = encodedKeys + + jwksFunc, err := keyfunc.New(jwksJSON) + if err != nil { + return nil, errors.Wrap(err, "failed to create key function") + } + + m.cache[keyId] = &keyCacheItem{ + expiration: expiration, + keyFunction: jwksFunc, + } + + return jwksFunc, nil +} + +func (m *memoryWebhookVerification) cacheWorker() { + for { + select { + case _ = <-m.cleanupTicker.C: + m.cleanup() + case promise := <-m.closer: + m.log.Debug("closing jwk cache, stopping background worker") + promise <- nil + return + } + } +} + +func (m *memoryWebhookVerification) cleanup() { + m.lock.Lock() + defer m.lock.Unlock() + + if len(m.cache) == 0 { + m.log.Debug("no items in Plaid jwk cache, nothing to cleanup") + return + } + + m.log.Debug("cleaning up Plaid jwk cache") + + itemsToRemove := make([]string, 0, len(m.cache)) + for key, item := range m.cache { + // If the item expiration is not in the future, then we need to add it to our list to be removed. + if !item.expiration.After(time.Now()) { + itemsToRemove = append(itemsToRemove, key) + } + } + + if len(itemsToRemove) == 0 { + m.log.Debug("no items have expired in cache") + return + } + + m.log.Debugf("found %d expired item(s); cleaning them up", len(itemsToRemove)) + + for _, key := range itemsToRemove { + delete(m.cache, key) + } + + return +} + +func (m *memoryWebhookVerification) Close() error { + if ok := atomic.CompareAndSwapUint32(&m.closed, 0, 1); !ok { + return errors.New("webhook verification is already closed") + } + + promise := make(chan error) + m.closer <- promise + + return <-promise +} diff --git a/pkg/internal/platypus/webhook_test.go b/pkg/internal/platypus/webhook_test.go new file mode 100644 index 00000000..a49255a5 --- /dev/null +++ b/pkg/internal/platypus/webhook_test.go @@ -0,0 +1,42 @@ +package platypus + +import ( + "context" + "github.com/brianvoe/gofakeit/v6" + "github.com/jarcoal/httpmock" + "github.com/monetr/rest-api/pkg/config" + "github.com/monetr/rest-api/pkg/internal/mock_plaid" + "github.com/monetr/rest-api/pkg/internal/testutils" + "github.com/monetr/rest-api/pkg/repository" + "github.com/monetr/rest-api/pkg/secrets" + "github.com/plaid/plaid-go/plaid" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestNewInMemoryWebhookVerification(t *testing.T) { + t.Run("simple", func(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + mock_plaid.MockGetWebhookVerificationKey(t) + + log := testutils.GetLog(t) + db := testutils.GetPgDatabaseTxn(t) + secret := secrets.NewPostgresPlaidSecretsProvider(log, db) + plaidRepo := repository.NewPlaidRepository(db) + + plaid := NewPlaid(log, secret, plaidRepo, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, + }) + + webhookVerification := NewInMemoryWebhookVerification(log, plaid, time.Second * 1) + + verify, err := webhookVerification.GetVerificationKey(context.Background(), gofakeit.UUID()) + assert.NoError(t, err, "must get verification") + assert.NotNil(t, verify, "verify must not be nil") + }) +} diff --git a/pkg/internal/testutils/account.go b/pkg/internal/testutils/account.go new file mode 100644 index 00000000..28e12f50 --- /dev/null +++ b/pkg/internal/testutils/account.go @@ -0,0 +1,14 @@ +package testutils + +import ( + "github.com/OneOfOne/xxhash" + "testing" +) + +// GetAccountIdForTest is used to create unique accountIds for individual tests. These accountIds are probably unique +// between tests. It is possible for two tests to have the same accountId, but it is very unlikely. This does make sure +// though that a test's accountId does stay the same between test runs as it is not random. This does not guarantee that +// the accountId is present in the database and is primarily aimed at mocks. +func GetAccountIdForTest(t *testing.T) uint64 { + return xxhash.Checksum64([]byte(t.Name())) +} diff --git a/pkg/internal/testutils/plaid.go b/pkg/internal/testutils/plaid.go index 5fb9d432..59024629 100644 --- a/pkg/internal/testutils/plaid.go +++ b/pkg/internal/testutils/plaid.go @@ -8,5 +8,5 @@ import ( type MockPlaidData struct { PlaidTokens map[string]models.PlaidToken PlaidLinks map[string]models.PlaidLink - BankAccounts map[string]map[string]plaid.Account + BankAccounts map[string]map[string]plaid.AccountBase } diff --git a/pkg/internal/testutils/seed.go b/pkg/internal/testutils/seed.go index 967fc1a7..2c08f05c 100644 --- a/pkg/internal/testutils/seed.go +++ b/pkg/internal/testutils/seed.go @@ -6,6 +6,7 @@ import ( "github.com/brianvoe/gofakeit/v6" "github.com/go-pg/pg/v10" "github.com/monetr/rest-api/pkg/hash" + "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/models" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/require" @@ -32,7 +33,7 @@ func SeedAccount(t *testing.T, db *pg.DB, options SeedAccountOption) (*models.Us plaidData := &MockPlaidData{ PlaidTokens: map[string]models.PlaidToken{}, PlaidLinks: map[string]models.PlaidLink{}, - BankAccounts: map[string]map[string]plaid.Account{}, + BankAccounts: map[string]map[string]plaid.AccountBase{}, } var user models.User @@ -190,38 +191,37 @@ func SeedAccount(t *testing.T, db *pg.DB, options SeedAccountOption) (*models.Us }, } - plaidData.BankAccounts[accessToken] = map[string]plaid.Account{ + checkingAccountSubType := plaid.ACCOUNTSUBTYPE_CHECKING + savingAccountSubType := plaid.ACCOUNTSUBTYPE_SAVINGS + + plaidData.BankAccounts[accessToken] = map[string]plaid.AccountBase{ checkingAccountId: { - AccountID: checkingAccountId, - Balances: plaid.AccountBalances{ - Available: float64(checkingBalance) / 100, - Current: float64(checkingBalance) / 100, - Limit: 0, - ISOCurrencyCode: "USD", - UnofficialCurrencyCode: "", + AccountId: checkingAccountId, + Balances: plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(checkingBalance) / 100)), + Current: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(checkingBalance) / 100)), + IsoCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), + UnofficialCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), }, - Mask: "1234", - Name: "Checking Account", - OfficialName: "Checking", - Subtype: "depository", - Type: "checking", - VerificationStatus: "", + Mask: *plaid.NewNullableString(myownsanity.StringP("1234")), + Name: "Checking Account", + OfficialName: *plaid.NewNullableString(myownsanity.StringP("Checking")), + Type: plaid.ACCOUNTTYPE_DEPOSITORY, + Subtype: *plaid.NewNullableAccountSubtype(&checkingAccountSubType), }, savingsAccountId: { - AccountID: savingsAccountId, - Balances: plaid.AccountBalances{ - Available: float64(savingBalance) / 100, - Current: float64(savingBalance) / 100, - Limit: 0, - ISOCurrencyCode: "USD", - UnofficialCurrencyCode: "", + AccountId: savingsAccountId, + Balances: plaid.AccountBalance{ + Available: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(savingBalance) / 100)), + Current: *plaid.NewNullableFloat32(myownsanity.Float32P(float32(savingBalance) / 100)), + IsoCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), + UnofficialCurrencyCode: *plaid.NewNullableString(myownsanity.StringP("USD")), }, - Mask: "2345", - Name: "Savings Account", - OfficialName: "Savings", - Subtype: "depository", - Type: "saving", - VerificationStatus: "", + Mask: *plaid.NewNullableString(myownsanity.StringP("2345")), + Name: "Savings Account", + OfficialName: *plaid.NewNullableString(myownsanity.StringP("Savings")), + Type: plaid.ACCOUNTTYPE_DEPOSITORY, + Subtype: *plaid.NewNullableAccountSubtype(&savingAccountSubType), }, } diff --git a/pkg/internal/vault_helper/vault.go b/pkg/internal/vault_helper/vault.go index 92ddc2e8..9ffdd028 100644 --- a/pkg/internal/vault_helper/vault.go +++ b/pkg/internal/vault_helper/vault.go @@ -24,6 +24,7 @@ type Config struct { Auth string Token string TokenFile string + Username, Password string Timeout time.Duration TLSCertificatePath string TLSKeyPath string @@ -217,6 +218,24 @@ func (v *vaultBase) authenticate() error { log := v.log.WithField("method", v.config.Auth) switch v.config.Auth { + case "userpass": + log.Trace("authenticating to vault") + result, err := v.client.Logical().Write("auth/userpass/login/"+v.config.Username, map[string]interface{}{ + "password": v.config.Password, + "role": v.config.Role, + }) + if err != nil { + log.WithError(err).Errorf("failed to authenticate to vault") + return errors.Wrap(err, "failed to authenticate to vault") + } + + if result.Auth == nil { + log.WithError(err).Fatalf("no authentication returned from vault") + return errors.Errorf("no authentication returned from vault") + } + + v.client.SetToken(result.Auth.ClientToken) + log.Trace("successfully authenticated to vault") case "token": v.client.SetToken(v.config.Token) case "kubernetes": diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index 97577a85..90771484 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -2,13 +2,13 @@ package jobs import ( "context" + "github.com/monetr/rest-api/pkg/internal/platypus" "math" "time" "github.com/go-pg/pg/v10" "github.com/gocraft/work" "github.com/gomodule/redigo/redis" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" "github.com/monetr/rest-api/pkg/metrics" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/pubsub" @@ -32,13 +32,12 @@ var ( _ JobManager = &nonDistributedJobManager{} ) - type jobManagerBase struct { log *logrus.Entry work *work.WorkerPool queue *work.Enqueuer db *pg.DB - plaidClient plaid_helper.Client + plaidClient platypus.Platypus plaidSecrets secrets.PlaidSecretsProvider stats *metrics.Stats ps pubsub.PublishSubscribe @@ -48,7 +47,7 @@ func NewNonDistributedJobManager( log *logrus.Entry, pool *redis.Pool, db *pg.DB, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, plaidSecrets secrets.PlaidSecretsProvider, ) JobManager { @@ -69,7 +68,7 @@ func NewJobManager( log *logrus.Entry, pool *redis.Pool, db *pg.DB, - plaidClient plaid_helper.Client, + plaidClient platypus.Platypus, stats *metrics.Stats, plaidSecrets secrets.PlaidSecretsProvider, ) JobManager { @@ -97,7 +96,6 @@ func NewJobManager( manager.work.Job(PullLatestTransactions, manager.pullLatestTransactions) manager.work.Job(PullHistoricalTransactions, manager.pullHistoricalTransactions) manager.work.Job(RemoveTransactions, manager.removeTransactions) - manager.work.Job(UpdateInstitutions, manager.updateInstitutions) manager.work.Job(RemoveLink, manager.removeLink) // Every 30 minutes. 0 */30 * * * * diff --git a/pkg/jobs/non_distributed.go b/pkg/jobs/non_distributed.go index 5f8c067e..3641b015 100644 --- a/pkg/jobs/non_distributed.go +++ b/pkg/jobs/non_distributed.go @@ -4,7 +4,7 @@ import ( "context" "fmt" "github.com/go-pg/pg/v10" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/metrics" "github.com/monetr/rest-api/pkg/pubsub" "github.com/monetr/rest-api/pkg/secrets" @@ -15,7 +15,7 @@ import ( type nonDistributedJobManager struct { log *logrus.Entry db *pg.DB - plaidClient plaid_helper.Client + plaidClient platypus.Platypus plaidSecrets secrets.PlaidSecretsProvider stats *metrics.Stats ps pubsub.PublishSubscribe diff --git a/pkg/jobs/pull_account_balances.go b/pkg/jobs/pull_account_balances.go index 3ceb4d7b..02f2c84d 100644 --- a/pkg/jobs/pull_account_balances.go +++ b/pkg/jobs/pull_account_balances.go @@ -13,7 +13,6 @@ import ( "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/repository" "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" ) const ( @@ -162,40 +161,31 @@ func (j *jobManagerBase) pullAccountBalances(job *work.Job) (err error) { log.Debugf("requesting information for %d bank account(s)", len(itemBankAccountIds)) - result, err := j.plaidClient.GetAccounts( + platypus, err := j.plaidClient.NewClient(span.Context(), link, accessToken) + if err != nil { + log.WithError(err).Error("failed to create plaid client") + return err + } + + result, err := platypus.GetAccounts( span.Context(), - accessToken, - plaid.GetAccountsOptions{ - AccountIDs: itemBankAccountIds, - }, + itemBankAccountIds..., ) if err != nil { log.WithError(err).Error("failed to retrieve bank accounts from plaid") - switch plaidErr := errors.Cause(err).(type) { - case plaid.Error: - switch plaidErr.ErrorType { - case "ITEM_ERROR": - link.LinkStatus = models.LinkStatusError - link.ErrorCode = &plaidErr.ErrorCode - if updateErr := repo.UpdateLink(span.Context(), link); updateErr != nil { - log.WithError(updateErr).Error("failed to update link to be an error state") - } - } - } - return errors.Wrap(err, "failed to retrieve bank accounts from plaid") } updatedBankAccounts := make([]models.BankAccount, 0, len(result)) for _, item := range result { - bankAccount := plaidIdsToBank[item.AccountID] + bankAccount := plaidIdsToBank[item.GetAccountId()] bankLog := log.WithFields(logrus.Fields{ "bankAccountId": bankAccount.BankAccountId, "linkId": bankAccount.LinkId, }) shouldUpdate := false - available := int64(item.Balances.Available * 100) - current := int64(item.Balances.Current * 100) + available := item.GetBalances().GetAvailable() + current := item.GetBalances().GetCurrent() if bankAccount.CurrentBalance != current { bankLog = bankLog.WithField("currentBalanceChanged", true) diff --git a/pkg/jobs/pull_account_balances_test.go b/pkg/jobs/pull_account_balances_test.go index eeb54f0e..97a2ccce 100644 --- a/pkg/jobs/pull_account_balances_test.go +++ b/pkg/jobs/pull_account_balances_test.go @@ -6,15 +6,16 @@ import ( "github.com/go-pg/pg/v10" "github.com/gocraft/work" "github.com/jarcoal/httpmock" + "github.com/monetr/rest-api/pkg/config" "github.com/monetr/rest-api/pkg/internal/mock_plaid" "github.com/monetr/rest-api/pkg/internal/mock_secrets" - "github.com/monetr/rest-api/pkg/internal/plaid_helper" + "github.com/monetr/rest-api/pkg/internal/platypus" "github.com/monetr/rest-api/pkg/internal/testutils" "github.com/monetr/rest-api/pkg/repository" + "github.com/monetr/rest-api/pkg/secrets" "github.com/plaid/plaid-go/plaid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "net/http" "testing" "time" ) @@ -42,11 +43,12 @@ func TestPullAccountBalances(t *testing.T) { return nil }), "must retrieve linkId") - plaidClient := plaid_helper.NewPlaidClient(log, plaid.ClientOptions{ - ClientID: gofakeit.UUID(), - Secret: gofakeit.UUID(), - Environment: plaid.Sandbox, - HTTPClient: http.DefaultClient, + secretProvider := secrets.NewPostgresPlaidSecretsProvider(log, db) + plaidRepo := repository.NewPlaidRepository(db) + plaidClient := platypus.NewPlaid(log, secretProvider, plaidRepo, config.Plaid{ + ClientID: gofakeit.UUID(), + ClientSecret: gofakeit.UUID(), + Environment: plaid.Sandbox, }) plaidSecrets := mock_secrets.NewMockPlaidSecrets() diff --git a/pkg/jobs/pull_historical_transactions.go b/pkg/jobs/pull_historical_transactions.go index 1872bb5b..b890fc63 100644 --- a/pkg/jobs/pull_historical_transactions.go +++ b/pkg/jobs/pull_historical_transactions.go @@ -9,7 +9,6 @@ import ( "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/repository" - "github.com/monetr/rest-api/pkg/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" "strconv" @@ -75,18 +74,6 @@ func (j *jobManagerBase) pullHistoricalTransactions(job *work.Job) (err error) { twoYearsAgo := time.Now().Add(-2 * 365 * 24 * time.Hour).UTC() return j.getRepositoryForJob(job, func(repo repository.Repository) error { - account, err := repo.GetAccount(span.Context()) - if err != nil { - log.WithError(err).Error("failed to retrieve account for job") - return err - } - - timezone, err := account.GetTimezone() - if err != nil { - log.WithError(err).Warn("failed to get account's time zone, defaulting to UTC") - timezone = time.UTC - } - link, err := repo.GetLink(span.Context(), linkId) if err != nil { log.WithError(err).Error("failed to retrieve link details to pull historical transactions") @@ -131,135 +118,29 @@ func (j *jobManagerBase) pullHistoricalTransactions(job *work.Job) (err error) { log.Debugf("retrieving transactions for %d bank account(s)", len(itemBankAccountIds)) - transactions, err := j.plaidClient.GetAllTransactions( - span.Context(), - accessToken, - twoYearsAgo, - time.Now(), - itemBankAccountIds, - ) - if err != nil { - log.WithError(err).Error("failed to retrieve transactions from plaid") - return errors.Wrap(err, "failed to retrieve transactions from plaid") - } - - plaidTransactionIds := make([]string, len(transactions)) - for i, transaction := range transactions { - plaidTransactionIds[i] = transaction.ID - } - transactionsByPlaidId, err := repo.GetTransactionsByPlaidId(span.Context(), linkId, plaidTransactionIds) + platypus, err := j.plaidClient.NewClient(span.Context(), link, accessToken) if err != nil { - log.WithError(err).Error("failed to retrieve transaction ids for updating plaid transactions") + log.WithError(err).Error("failed to create plaid client for link") return err } - transactionsToUpdate := make([]*models.Transaction, 0) - transactionsToInsert := make([]models.Transaction, 0) - now := time.Now().UTC() - for _, plaidTransaction := range transactions { - amount := int64(plaidTransaction.Amount * 100) - - date, _ := util.ParseInLocal("2006-01-02", plaidTransaction.Date, timezone) - var authorizedDate *time.Time - if plaidTransaction.AuthorizedDate != "" { - authDate, _ := util.ParseInLocal("2006-01-02", plaidTransaction.AuthorizedDate, timezone) - authorizedDate = &authDate - } - - var pendingPlaidTransactionId *string - if plaidTransaction.PendingTransactionID != "" { - pendingPlaidTransactionId = &plaidTransaction.PendingTransactionID - } - - transactionName := plaidTransaction.Name - - // We only want to make the transaction name be the merchant name if the merchant name is shorter. This is - // due to something I observed with a dominos transaction, where the merchant was improperly parsed and the - // transaction ended up being called `Mnuslindstrom` rather than `Domino's`. This should fix that problem. - if plaidTransaction.MerchantName != "" && len(plaidTransaction.MerchantName) < len(transactionName) { - transactionName = plaidTransaction.MerchantName - } - - existingTransaction, ok := transactionsByPlaidId[plaidTransaction.ID] - if !ok { - transactionsToInsert = append(transactionsToInsert, models.Transaction{ - AccountId: accountId, - BankAccountId: plaidIdsToBankIds[plaidTransaction.AccountID], - PlaidTransactionId: plaidTransaction.ID, - Amount: amount, - SpendingId: nil, - Spending: nil, - Categories: plaidTransaction.Category, - OriginalCategories: plaidTransaction.Category, - Date: date, - AuthorizedDate: authorizedDate, - Name: transactionName, - OriginalName: plaidTransaction.Name, - MerchantName: plaidTransaction.MerchantName, - OriginalMerchantName: plaidTransaction.MerchantName, - IsPending: plaidTransaction.Pending, - CreatedAt: now, - PendingPlaidTransactionId: pendingPlaidTransactionId, - }) - continue - } - - var shouldUpdate bool - if existingTransaction.Amount != amount { - shouldUpdate = true - } - - if existingTransaction.IsPending != plaidTransaction.Pending { - shouldUpdate = true - } - - if !myownsanity.TimesPEqual(existingTransaction.AuthorizedDate, authorizedDate) { - shouldUpdate = true - } - - if existingTransaction.PendingPlaidTransactionId != pendingPlaidTransactionId { - shouldUpdate = true - } - - existingTransaction.Amount = amount - existingTransaction.IsPending = plaidTransaction.Pending - existingTransaction.AuthorizedDate = authorizedDate - existingTransaction.PendingPlaidTransactionId = pendingPlaidTransactionId - - // Update old transactions calculated name as we can. - if existingTransaction.Name != transactionName { - existingTransaction.Name = transactionName - shouldUpdate = true - } - - // Fix timezone of records. - if !existingTransaction.Date.Equal(date) { - existingTransaction.Date = date - shouldUpdate = true - } - - if shouldUpdate { - transactionsToUpdate = append(transactionsToUpdate, &existingTransaction) - } - } - - if len(transactionsToUpdate) > 0 { - if err = repo.UpdateTransactions(span.Context(), transactionsToUpdate); err != nil { - log.WithError(err).Errorf("failed to update transactions for job") - return err - } + transactions, err := platypus.GetAllTransactions(span.Context(), twoYearsAgo, time.Now(), itemBankAccountIds) + if err != nil { + log.WithError(err).Error("failed to retrieve transactions from plaid") + return errors.Wrap(err, "failed to retrieve transactions from plaid") } - if len(transactionsToInsert) > 0 { - // Reverse the list so the oldest records are inserted first. - for i, j := 0, len(transactionsToInsert)-1; i < j; i, j = i+1, j-1 { - transactionsToInsert[i], transactionsToInsert[j] = transactionsToInsert[j], transactionsToInsert[i] - } - if err = repo.InsertTransactions(span.Context(), transactionsToInsert); err != nil { - log.WithError(err).Error("failed to insert new transactions") - return err - } + if err = j.upsertTransactions( + span.Context(), + log, + repo, + link, + plaidIdsToBankIds, + transactions, + ); err != nil { + log.WithError(err).Error("failed to upsert transactions from plaid") + return err } link.LastSuccessfulUpdate = myownsanity.TimeP(time.Now().UTC()) diff --git a/pkg/jobs/pull_initial_transactions.go b/pkg/jobs/pull_initial_transactions.go index dcf49236..8fff1cf5 100644 --- a/pkg/jobs/pull_initial_transactions.go +++ b/pkg/jobs/pull_initial_transactions.go @@ -8,9 +8,7 @@ import ( "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/repository" - "github.com/monetr/rest-api/pkg/util" "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" "strconv" "time" ) @@ -54,18 +52,6 @@ func (j *jobManagerBase) pullInitialTransactions(job *work.Job) (err error) { }) return j.getRepositoryForJob(job, func(repo repository.Repository) error { - account, err := repo.GetAccount(span.Context()) - if err != nil { - log.WithError(err).Error("failed to retrieve account for job") - return err - } - - timezone, err := account.GetTimezone() - if err != nil { - log.WithError(err).Warn("failed to get account's time zone, defaulting to UTC") - timezone = time.UTC - } - link, err := repo.GetLink(span.Context(), linkId) if err != nil { log.WithError(err).Error("cannot pull initial transactions for link provided") @@ -93,39 +79,26 @@ func (j *jobManagerBase) pullInitialTransactions(job *work.Job) (err error) { return nil } - bankAccountIdsByPlaid := map[string]uint64{} + plaidIdsToBankIds := map[string]uint64{} bankAccountIds := make([]string, len(link.BankAccounts)) for i, bankAccount := range link.BankAccounts { bankAccountIds[i] = bankAccount.PlaidAccountId - bankAccountIdsByPlaid[bankAccount.PlaidAccountId] = bankAccount.BankAccountId + plaidIdsToBankIds[bankAccount.PlaidAccountId] = bankAccount.BankAccountId } now := time.Now().UTC() - plaidTransactions, err := j.plaidClient.GetAllTransactions( - span.Context(), - accessToken, - now.Add(-30*24*time.Hour), - now, - bankAccountIds, - ) + platypus, err := j.plaidClient.NewClient(span.Context(), link, accessToken) if err != nil { - log.WithError(err).Error("failed to retrieve initial transactions") - - switch plaidErr := errors.Cause(err).(type) { - case plaid.Error: - switch plaidErr.ErrorType { - case "ITEM_ERROR": - link.LinkStatus = models.LinkStatusError - link.ErrorCode = &plaidErr.ErrorCode - if updateErr := repo.UpdateLink(span.Context(), link); updateErr != nil { - log.WithError(updateErr).Error("failed to update link to be an error state") - } - } - } - + log.WithError(err).Error("failed to create plaid client for link") return err } + plaidTransactions, err := platypus.GetAllTransactions(span.Context(), now.Add(-30*24*time.Hour), now, bankAccountIds) + if err != nil { + log.WithError(err).Error("failed to retrieve initial transactions from plaid") + return errors.Wrap(err, "failed to retrieve initial transactions from plaid") + } + if len(plaidTransactions) == 0 { log.Warn("no transactions were retrieved from plaid") return nil @@ -133,50 +106,15 @@ func (j *jobManagerBase) pullInitialTransactions(job *work.Job) (err error) { log.Debugf("retreived %d transaction(s) from plaid, processing now", len(plaidTransactions)) - transactions := make([]models.Transaction, len(plaidTransactions)) - for i, plaidTransaction := range plaidTransactions { - date, _ := util.ParseInLocal("2006-01-02", plaidTransaction.Date, timezone) - var authorizedDate *time.Time - if plaidTransaction.AuthorizedDate != "" { - authDate, _ := util.ParseInLocal("2006-01-02", plaidTransaction.AuthorizedDate, timezone) - authorizedDate = &authDate - } - - transactionName := plaidTransaction.Name - - // We only want to make the transaction name be the merchant name if the merchant name is shorter. This is - // due to something I observed with a dominos transaction, where the merchant was improperly parsed and the - // transaction ended up being called `Mnuslindstrom` rather than `Domino's`. This should fix that problem. - if plaidTransaction.MerchantName != "" && len(plaidTransaction.MerchantName) < len(transactionName) { - transactionName = plaidTransaction.MerchantName - } - - transactions[i] = models.Transaction{ - AccountId: repo.AccountId(), - BankAccountId: bankAccountIdsByPlaid[plaidTransaction.AccountID], - PlaidTransactionId: plaidTransaction.ID, - Amount: int64(plaidTransaction.Amount * 100), - SpendingId: nil, - Categories: plaidTransaction.Category, - OriginalCategories: plaidTransaction.Category, - Date: date, - AuthorizedDate: authorizedDate, - Name: transactionName, - OriginalName: plaidTransaction.Name, - MerchantName: plaidTransaction.MerchantName, - OriginalMerchantName: plaidTransaction.MerchantName, - IsPending: plaidTransaction.Pending, - CreatedAt: now, - } - } - - // Reverse the list so the oldest records are inserted first. - for i, j := 0, len(transactions)-1; i < j; i, j = i+1, j-1 { - transactions[i], transactions[j] = transactions[j], transactions[i] - } - - if err = repo.InsertTransactions(span.Context(), transactions); err != nil { - log.WithError(err).Error("failed to store initial transactions") + if err = j.upsertTransactions( + span.Context(), + log, + repo, + link, + plaidIdsToBankIds, + plaidTransactions, + ); err != nil { + log.WithError(err).Error("failed to upsert transactions from plaid") return err } diff --git a/pkg/jobs/pull_latest_transactions.go b/pkg/jobs/pull_latest_transactions.go index 23a25bca..bdc2c073 100644 --- a/pkg/jobs/pull_latest_transactions.go +++ b/pkg/jobs/pull_latest_transactions.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/monetr/rest-api/pkg/crumbs" "strconv" - "strings" "time" "github.com/getsentry/sentry-go" @@ -13,9 +12,7 @@ import ( "github.com/monetr/rest-api/pkg/internal/myownsanity" "github.com/monetr/rest-api/pkg/models" "github.com/monetr/rest-api/pkg/repository" - "github.com/monetr/rest-api/pkg/util" "github.com/pkg/errors" - "github.com/plaid/plaid-go/plaid" "github.com/sirupsen/logrus" ) @@ -123,18 +120,6 @@ func (j *jobManagerBase) pullLatestTransactions(job *work.Job) (err error) { }) return j.getRepositoryForJob(job, func(repo repository.Repository) error { - account, err := repo.GetAccount(span.Context()) - if err != nil { - log.WithError(err).Error("failed to retrieve account for job") - return err - } - - timezone, err := account.GetTimezone() - if err != nil { - log.WithError(err).Warn("failed to get account's time zone, defaulting to UTC") - timezone = time.UTC - } - link, err := repo.GetLink(span.Context(), linkId) if err != nil { log.WithError(err).Error("failed to retrieve link details to pull transactions") @@ -193,156 +178,28 @@ func (j *jobManagerBase) pullLatestTransactions(job *work.Job) (err error) { } end := time.Now() - transactions, err := j.plaidClient.GetAllTransactions( - span.Context(), - accessToken, - start, - end, - itemBankAccountIds, - ) - if err != nil { - log.WithError(err).Error("failed to retrieve transactions from plaid") - switch plaidErr := errors.Cause(err).(type) { - case plaid.Error: - link.LinkStatus = models.LinkStatusError - link.ErrorCode = myownsanity.StringP(strings.Join([]string{ - plaidErr.ErrorType, - plaidErr.ErrorCode, - }, ".")) - if updateErr := repo.UpdateLink(span.Context(), link); updateErr != nil { - log.WithError(updateErr).Error("failed to update link to be an error state") - return err - } - - // Don't return an error here, we set the link to an error state and we don't want to have retry logic - // for this right now. - return nil - default: - log.WithError(err).Warnf("unknown error type from Plaid client: %T", plaidErr) - } - - return errors.Wrap(err, "failed to retrieve transactions from plaid") - } - - plaidTransactionIds := make([]string, len(transactions)) - for i, transaction := range transactions { - plaidTransactionIds[i] = transaction.ID - } - - transactionsByPlaidId, err := repo.GetTransactionsByPlaidId(span.Context(), linkId, plaidTransactionIds) + platypus, err := j.plaidClient.NewClient(span.Context(), link, accessToken) if err != nil { - log.WithError(err).Error("failed to retrieve transaction ids for updating plaid transactions") + log.WithError(err).Error("failed to create plaid client for link") return err } - transactionsToUpdate := make([]*models.Transaction, 0) - transactionsToInsert := make([]models.Transaction, 0) - now := time.Now().UTC() - for _, plaidTransaction := range transactions { - amount := int64(plaidTransaction.Amount * 100) - - date, _ := util.ParseInLocal("2006-01-02", plaidTransaction.Date, timezone) - var authorizedDate *time.Time - if plaidTransaction.AuthorizedDate != "" { - authDate, _ := util.ParseInLocal("2006-01-02", plaidTransaction.AuthorizedDate, timezone) - authorizedDate = &authDate - } - - var pendingPlaidTransactionId *string - if plaidTransaction.PendingTransactionID != "" { - pendingPlaidTransactionId = &plaidTransaction.PendingTransactionID - } - - transactionName := plaidTransaction.Name - - // We only want to make the transaction name be the merchant name if the merchant name is shorter. This is - // due to something I observed with a dominos transaction, where the merchant was improperly parsed and the - // transaction ended up being called `Mnuslindstrom` rather than `Domino's`. This should fix that problem. - if plaidTransaction.MerchantName != "" && len(plaidTransaction.MerchantName) < len(transactionName) { - transactionName = plaidTransaction.MerchantName - } - - existingTransaction, ok := transactionsByPlaidId[plaidTransaction.ID] - if !ok { - transactionsToInsert = append(transactionsToInsert, models.Transaction{ - AccountId: accountId, - BankAccountId: plaidIdsToBankIds[plaidTransaction.AccountID], - PlaidTransactionId: plaidTransaction.ID, - Amount: amount, - SpendingId: nil, - Spending: nil, - Categories: plaidTransaction.Category, - OriginalCategories: plaidTransaction.Category, - Date: date, - AuthorizedDate: authorizedDate, - Name: transactionName, - OriginalName: plaidTransaction.Name, - MerchantName: plaidTransaction.MerchantName, - OriginalMerchantName: plaidTransaction.MerchantName, - IsPending: plaidTransaction.Pending, - CreatedAt: now, - PendingPlaidTransactionId: pendingPlaidTransactionId, - }) - continue - } - - var shouldUpdate bool - if existingTransaction.Amount != amount { - shouldUpdate = true - } - - if existingTransaction.IsPending != plaidTransaction.Pending { - shouldUpdate = true - } - - if !myownsanity.TimesPEqual(existingTransaction.AuthorizedDate, authorizedDate) { - shouldUpdate = true - } - - if existingTransaction.PendingPlaidTransactionId != pendingPlaidTransactionId { - shouldUpdate = true - } - - existingTransaction.Amount = amount - existingTransaction.IsPending = plaidTransaction.Pending - existingTransaction.AuthorizedDate = authorizedDate - existingTransaction.PendingPlaidTransactionId = pendingPlaidTransactionId - - // Update old transactions calculated name as we can. - if existingTransaction.Name != transactionName { - existingTransaction.Name = transactionName - shouldUpdate = true - } - - // Fix timezone of records. - if !existingTransaction.Date.Equal(date) { - existingTransaction.Date = date - shouldUpdate = true - } - - if shouldUpdate { - transactionsToUpdate = append(transactionsToUpdate, &existingTransaction) - } - } - - if len(transactionsToUpdate) > 0 { - log.Infof("updating %d transactions", len(transactionsToUpdate)) - if err = repo.UpdateTransactions(span.Context(), transactionsToUpdate); err != nil { - log.WithError(err).Errorf("failed to update transactions for job") - return err - } + transactions, err := platypus.GetAllTransactions(span.Context(), start, end, itemBankAccountIds) + if err != nil { + log.WithError(err).Error("failed to retrieve transactions from plaid") + return errors.Wrap(err, "failed to retrieve transactions from plaid") } - if len(transactionsToInsert) > 0 { - log.Infof("creating %d transactions", len(transactionsToInsert)) - // Reverse the list so the oldest records are inserted first. - for i, j := 0, len(transactionsToInsert)-1; i < j; i, j = i+1, j-1 { - transactionsToInsert[i], transactionsToInsert[j] = transactionsToInsert[j], transactionsToInsert[i] - } - if err = repo.InsertTransactions(span.Context(), transactionsToInsert); err != nil { - log.WithError(err).Error("failed to insert new transactions") - return err - } + if err = j.upsertTransactions( + span.Context(), + log, + repo, + link, + plaidIdsToBankIds, + transactions, + ); err != nil { + log.WithError(err).Error("failed to upsert transactions from plaid") + return err } link.LastSuccessfulUpdate = myownsanity.TimeP(time.Now().UTC()) diff --git a/pkg/jobs/transactions.go b/pkg/jobs/transactions.go new file mode 100644 index 00000000..e5ff04c6 --- /dev/null +++ b/pkg/jobs/transactions.go @@ -0,0 +1,143 @@ +package jobs + +import ( + "context" + "github.com/getsentry/sentry-go" + "github.com/monetr/rest-api/pkg/internal/platypus" + "github.com/monetr/rest-api/pkg/models" + "github.com/monetr/rest-api/pkg/repository" + "github.com/sirupsen/logrus" + "time" +) + +func (j *jobManagerBase) upsertTransactions( + ctx context.Context, + log *logrus.Entry, + repo repository.BaseRepository, + link *models.Link, + plaidIdsToBankIds map[string]uint64, + plaidTransactions []platypus.Transaction, +) error { + span := sentry.StartSpan(ctx, "Job - Upsert Transactions") + defer span.Finish() + + account, err := repo.GetAccount(span.Context()) + if err != nil { + log.WithError(err).Error("failed to retrieve account for job") + return err + } + + timezone, err := account.GetTimezone() + if err != nil { + log.WithError(err).Warn("failed to get account's time zone, defaulting to UTC") + timezone = time.UTC + } + + plaidTransactionIds := make([]string, len(plaidTransactions)) + for i, transaction := range plaidTransactions { + plaidTransactionIds[i] = transaction.GetTransactionId() + } + + transactionsByPlaidId, err := repo.GetTransactionsByPlaidId(span.Context(), link.LinkId, plaidTransactionIds) + if err != nil { + log.WithError(err).Error("failed to retrieve transaction ids for updating plaid transactions") + return err + } + + transactionsToUpdate := make([]*models.Transaction, 0) + transactionsToInsert := make([]models.Transaction, 0) + now := time.Now().UTC() + for _, plaidTransaction := range plaidTransactions { + amount := plaidTransaction.GetAmount() + + date := plaidTransaction.GetDateLocal(timezone) + + transactionName := plaidTransaction.GetName() + + // We only want to make the transaction name be the merchant name if the merchant name is shorter. This is + // due to something I observed with a dominos transaction, where the merchant was improperly parsed and the + // transaction ended up being called `Mnuslindstrom` rather than `Domino's`. This should fix that problem. + if plaidTransaction.GetMerchantName() != "" && len(plaidTransaction.GetMerchantName()) < len(transactionName) { + transactionName = plaidTransaction.GetMerchantName() + } + + existingTransaction, ok := transactionsByPlaidId[plaidTransaction.GetTransactionId()] + if !ok { + transactionsToInsert = append(transactionsToInsert, models.Transaction{ + AccountId: repo.AccountId(), + BankAccountId: plaidIdsToBankIds[plaidTransaction.GetBankAccountId()], + PlaidTransactionId: plaidTransaction.GetTransactionId(), + Amount: amount, + SpendingId: nil, + Spending: nil, + Categories: plaidTransaction.GetCategory(), + OriginalCategories: plaidTransaction.GetCategory(), + Date: date, + Name: transactionName, + OriginalName: plaidTransaction.GetName(), + MerchantName: plaidTransaction.GetMerchantName(), + OriginalMerchantName: plaidTransaction.GetMerchantName(), + IsPending: plaidTransaction.GetIsPending(), + CreatedAt: now, + PendingPlaidTransactionId: plaidTransaction.GetPendingTransactionId(), + }) + continue + } + + var shouldUpdate bool + if existingTransaction.Amount != amount { + shouldUpdate = true + } + + if existingTransaction.IsPending != plaidTransaction.GetIsPending() { + shouldUpdate = true + } + + // This won't work quite right, I need to compare the values, not the pointers. + if existingTransaction.PendingPlaidTransactionId != plaidTransaction.GetPendingTransactionId() { + shouldUpdate = true + } + + existingTransaction.Amount = amount + existingTransaction.IsPending = plaidTransaction.GetIsPending() + existingTransaction.PendingPlaidTransactionId = plaidTransaction.GetPendingTransactionId() + + // Update old transactions calculated name as we can. + if existingTransaction.Name != transactionName { + existingTransaction.Name = transactionName + shouldUpdate = true + } + + // Fix timezone of records. + if !existingTransaction.Date.Equal(date) { + existingTransaction.Date = date + shouldUpdate = true + } + + if shouldUpdate { + transactionsToUpdate = append(transactionsToUpdate, &existingTransaction) + } + } + + if len(transactionsToUpdate) > 0 { + log.Infof("updating %d transactions", len(transactionsToUpdate)) + if err = repo.UpdateTransactions(span.Context(), transactionsToUpdate); err != nil { + log.WithError(err).Errorf("failed to update transactions for job") + return err + } + } + + if len(transactionsToInsert) > 0 { + log.Infof("creating %d transactions", len(transactionsToInsert)) + // Reverse the list so the oldest records are inserted first. + for i, j := 0, len(transactionsToInsert)-1; i < j; i, j = i+1, j-1 { + transactionsToInsert[i], transactionsToInsert[j] = transactionsToInsert[j], transactionsToInsert[i] + } + if err = repo.InsertTransactions(span.Context(), transactionsToInsert); err != nil { + log.WithError(err).Error("failed to insert new transactions") + return err + } + } + + return nil +} diff --git a/pkg/jobs/update_institutions.go b/pkg/jobs/update_institutions.go deleted file mode 100644 index 311f70c0..00000000 --- a/pkg/jobs/update_institutions.go +++ /dev/null @@ -1,104 +0,0 @@ -package jobs - -import ( - "context" - "github.com/getsentry/sentry-go" - "github.com/gocraft/work" - "github.com/monetr/rest-api/pkg/models" - "github.com/monetr/rest-api/pkg/repository" - "github.com/plaid/plaid-go/plaid" -) - -const ( - UpdateInstitutions = "UpdateInstitutions" -) - -func (j *jobManagerBase) updateInstitutions(job *work.Job) error { - span := sentry.StartSpan(context.Background(), "Job", sentry.TransactionName("Update Institutions")) - defer span.Finish() - - log := j.getLogForJob(job) - - institutions, err := j.plaidClient.GetAllInstitutions(span.Context(), []string{"US"}, plaid.GetInstitutionsOptions{ - Products: []string{ - "transactions", - }, - IncludeOptionalMetadata: true, - }) - if err != nil { - log.WithError(err).Error("failed to retrieve institutions for update") - return err - } - - if len(institutions) == 0 { - log.Warn("no institutions found") - return nil - } - - plaidInstitutionIds := make([]string, len(institutions)) - for i, institution := range institutions { - plaidInstitutionIds[i] = institution.ID - } - - return j.getJobHelperRepository(job, func(repo repository.JobRepository) error { - byPlaidId, err := repo.GetInstitutionsByPlaidID(span.Context(), plaidInstitutionIds) - if err != nil { - return err - } - - institutionsToUpdate := make([]*models.Institution, 0) - institutionsToCreate := make([]*models.Institution, 0) - for _, institution := range institutions { - existingInstitution, ok := byPlaidId[institution.ID] - if !ok { - var url, color, logo *string - if institution.URL != "" { - url = &institution.URL - } - - if institution.PrimaryColor != "" { - color = &institution.PrimaryColor - } - - if institution.Logo != "" { - logo = &institution.Logo - } - - institutionsToCreate = append(institutionsToCreate, &models.Institution{ - Name: institution.Name, - PlaidInstitutionId: &institution.ID, - PlaidProducts: institution.Products, - URL: url, - PrimaryColor: color, - Logo: logo, - }) - } - - if existingInstitution.Name != institution.Name { - existingInstitution.Name = institution.Name - institutionsToUpdate = append(institutionsToUpdate, &existingInstitution) - } - } - - if len(institutionsToUpdate) == 0 && len(institutionsToCreate) == 0 { - log.Infof("no institution changes") - return nil - } - - if len(institutionsToUpdate) > 0 { - log.Infof("updating %d institutions", len(institutionsToUpdate)) - if err = repo.UpdateInstitutions(span.Context(), institutionsToUpdate); err != nil { - return err - } - } - - if len(institutionsToCreate) > 0 { - log.Infof("creating %d institutions", len(institutionsToCreate)) - if err = repo.CreateInstitutions(span.Context(), institutionsToCreate); err != nil { - return err - } - } - - return nil - }) -} diff --git a/pkg/logging/pg.go b/pkg/logging/pg.go index 12c81694..d30c627c 100644 --- a/pkg/logging/pg.go +++ b/pkg/logging/pg.go @@ -88,7 +88,7 @@ func (h *PostgresHooks) AfterQuery(ctx context.Context, event *pg.QueryEvent) er Category: "postgres", Message: queryString, Data: map[string]interface{}{ - "queryTime": queryTime, + "queryTime": queryTime.String(), }, Level: "debug", Timestamp: event.StartTime, @@ -99,7 +99,7 @@ func (h *PostgresHooks) AfterQuery(ctx context.Context, event *pg.QueryEvent) er Category: "postgres", Message: queryString, Data: map[string]interface{}{ - "queryTime": queryTime, + "queryTime": queryTime.String(), "error": event.Err.Error(), }, Level: "error", diff --git a/pkg/repository/plaid_link.go b/pkg/repository/plaid_link.go index d50ae96c..45fa9697 100644 --- a/pkg/repository/plaid_link.go +++ b/pkg/repository/plaid_link.go @@ -5,6 +5,7 @@ package repository import ( "context" "github.com/getsentry/sentry-go" + "github.com/go-pg/pg/v10" "github.com/monetr/rest-api/pkg/models" "github.com/pkg/errors" ) @@ -24,4 +25,65 @@ func (r *repositoryBase) UpdatePlaidLink(ctx context.Context, link *models.Plaid span.SetTag("accountId", r.AccountIdStr()) _, err := r.txn.ModelContext(span.Context(), link).WherePK().Update(link) return errors.Wrap(err, "failed to update Plaid link") -} \ No newline at end of file +} + +type PlaidRepository interface { + GetLinkByItemId(ctx context.Context, itemId string) (*models.Link, error) + GetLink(ctx context.Context, accountId, linkId uint64) (*models.Link, error) +} + +func NewPlaidRepository(db pg.DBI) PlaidRepository { + return &plaidRepositoryBase{ + txn: db, + } +} + +type plaidRepositoryBase struct { + txn pg.DBI +} + +func (r *plaidRepositoryBase) GetLinkByItemId(ctx context.Context, itemId string) (*models.Link, error) { + span := sentry.StartSpan(ctx, "GetLinkByItemId") + defer span.Finish() + + span.Data = map[string]interface{}{ + "itemId": itemId, + } + + var link models.Link + err := r.txn.ModelContext(span.Context(), &link). + Relation("PlaidLink"). + Relation("BankAccounts"). + Where(`"plaid_link"."item_id" = ?`, itemId). + Limit(1). + Select(&link) + if err != nil { + return nil, errors.Wrap(err, "failed to retrieve link by item Id") + } + + return &link, nil +} + +func (r *plaidRepositoryBase) GetLink(ctx context.Context, accountId, linkId uint64) (*models.Link, error) { + span := sentry.StartSpan(ctx, "GetLink") + defer span.Finish() + + span.Data = map[string]interface{}{ + "accountId": accountId, + "linkId": linkId, + } + + var link models.Link + err := r.txn.ModelContext(span.Context(), &link). + Relation("PlaidLink"). + Relation("BankAccounts"). + Where(`"link"."account_id" = ?`, accountId). + Where(`"link"."link_id" = ?`, linkId). + Limit(1). + Select(&link) + if err != nil { + return nil, errors.Wrap(err, "failed to retrieve link") + } + + return &link, nil +} diff --git a/pkg/repository/plaid_link_test.go b/pkg/repository/plaid_link_test.go new file mode 100644 index 00000000..86246695 --- /dev/null +++ b/pkg/repository/plaid_link_test.go @@ -0,0 +1,111 @@ +package repository + +import ( + "context" + "github.com/brianvoe/gofakeit/v6" + "github.com/monetr/rest-api/pkg/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestPlaidRepositoryBase_GetLink(t *testing.T) { + repo := GetTestAuthenticatedRepository(t) + txn := repo.(*repositoryBase).txn + require.NotNil(t, txn, "must be able to pull the transaction for test") + + plaidLink := &models.PlaidLink{ + ItemId: gofakeit.UUID(), + Products: []string{ + "transactions", + }, + WebhookUrl: "https://monetr.test/webhook", + } + + link := &models.Link{ + AccountId: repo.AccountId(), + LinkType: models.PlaidLinkType, + PlaidLinkId: nil, + LinkStatus: models.LinkStatusSetup, + InstitutionName: "Institution " + t.Name(), + CustomInstitutionName: "Institution " + t.Name(), + CreatedAt: time.Now(), + CreatedByUserId: repo.UserId(), + UpdatedAt: time.Now(), + UpdatedByUserId: nil, + LastSuccessfulUpdate: nil, + } + + { // Create the links. + require.NoError(t, repo.CreatePlaidLink(context.Background(), plaidLink), "must create plaid link") + link.PlaidLinkId = &plaidLink.PlaidLinkID + require.NoError(t, repo.CreateLink(context.Background(), link), "must create link") + } + + plaidRepo := NewPlaidRepository(txn) + + t.Run("simple", func(t *testing.T) { + readLink, err := plaidRepo.GetLink(context.Background(), link.AccountId, link.LinkId) + assert.NoError(t, err, "failed to retrieve link") + assert.NotNil(t, readLink.PlaidLink, "must include plaid link child") + assert.EqualValues(t, link.LinkId, readLink.LinkId, "link Id must match") + assert.EqualValues(t, plaidLink.PlaidLinkID, readLink.PlaidLink.PlaidLinkID, "plaid link Id must match") + }) + + t.Run("not found", func(t *testing.T) { + readLink, err := plaidRepo.GetLink(context.Background(), link.AccountId, link.LinkId + 100) + assert.EqualError(t, err, "failed to retrieve link: pg: no rows in result set") + assert.Nil(t, readLink, "link must be nil") + }) +} + +func TestPlaidRepositoryBase_GetLinkByItemId(t *testing.T) { + repo := GetTestAuthenticatedRepository(t) + txn := repo.(*repositoryBase).txn + require.NotNil(t, txn, "must be able to pull the transaction for test") + + plaidLink := &models.PlaidLink{ + ItemId: gofakeit.UUID(), + Products: []string{ + "transactions", + }, + WebhookUrl: "https://monetr.test/webhook", + } + + link := &models.Link{ + AccountId: repo.AccountId(), + LinkType: models.PlaidLinkType, + PlaidLinkId: nil, + LinkStatus: models.LinkStatusSetup, + InstitutionName: "Institution " + t.Name(), + CustomInstitutionName: "Institution " + t.Name(), + CreatedAt: time.Now(), + CreatedByUserId: repo.UserId(), + UpdatedAt: time.Now(), + UpdatedByUserId: nil, + LastSuccessfulUpdate: nil, + } + + { // Create the links. + require.NoError(t, repo.CreatePlaidLink(context.Background(), plaidLink), "must create plaid link") + link.PlaidLinkId = &plaidLink.PlaidLinkID + require.NoError(t, repo.CreateLink(context.Background(), link), "must create link") + } + + plaidRepo := NewPlaidRepository(txn) + + t.Run("simple", func(t *testing.T) { + readLink, err := plaidRepo.GetLinkByItemId(context.Background(), plaidLink.ItemId) + assert.NoError(t, err, "failed to retrieve link") + assert.NotNil(t, readLink.PlaidLink, "must include plaid link child") + assert.EqualValues(t, link.LinkId, readLink.LinkId, "link Id must match") + assert.EqualValues(t, plaidLink.PlaidLinkID, readLink.PlaidLink.PlaidLinkID, "plaid link Id must match") + }) + + t.Run("not found", func(t *testing.T) { + readLink, err := plaidRepo.GetLinkByItemId(context.Background(), "not a real item id") + assert.EqualError(t, err, "failed to retrieve link by item Id: pg: no rows in result set") + assert.Nil(t, readLink, "link must be nil") + }) +} diff --git a/templates/api-config.yaml b/templates/api-config.yaml index 89a1be0f..ff00b1bb 100644 --- a/templates/api-config.yaml +++ b/templates/api-config.yaml @@ -19,6 +19,7 @@ data: MONETR_PLAID_RETURNING_EXPERIENCE: {{ quote .Values.api.plaid.enableReturningUserExperience }} MONETR_PLAID_WEBHOOKS_ENABLED: {{ quote .Values.api.plaid.webhooksEnabled }} MONETR_PLAID_WEBHOOKS_DOMAIN: {{ quote .Values.api.plaid.webhooksDomain }} + MONETR_PLAID_OAUTH_DOMAIN: {{ quote .Values.api.plaid.oauthDomain }} MONETR_PG_ADDRESS: {{ quote .Values.api.postgreSql.address }} MONETR_PG_PORT: {{ quote .Values.api.postgreSql.port }} MONETR_PG_DATABASE: {{ quote .Values.api.postgreSql.database }} diff --git a/tests/pg/foreign_keys.sql b/tests/pg/foreign_keys.sql new file mode 100644 index 00000000..3c38d08c --- /dev/null +++ b/tests/pg/foreign_keys.sql @@ -0,0 +1,9 @@ +BEGIN; +SELECT plan(2); + +SELECT has_fk('transactions'); -- Make sure the transactions table has a foreign key. +SELECT col_is_fk('transactions', 'account_id'); -- Account ID should always be a foreign key. + +SELECT * +FROM finish(); +ROLLBACK; diff --git a/tests/pg/indexes.sql b/tests/pg/indexes.sql new file mode 100644 index 00000000..52df9c7b --- /dev/null +++ b/tests/pg/indexes.sql @@ -0,0 +1,12 @@ +BEGIN; +SELECT plan(2); + +-- Make sure that the index we specify does get renamed. I don't want to change the migration script as it's old. So I +-- want to make sure that it is properly applied. If this fails then that means the PostgreSQL behavior has changed in +-- whatever version that is being used. And that this index (when applied from scratch) is no longer being renamed. +SELECT hasnt_index('plaid_links', 'ix_uq_plaid_links_item_id'); +SELECT has_index('plaid_links', 'uq_plaid_links_item_id'); + +SELECT * +FROM finish(); +ROLLBACK; diff --git a/values.acceptance.yaml b/values.acceptance.yaml index 88950687..c5f40f1b 100644 --- a/values.acceptance.yaml +++ b/values.acceptance.yaml @@ -129,7 +129,8 @@ api: environment: "https://development.plaid.com" enableReturningUserExperience: true webhooksEnabled: true - webhooksDomain: https://api.acceptance.monetr.dev + webhooksDomain: api.acceptance.monetr.dev + oauthDomain: app.acceptance.monetr.dev cors: allowedOrigins: - "https://acceptance.monetr.dev" diff --git a/values.dog.yaml b/values.dog.yaml index ca218613..794ff397 100644 --- a/values.dog.yaml +++ b/values.dog.yaml @@ -5,7 +5,7 @@ image: pullPolicy: Always tag: "" # Will be overwritten with the SHA for the commit of this deploy -imagePullSecrets: [] +imagePullSecrets: [ ] podAnnotations: monetr.dev/branch: "" # Branch of the deployment will be put here @@ -108,7 +108,7 @@ api: beta: enableBetaCodes: true postgreSql: - address: newfoundland.monetr-acceptance.svc.cluster.local + address: postgres.google.acceptance.monetr.in port: 5432 database: monetr smtp: @@ -127,14 +127,15 @@ api: environment: "https://development.plaid.com" enableReturningUserExperience: true webhooksEnabled: true - webhooksDomain: https://api.monetr.dog + webhooksDomain: api.monetr.dog + oauthDomain: ui.monetr.dog cors: allowedOrigins: - "https://monetr.dog" debug: false redis: enabled: true - address: redis.monetr-acceptance.svc.cluster.local + address: redis.redis.kluster.monetr.in port: 6379 logging: level: debug diff --git a/values.staging.yaml b/values.staging.yaml index 0fc0420e..b466db1f 100644 --- a/values.staging.yaml +++ b/values.staging.yaml @@ -127,7 +127,8 @@ api: environment: "https://sandbox.plaid.com" enableReturningUserExperience: true webhooksEnabled: true - webhooksDomain: https://api.staging.monetr.dev + webhooksDomain: api.staging.monetr.dev + oauthDomain: app.staging.monetr.dev cors: allowedOrigins: - "https://app.staging.monetr.dev" diff --git a/values.yaml b/values.yaml index e22c5075..91addf44 100644 --- a/values.yaml +++ b/values.yaml @@ -135,6 +135,7 @@ api: enableReturningUserExperience: false webhooksEnabled: false webhooksDomain: "" + oauthDomain: "" cors: allowedOrigins: - "*"