diff --git a/Makefile.local b/Makefile.local index cc038775..e42ed42b 100644 --- a/Makefile.local +++ b/Makefile.local @@ -94,6 +94,7 @@ start-minikube: --delete-on-failure \ --driver=$(MINIKUBE_DRIVER) \ --kubernetes-version=$(KUBERNETES_VERSION) \ + --cni=cilium \ --embed-certs setup-etc-hosts: @@ -272,44 +273,12 @@ deploy-redis: USE_PGO_OPERATOR ?=false ifeq ($(USE_PGO_OPERATOR),true) -POSTGRES_OPERATOR_VERSION ?=v4.7.0 -POSTGRES_OPERATOR_CLIENT_SETUP=$(LOCAL_TMP)/pgo_client_setup.sh -POSTGRES_OPERATOR_BASE_URL=https://raw.githubusercontent.com/CrunchyData/postgres-operator/$(POSTGRES_OPERATOR_VERSION) - -POSTGRES_OPERATOR_INSTALLER_YAML=$(LOCAL_TMP)/postgres-operator.yaml -$(POSTGRES_OPERATOR_INSTALLER_YAML): $(LOCAL_TMP) - $(call infoMsg,Downloading Crunchy Data PostgreSQL operator installer yaml) - curl $(POSTGRES_OPERATOR_BASE_URL)/installers/kubectl/postgres-operator.yml > $(POSTGRES_OPERATOR_INSTALLER_YAML) - sed -i -e 's/namespace: "pgo"/namespace: "$(MINIKUBE_NAMESPACE)"/g' $(POSTGRES_OPERATOR_INSTALLER_YAML) - sed -i -e 's/namespace: pgo/namespace: "$(MINIKUBE_NAMESPACE)"/g' $(POSTGRES_OPERATOR_INSTALLER_YAML) - $(KUBECTL) apply -f $(POSTGRES_OPERATOR_INSTALLER_YAML) -n $(MINIKUBE_NAMESPACE) - $(KUBECTL) wait --for=condition=complete job/pgo-deploy -n $(MINIKUBE_NAMESPACE) --timeout=180s - $(KUBECTL) delete -f $(POSTGRES_OPERATOR_INSTALLER_YAML) -n $(MINIKUBE_NAMESPACE) - -ifeq ($(OS),linux) -PGO_BIN_NAME="pgo" -else ifeq ($(OS),darwin) -PGO_BIN_NAME="pgo-mac" -else -$(error Operating system [$(OS)] is not supported by pgo) -endif - -define kubectl_get_private -touch "$1" && chmod a-rwx,u+rw "$1" && $(KUBECTL) get > "$1" "${@:2}" -endef +deploy-pgo: + $(KUBECTL) apply -f $(PWD)/production/operator/crunchy.yaml -n $(PROD_NAMESPACE) + $(KUBECTL) wait --for=condition=complete job/pgo-deploy --timeout=180s -n $(PROD_NAMESPACE) + $(KUBECTL) delete -f $(PWD)/production/operator/crunchy.yaml -n $(PROD_NAMESPACE) -PGO_DIR=$(LOCAL_TMP)/.pgo -$(POSTGRES_OPERATOR_CLIENT_SETUP): $(LOCAL_TMP) - $(call kubectl_get_private "$(PGO_DIR)/pgouser" secret -n $(MINIKUBE_NAMESPACE) "pgouser-admin" -o 'go-template={{ .data.username | base64decode }}:{{ .data.password | base64decode }}') - $(call kubectl_get_private "$(PGO_DIR)/client.crt" secret -n $(MINIKUBE_NAMESPACE) pgo.tls -o 'go-template={{ index .data "tls.crt" | base64decode }}') - $(call kubectl_get_private "$(PGO_DIR)/client.key" secret -n $(MINIKUBE_NAMESPACE) pgo.tls -o 'go-template={{ index .data "tls.key" | base64decode }}') - -POSTGRES_OPERATOR_BIN_DOWNLOAD=https://github.com/CrunchyData/postgres-operator/releases/download/$(POSTGRES_OPERATOR_VERSION)/$(PGO_BIN_NAME) -$(PGO): $(LOCAL_BIN) - $(call infoMsg,Downloading Crunchy Data PostgreSQL operator client) - curl -Lo $(PGO) $(POSTGRES_OPERATOR_BIN_DOWNLOAD) - -deploy-postgres: $(LOCAL_TMP) $(POSTGRES_OPERATOR_INSTALLER_YAML) +deploy-postgres: $(call infoMsg,Deploying Crunchy Data PostgreSQL to the minikube cluster) $(KUBECTL) apply -f $(PWD)/minikube/crunchy-postgres.yaml -n $(MINIKUBE_NAMESPACE) diff --git a/pkg/cache/transactions.go b/pkg/cache/transactions.go new file mode 100644 index 00000000..4470f63d --- /dev/null +++ b/pkg/cache/transactions.go @@ -0,0 +1,59 @@ +package cache + +import ( + "context" + "fmt" + "github.com/getsentry/sentry-go" + "github.com/monetrapp/rest-api/pkg/models" + "github.com/sirupsen/logrus" + "time" +) + +type RemovedTransactionsCache interface { + CacheDeletedTransaction(ctx context.Context, transaction models.Transaction) bool + LookupDeletedTransaction(ctx context.Context, transactionId string) (*models.Transaction, bool) + Close() error +} + +var ( + _ RemovedTransactionsCache = &noopRemovedTransactionsCache{} + _ RemovedTransactionsCache = &redisRemovedTransactionsCache{} +) + +type noopRemovedTransactionsCache struct{} + +func (n noopRemovedTransactionsCache) CacheDeletedTransaction(ctx context.Context, transaction models.Transaction) bool { + return false +} + +func (n noopRemovedTransactionsCache) LookupDeletedTransaction(ctx context.Context, transactionId string) (*models.Transaction, bool) { + return nil, false +} + +func (n noopRemovedTransactionsCache) Close() error { + return nil +} + +type redisRemovedTransactionsCache struct { + log *logrus.Entry + client Cache +} + +func (r *redisRemovedTransactionsCache) CacheDeletedTransaction(ctx context.Context, transaction models.Transaction) bool { + span := sentry.StartSpan(ctx, "CacheDeletedTransaction") + defer span.Finish() + + return r.client.SetEzTTL(span.Context(), r.getKey(transaction), transaction, 30*time.Minute) == nil +} + +func (r *redisRemovedTransactionsCache) LookupDeletedTransaction(ctx context.Context, transactionId string) (*models.Transaction, bool) { + panic("implement me") +} + +func (r *redisRemovedTransactionsCache) Close() error { + panic("implement me") +} + +func (r *redisRemovedTransactionsCache) getKey(transaction models.Transaction) string { + return fmt.Sprintf("plaid:deleted_transactions:%d:%s", transaction.AccountId, transaction.PlaidTransactionId) +} diff --git a/pkg/cache/wrapper.go b/pkg/cache/wrapper.go new file mode 100644 index 00000000..bc72526a --- /dev/null +++ b/pkg/cache/wrapper.go @@ -0,0 +1,73 @@ +package cache + +import ( + "context" + "encoding/json" + "github.com/getsentry/sentry-go" + "github.com/gomodule/redigo/redis" + "github.com/pkg/errors" + "time" +) + +type Cache interface { + Set(ctx context.Context, key string, value []byte) error + SetTTL(ctx context.Context, key string, value []byte, lifetime time.Duration) error + SetEz(ctx context.Context, key string, object interface{}) error + SetEzTTL(ctx context.Context, key string, object interface{}, lifetime time.Duration) error + Get(ctx context.Context, key string) ([]byte, error) + GetEz(ctx context.Context, key string, output interface{}) error + Delete(ctx context.Context, key string) error + Close() error +} + +var ( + _ Cache = &redisCache{} +) + +type redisCache struct { + client redis.Conn +} + +func (r *redisCache) Set(ctx context.Context, key string, value []byte) error { + span := sentry.StartSpan(ctx, "Redis - Set") + defer span.Finish() + return errors.Wrap(r.client.Send("SET", key, value), "failed to store item in cache") +} + +func (r *redisCache) SetTTL(ctx context.Context, key string, value []byte, lifetime time.Duration) error { + span := sentry.StartSpan(ctx, "Redis - SetTTL") + defer span.Finish() + return errors.Wrap(r.client.Send("SET", key, value, "EXAT", time.Now().Add(lifetime)), "failed to store item in cache") +} + +func (r *redisCache) SetEz(ctx context.Context, key string, object interface{}) error { + span := sentry.StartSpan(ctx, "Redis - SetEz") + defer span.Finish() + + data, err := json.Marshal(object) + if err != nil { + return errors.Wrap(err, "failed to marshal item to be cached") + } + + return r.Set(span.Context(), key, data) +} + +func (r *redisCache) SetEzTTL(ctx context.Context, key string, object interface{}, lifetime time.Duration) error { + panic("implement me") +} + +func (r *redisCache) Get(ctx context.Context, key string) ([]byte, error) { + panic("implement me") +} + +func (r *redisCache) GetEz(ctx context.Context, key string, output interface{}) error { + panic("implement me") +} + +func (r *redisCache) Delete(ctx context.Context, key string) error { + panic("implement me") +} + +func (r *redisCache) Close() error { + panic("implement me") +} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index 0cac1323..2c8760b8 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -46,6 +46,7 @@ type Repository interface { GetTransactions(bankAccountId uint64, limit, offset int) ([]models.Transaction, error) GetTransactionsByPlaidId(linkId uint64, plaidTransactionIds []string) (map[string]models.Transaction, error) GetTransactionsByPlaidTransactionId(ctx context.Context, linkId uint64, plaidTransactionIds []string) ([]models.Transaction, error) + GetTransactionsForSpending(ctx context.Context, bankAccountId, spendingId uint64, limit, offset int) ([]models.Transaction, error) InsertTransactions(ctx context.Context, transactions []models.Transaction) error ProcessTransactionSpentFrom(ctx context.Context, bankAccountId uint64, input, existing *models.Transaction) (updatedExpenses []models.Spending, _ error) UpdateBankAccounts(accounts []models.BankAccount) error diff --git a/pkg/repository/transaction.go b/pkg/repository/transaction.go index 6468aeeb..a982e952 100644 --- a/pkg/repository/transaction.go +++ b/pkg/repository/transaction.go @@ -80,6 +80,27 @@ func (r *repositoryBase) GetTransactions(bankAccountId uint64, limit, offset int return items, nil } +func (r *repositoryBase) GetTransactionsForSpending(ctx context.Context, bankAccountId, spendingId uint64, limit, offset int) ([]models.Transaction, error) { + span := sentry.StartSpan(ctx, "GetTransactionsForSpending") + defer span.Finish() + + var items []models.Transaction + err := r.txn.ModelContext(span.Context(), &items). + Where(`"transaction"."account_id" = ?`, r.AccountId()). + Where(`"transaction"."bank_account_id" = ?`, bankAccountId). + Where(`"transaction"."spending_id" = ?`, spendingId). + Limit(limit). + Offset(offset). + Order(`date DESC`). + Order(`transaction_id DESC`). + Select(&items) + if err != nil { + return nil, errors.Wrap(err, "failed to retrieve transactions for spending") + } + + return items, nil +} + func (r *repositoryBase) GetTransaction(bankAccountId, transactionId uint64) (*models.Transaction, error) { var result models.Transaction err := r.txn.Model(&result). diff --git a/values.acceptance.yaml b/values.acceptance.yaml index 24473e49..a942aa48 100644 --- a/values.acceptance.yaml +++ b/values.acceptance.yaml @@ -32,7 +32,7 @@ resources: cpu: 100m memory: 128Mi requests: - cpu: 100m + cpu: 50m memory: 128Mi nodeSelector: diff --git a/values.staging.yaml b/values.staging.yaml index b99b4a7b..910f0d8b 100644 --- a/values.staging.yaml +++ b/values.staging.yaml @@ -32,7 +32,7 @@ resources: cpu: 100m memory: 128Mi requests: - cpu: 100m + cpu: 50m memory: 128Mi nodeSelector: