From 1a734cf9cf23d065a0a3aed9dd5619eb20b31430 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 17 May 2024 11:53:03 -0700 Subject: [PATCH 01/16] chore: sessions add session cronjob to compactor --- aggregator/.gitignore | 1 + aggregator/README.md | 17 +- .../5bfc2a71a71f_create_sessions_table.py | 37 ++++ aggregator/compactor.go | 4 +- aggregator/go.mod | 16 +- aggregator/go.sum | 41 ++-- aggregator/keycloak/tokenManager.go | 179 ++++++++++++++++++ aggregator/model/client_events.go | 26 ++- aggregator/model/client_sessions.go | 131 +++++++++++++ aggregator/models.py | 10 + helm/aggregator/README.md | 4 + 11 files changed, 429 insertions(+), 37 deletions(-) create mode 100644 aggregator/alembic/versions/5bfc2a71a71f_create_sessions_table.py create mode 100644 aggregator/keycloak/tokenManager.go create mode 100644 aggregator/model/client_sessions.go diff --git a/aggregator/.gitignore b/aggregator/.gitignore index c055b22..b0d25a7 100644 --- a/aggregator/.gitignore +++ b/aggregator/.gitignore @@ -2,3 +2,4 @@ config.json build __pycache__ .env +venv diff --git a/aggregator/README.md b/aggregator/README.md index cb5b623..3b8cf5f 100644 --- a/aggregator/README.md +++ b/aggregator/README.md @@ -7,7 +7,10 @@ In order to avoid the custom codebase parsing the requests, it relies on `Grafan ## Compactor -A lightweight Go server running a job scheduling to delete the old aggregated data upserted by the aggregators. +A lightweight Go server running scheduled jobs. There are two cronjobs it controls: + +1. Deleting old client events. +2. Collecting and clearing client session counts. ## Environment Variables @@ -18,6 +21,18 @@ A lightweight Go server running a job scheduling to delete the old aggregated da - `DB_PASSWORD`: the password to be used for password authentication. - `RETENTION_PERIOD`: the duration of time to keep the aggregated data. - please see [Postgres Interval Input](https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-INTERVAL-INPUT) for the unit convention. +- `DEV_KEYCLOAK_URL`: The development keycloak base URL +- `DEV_KEYCLOAK_CLIENT_ID`: The development keycloak client id +- `DEV_KEYCLOAK_USERNAME`: The development keycloak username +- `DEV_KEYCLOAK_PASSWORD`: The development keycloak passowrd +- `TEST_KEYCLOAK_URL`: The test keycloak base URL +- `TEST_KEYCLOAK_CLIENT_ID`: The test keycloak client id +- `TEST_KEYCLOAK_USERNAME`: The test keycloak username +- `TEST_KEYCLOAK_PASSWORD`: The test keycloak passowrd +- `PROD_KEYCLOAK_URL`: The prod keycloak base URL +- `PROD_KEYCLOAK_CLIENT_ID`: The prod keycloak client id +- `PROD_KEYCLOAK_USERNAME`: The prod keycloak username +- `PROD_KEYCLOAK_PASSWORD`: The prod keycloak passowrd ## Local development setup diff --git a/aggregator/alembic/versions/5bfc2a71a71f_create_sessions_table.py b/aggregator/alembic/versions/5bfc2a71a71f_create_sessions_table.py new file mode 100644 index 0000000..8aba4f2 --- /dev/null +++ b/aggregator/alembic/versions/5bfc2a71a71f_create_sessions_table.py @@ -0,0 +1,37 @@ +"""create sessions table + +Revision ID: 5bfc2a71a71f +Revises: 3999a4f6f9c0 +Create Date: 2024-05-15 17:05:50.453140 + +""" +from alembic import op +import sqlalchemy as sa +from datetime import datetime + +# revision identifiers, used by Alembic. +revision = '5bfc2a71a71f' +down_revision = '3999a4f6f9c0' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('client_sessions', + sa.Column('environment', sa.String(length=255), nullable=False), + sa.Column('realm_id', sa.String(length=255), nullable=False), + sa.Column('client_id', sa.String(length=255), nullable=False), + sa.Column('active_sessions', sa.Integer(), nullable=False), + sa.Column('offline_sessions', sa.Integer(), nullable=False), + sa.Column('date', sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.func.current_timestamp()), + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('client_sessions') + # ### end Alembic commands ### diff --git a/aggregator/compactor.go b/aggregator/compactor.go index 6defbc1..87036b7 100644 --- a/aggregator/compactor.go +++ b/aggregator/compactor.go @@ -9,5 +9,7 @@ import ( func main() { log.Printf("cronjob starts...") - model.RunCronJob() + model.RunEventsJob() + model.RunSessionsJob() + select {} } diff --git a/aggregator/go.mod b/aggregator/go.mod index 3b9ca36..89a08ed 100644 --- a/aggregator/go.mod +++ b/aggregator/go.mod @@ -5,6 +5,7 @@ go 1.21.0 require ( github.com/go-co-op/gocron v1.18.0 github.com/go-pg/pg v8.0.7+incompatible + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/gorilla/mux v1.8.0 github.com/grafana/dskit v0.0.0-20221212120341-3e308a49441b github.com/grafana/loki v1.6.2-0.20221216202714-209b281593b2 @@ -50,6 +51,7 @@ require ( github.com/go-openapi/swag v0.21.1 // indirect github.com/go-openapi/validate v0.21.0 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect + github.com/go-resty/resty/v2 v2.7.0 // indirect github.com/gogo/googleapis v1.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gogo/status v1.1.1 // indirect @@ -113,7 +115,7 @@ require ( github.com/sony/gobreaker v0.5.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/stretchr/testify v1.8.1 // indirect + github.com/stretchr/testify v1.8.2 // indirect github.com/thanos-io/thanos v0.28.0 // indirect github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect @@ -133,16 +135,16 @@ require ( go.uber.org/zap v1.21.0 // indirect go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect - golang.org/x/crypto v0.4.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.3.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.1.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.2.0 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect google.golang.org/grpc v1.50.1 // indirect diff --git a/aggregator/go.sum b/aggregator/go.sum index 62e0a49..9d35d5a 100644 --- a/aggregator/go.sum +++ b/aggregator/go.sum @@ -274,8 +274,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhYRFk24TvhTZWU0q8lfCojxZQFi3Ou7+uY= -github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= @@ -322,6 +322,8 @@ github.com/gogo/status v1.1.1 h1:DuHXlSFHNKqTQ+/ACf5Vs6r4X/dH2EgIzR9Vr+H65kg= github.com/gogo/status v1.1.1/go.mod h1:jpG3dM5QPcqu19Hg8lkUhBFBa3TcLs1DG7+2Jqci7oU= github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -749,8 +751,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/thanos-io/thanos v0.28.0 h1:g0LByBE0ANA30/t/a2C/mceYhO3VtIPQFoxCsqrYM9I= github.com/thanos-io/thanos v0.28.0/go.mod h1:pqjpOBxOCME9Yn1QztV8bP9C4rkhWvWtyyavdBZ8lDk= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= @@ -837,8 +839,6 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE= go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA= go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= -go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4= -go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 h1:lGdhQUN/cnWdSH3291CUuxSEqc+AsGTiDxPP3r2J0l4= go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -855,8 +855,8 @@ golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -900,8 +900,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -946,6 +946,7 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -956,8 +957,8 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1079,13 +1080,13 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1095,8 +1096,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1167,8 +1168,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/aggregator/keycloak/tokenManager.go b/aggregator/keycloak/tokenManager.go new file mode 100644 index 0000000..f9dedb1 --- /dev/null +++ b/aggregator/keycloak/tokenManager.go @@ -0,0 +1,179 @@ +package keycloak + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "strings" + + // "io/ioutil" + // "log" + "errors" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +type TokenManager struct { + token string + refreshToken string + password string + username string + BaseUrl string + clientId string +} + +func NewTokenManager(clientId string, password string, username string, baseUrl string) *TokenManager { + return &TokenManager{ + clientId: clientId, + password: password, + username: username, + BaseUrl: baseUrl, + } +} + +/* +Method to make a post request for a new token or to refresh a token. Pass in the relevant data for the request type, +e.g grant_type=password/client_credentials for a new token, grant_type=refresh_token for a refresh token. +*/ +func (tm *TokenManager) getTokens(data url.Values) (string, error) { + req, err := http.NewRequest("POST", tm.BaseUrl+"/realms/master/protocol/openid-connect/token", strings.NewReader(data.Encode())) + + if err != nil { + log.Fatalf("Error occurred creating request: %v", err) + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + client := &http.Client{} + resp, err := client.Do(req) + + if err != nil { + log.Fatalf("Error occurred sending request to API endpoint: %v", err) + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + log.Fatalf("Error reading response body: %v", err) + } + + type TokenResponse struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int `json:"expires_in"` + } + var tokenResponse TokenResponse + + if err := json.Unmarshal(body, &tokenResponse); err != nil { + log.Fatalf("Error parsing JSON response: %v", err) + } + + tm.token = tokenResponse.AccessToken + tm.refreshToken = tokenResponse.RefreshToken + return tm.token, nil +} + +// Method to get the token, refreshing if necessary +func (tm *TokenManager) getToken() (string, error) { + // Fetch a new token if not set yet + if tm.token == "" { + formData := url.Values{ + "grant_type": {"password"}, + "client_id": {tm.clientId}, + "password": {tm.password}, + "username": {tm.username}, + } + + token, err := tm.getTokens(formData) + return token, err + } + + // Check expiry and refresh if necessary + expired, err := tm.IsTokenExpired() + if err != nil { + return "", err + } + + if !expired { + return tm.token, nil + } + + err = tm.refreshAccessToken() + if err != nil { + return "", err + } + + return tm.token, nil +} + +// Checks if access token is expired +func (tm *TokenManager) IsTokenExpired() (bool, error) { + if tm.token == "" { + return true, nil + } + + token, _, err := new(jwt.Parser).ParseUnverified(tm.token, jwt.MapClaims{}) + if err != nil { + log.Fatalf("Error parsing token: %v", err) + } + + if claims, ok := token.Claims.(jwt.MapClaims); ok { + if exp, ok := claims["exp"].(float64); ok { + now := time.Now().UTC() + expire := time.Unix(int64(exp), 0).UTC() + return expire.Before(now), nil + } else { + fmt.Println("exp claim not found or not a float64") + return false, errors.New("cannot read exp claim") + } + } else { + log.Fatalf("Invalid token claims") + return false, errors.New("Invalid token claims") + } +} + +// Method to refresh the token +func (tm *TokenManager) refreshAccessToken() error { + formData := url.Values{ + "grant_type": {"refresh_token"}, + "client_id": {tm.clientId}, + "refresh_token": {tm.refreshToken}, + } + + tm.getTokens(formData) + return nil +} + +// Method to perform an HTTP request with token management +func (tm *TokenManager) DoRequest(req *http.Request) (*http.Response, error) { + token, err := tm.getToken() + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", "Bearer "+token) + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusUnauthorized { + // Try to refresh the token and retry the request + err := tm.refreshAccessToken() + if err != nil { + return nil, err + } + + // Update the token in the request and retry + req.Header.Set("Authorization", "Bearer "+tm.token) + resp, err = client.Do(req) + } + + return resp, err +} diff --git a/aggregator/model/client_events.go b/aggregator/model/client_events.go index 27da8a6..1bc5b30 100644 --- a/aggregator/model/client_events.go +++ b/aggregator/model/client_events.go @@ -33,21 +33,31 @@ func deleteOldClientEvents() error { retention_period := utils.GetEnv("RETENTION_PERIOD", "1 year") // see https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-INTERVAL-INPUT - query := "DELETE FROM client_events WHERE date < current_date - interval ?;" - _, err := pgdb.Query(nil, query, retention_period) - defer pgdb.Close() - if err != nil { - log.Println(err) - return err + eventsQuery := "DELETE FROM client_events WHERE date < current_date - interval ?;" + + // Using same retention period for sessions stats. May need to change that in the future. + sessionsQuery := "DELETE FROM client_sessions WHERE date < current_timestamp - interval ?;" + + _, eventsErr := pgdb.Query(nil, eventsQuery, retention_period) + _, sessionsErr := pgdb.Query(nil, sessionsQuery, retention_period) + + if eventsErr != nil { + log.Println(eventsErr) + return eventsErr + } + + if sessionsErr != nil { + log.Println(sessionsErr) + return sessionsErr } return nil } -func RunCronJob() { +func RunEventsJob() { loc := config.LoadTimeLocation() cron := gocron.NewScheduler(loc) cron.Every(1).Day().At("02:00").Do(func() { deleteOldClientEvents() }) - cron.StartBlocking() + cron.StartAsync() } diff --git a/aggregator/model/client_sessions.go b/aggregator/model/client_sessions.go new file mode 100644 index 0000000..b60a2cd --- /dev/null +++ b/aggregator/model/client_sessions.go @@ -0,0 +1,131 @@ +package model + +import ( + "encoding/json" + "log" + "net/http" + "strings" + + "io" + + "github.com/go-co-op/gocron" + "sso-dashboard.bcgov.com/aggregator/config" + "sso-dashboard.bcgov.com/aggregator/keycloak" + "sso-dashboard.bcgov.com/aggregator/utils" +) + +type RealmInfo struct { + Realm string `json:"realm"` +} + +type SessionStats struct { + ActiveSessions string `json:"active"` + ClientId string `json:"clientId"` + OfflineSessions string `json:"offline"` +} + +func GetRealms(tm *keycloak.TokenManager) []string { + req, err := http.NewRequest("GET", tm.BaseUrl+"/admin/realms", nil) + if err != nil { + log.Fatalf("Error occurred creating request: %v", err) + } + + resp, _ := tm.DoRequest(req) + + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatalf("Error reading response body: %v", err) + } + + var realms []RealmInfo + if err := json.Unmarshal([]byte(body), &realms); err != nil { + log.Fatalf("Error unmarshalling response body: %v", err) + } + + var realmNames []string + + for _, realm := range realms { + realmNames = append(realmNames, realm.Realm) + } + + return realmNames +} + +func GetClientStats(tm *keycloak.TokenManager, realms []string, env string) error { + for _, realm := range realms { + req, err := http.NewRequest("GET", tm.BaseUrl+"/admin/realms/"+realm+"/client-session-stats", nil) + if err != nil { + log.Fatalf("Error occurred creating request: %v", err) + return nil + } + + resp, _ := tm.DoRequest(req) + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + + if err != nil { + log.Fatalf("Error reading response body: %v", err) + return err + } + + var responseObjects []SessionStats + err = json.Unmarshal(body, &responseObjects) + if err != nil { + log.Fatalf("Error unmarshaling JSON response: %v", err) + return err + } + + for _, obj := range responseObjects { + err := InsertActiveSessions(env, realm, obj.ClientId, obj.ActiveSessions, obj.OfflineSessions) + if err != nil { + log.Fatalf("Error inserting active sessions: %v", err) + } + } + } + return nil +} + +func InsertActiveSessions(environment string, realmID string, clientID string, activeSessions string, offlineSessions string) error { + query := "INSERT INTO client_sessions (environment, realm_id, client_id, active_sessions, offline_sessions) VALUES(?,?,?,?,?);" + _, err := pgdb.Query(nil, query, environment, realmID, clientID, activeSessions, offlineSessions) + if err != nil { + log.Println(err) + return err + } + return nil +} + +func ActiveSessions(env string) { + log.Println("Getting active sessions for " + env + " environment") + baseUrl := utils.GetEnv(env+"_KEYCLOAK_URL", "") + clientId := utils.GetEnv(env+"_KEYCLOAK_CLIENT_ID", "") + username := utils.GetEnv(env+"_KEYCLOAK_USERNAME", "") + password := utils.GetEnv(env+"_KEYCLOAK_PASSWORD", "") + + tm := keycloak.NewTokenManager(clientId, password, username, baseUrl) + + realms := GetRealms(tm) + GetClientStats(tm, realms, strings.ToLower(env)) +} + +func AllActiveSessions() { + go func() { + ActiveSessions("DEV") + }() + + go func() { + ActiveSessions("TEST") + }() + + go func() { + ActiveSessions("PROD") + }() +} + +func RunSessionsJob() { + loc := config.LoadTimeLocation() + cron := gocron.NewScheduler(loc) + cron.Every(1).Hour().Do(AllActiveSessions) + cron.StartAsync() +} diff --git a/aggregator/models.py b/aggregator/models.py index 36c656d..4eb11a0 100755 --- a/aggregator/models.py +++ b/aggregator/models.py @@ -2,6 +2,16 @@ from sqlalchemy.types import TIMESTAMP from database import Base +class ClientSession(Base): + __tablename__ = "client_sessions" + + environment = Column(String(255), nullable=False) + realm_id = Column(String(255), nullable=False) + client_id = Column(String(255), nullable=False) + count = Column(Integer, nullable=False) + date = Column(TIMESTAMP(timezone=True), nullable=False) + id = Column(Integer, primary_key=True) + class ClientEvent(Base): __tablename__ = "client_events" diff --git a/helm/aggregator/README.md b/helm/aggregator/README.md index c131f4f..4d601ee 100644 --- a/helm/aggregator/README.md +++ b/helm/aggregator/README.md @@ -2,6 +2,10 @@ A Helm chart for deploying [Keycloak event log aggregator](../../aggregator). +## Setup + +This chart includes two different deployments, running the same dockerfile. The dockerfile build copies the compactor into app/compactor, and the aggregator into app/aggregator. In the compactor [deployment template](./templates/deployment-compactor.yaml) the command is used to run the compactor, whereas the [entry shell file](../../aggregator/docker-entrypoint.sh) will run the aggregator by default (last line). + ## Local deployment via Helm chart ### Pre-Requisites From f2ee039e0aca363040dfcb491b7eb72063c6c1c6 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 17 May 2024 11:56:55 -0700 Subject: [PATCH 02/16] chore: cleanup remove commented code --- aggregator/keycloak/tokenManager.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/aggregator/keycloak/tokenManager.go b/aggregator/keycloak/tokenManager.go index f9dedb1..8da3869 100644 --- a/aggregator/keycloak/tokenManager.go +++ b/aggregator/keycloak/tokenManager.go @@ -2,16 +2,13 @@ package keycloak import ( "encoding/json" + "errors" "fmt" - "io/ioutil" + "io" "log" "net/http" "net/url" "strings" - - // "io/ioutil" - // "log" - "errors" "time" "github.com/golang-jwt/jwt/v5" @@ -56,7 +53,7 @@ func (tm *TokenManager) getTokens(data url.Values) (string, error) { } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { log.Fatalf("Error reading response body: %v", err) From 7c44f03efd81493e286e5eadc8a900e08be9f01f Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:32:58 -0700 Subject: [PATCH 03/16] feat: webhook add webhook and tests --- aggregator/keycloak/tokenManager.go | 168 +++++++++++--------- aggregator/keycloak/token_manager_test.go | 84 ++++++++++ aggregator/keycloak/util.go | 25 +++ aggregator/model/client_events.go | 10 -- aggregator/model/client_sessions.go | 119 +++++++++----- aggregator/model/client_sessions_test.go | 183 ++++++++++++++++++++++ aggregator/webhooks/index.go | 64 ++++++++ 7 files changed, 535 insertions(+), 118 deletions(-) create mode 100644 aggregator/keycloak/token_manager_test.go create mode 100644 aggregator/keycloak/util.go create mode 100644 aggregator/model/client_sessions_test.go create mode 100644 aggregator/webhooks/index.go diff --git a/aggregator/keycloak/tokenManager.go b/aggregator/keycloak/tokenManager.go index 8da3869..33ac559 100644 --- a/aggregator/keycloak/tokenManager.go +++ b/aggregator/keycloak/tokenManager.go @@ -14,21 +14,40 @@ import ( "github.com/golang-jwt/jwt/v5" ) -type TokenManager struct { - token string - refreshToken string - password string - username string - BaseUrl string - clientId string +type TokenStrategy interface { + GetTokens(data url.Values, authUrl string) (string, string, error) + IsTokenExpired(token string) (bool, error) } -func NewTokenManager(clientId string, password string, username string, baseUrl string) *TokenManager { - return &TokenManager{ - clientId: clientId, - password: password, - username: username, - BaseUrl: baseUrl, +type RequestHandler struct { + ApiBaseUrl string + AuthBaseUrl string + + AccessToken string + RefreshToken string + + password string + username string + clientId string + + tokenStrategy TokenStrategy +} + +func NewRequestHandler( + tokenStrategy TokenStrategy, + ApiUrl string, + AuthUrl string, + password string, + username string, + clientId string, +) *RequestHandler { + return &RequestHandler{ + tokenStrategy: tokenStrategy, + ApiBaseUrl: ApiUrl, + AuthBaseUrl: AuthUrl, + password: password, + username: username, + clientId: clientId, } } @@ -36,11 +55,11 @@ func NewTokenManager(clientId string, password string, username string, baseUrl Method to make a post request for a new token or to refresh a token. Pass in the relevant data for the request type, e.g grant_type=password/client_credentials for a new token, grant_type=refresh_token for a refresh token. */ -func (tm *TokenManager) getTokens(data url.Values) (string, error) { - req, err := http.NewRequest("POST", tm.BaseUrl+"/realms/master/protocol/openid-connect/token", strings.NewReader(data.Encode())) +func (tm *RequestHandler) GetTokens(data url.Values, authUrl string) (string, string, error) { + req, err := http.NewRequest("POST", authUrl+"/realms/master/protocol/openid-connect/token", strings.NewReader(data.Encode())) if err != nil { - log.Fatalf("Error occurred creating request: %v", err) + log.Printf("Error occurred creating request: %v", err) } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") @@ -49,14 +68,22 @@ func (tm *TokenManager) getTokens(data url.Values) (string, error) { resp, err := client.Do(req) if err != nil { - log.Fatalf("Error occurred sending request to API endpoint: %v", err) + log.Printf("Error occurred sending request to API endpoint: %v", err) + return "", "", err } defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + log.Printf("non 200 status code returned from token request: %v", resp.Status) + return "", "", errors.New("non 200 status code returned from token request") + } + body, err := io.ReadAll(resp.Body) if err != nil { - log.Fatalf("Error reading response body: %v", err) + log.Printf("Error reading response body: %v", err) + return "", "", err } type TokenResponse struct { @@ -67,59 +94,78 @@ func (tm *TokenManager) getTokens(data url.Values) (string, error) { var tokenResponse TokenResponse if err := json.Unmarshal(body, &tokenResponse); err != nil { - log.Fatalf("Error parsing JSON response: %v", err) + log.Printf("Error parsing JSON response: %v", err) + return "", "", err } - tm.token = tokenResponse.AccessToken - tm.refreshToken = tokenResponse.RefreshToken - return tm.token, nil + return tokenResponse.AccessToken, tokenResponse.RefreshToken, nil } // Method to get the token, refreshing if necessary -func (tm *TokenManager) getToken() (string, error) { - // Fetch a new token if not set yet - if tm.token == "" { +func (rm *RequestHandler) GetToken() (string, string, error) { + if rm.AccessToken == "" { formData := url.Values{ "grant_type": {"password"}, - "client_id": {tm.clientId}, - "password": {tm.password}, - "username": {tm.username}, + "client_id": {rm.clientId}, + "password": {rm.password}, + "username": {rm.username}, } - token, err := tm.getTokens(formData) - return token, err + accessToken, refreshToken, err := rm.tokenStrategy.GetTokens(formData, rm.AuthBaseUrl) + return accessToken, refreshToken, err } // Check expiry and refresh if necessary - expired, err := tm.IsTokenExpired() + accessTokenExpired, err := rm.IsTokenExpired(rm.AccessToken) + if err != nil { - return "", err + return "", "", err } - if !expired { - return tm.token, nil + if !accessTokenExpired { + return rm.AccessToken, rm.RefreshToken, nil } - err = tm.refreshAccessToken() + refreshTokenExpired, err := rm.IsTokenExpired(rm.RefreshToken) + + var formData url.Values + + if refreshTokenExpired || err != nil { + formData = url.Values{ + "grant_type": {"password"}, + "client_id": {rm.clientId}, + "password": {rm.password}, + "username": {rm.username}, + } + } else { + formData = url.Values{ + "grant_type": {"refresh_token"}, + "client_id": {rm.clientId}, + "refresh_token": {rm.RefreshToken}, + } + } + + accessToken, refreshToken, err := rm.tokenStrategy.GetTokens(formData, rm.AuthBaseUrl) + if err != nil { - return "", err + return "", "", err } - return tm.token, nil + return accessToken, refreshToken, nil } // Checks if access token is expired -func (tm *TokenManager) IsTokenExpired() (bool, error) { - if tm.token == "" { +func (tm *RequestHandler) IsTokenExpired(token string) (bool, error) { + if token == "" { return true, nil } - token, _, err := new(jwt.Parser).ParseUnverified(tm.token, jwt.MapClaims{}) + parsedToken, _, err := new(jwt.Parser).ParseUnverified(token, jwt.MapClaims{}) if err != nil { - log.Fatalf("Error parsing token: %v", err) + log.Printf("Error parsing token: %v", err) } - if claims, ok := token.Claims.(jwt.MapClaims); ok { + if claims, ok := parsedToken.Claims.(jwt.MapClaims); ok { if exp, ok := claims["exp"].(float64); ok { now := time.Now().UTC() expire := time.Unix(int64(exp), 0).UTC() @@ -129,48 +175,28 @@ func (tm *TokenManager) IsTokenExpired() (bool, error) { return false, errors.New("cannot read exp claim") } } else { - log.Fatalf("Invalid token claims") + log.Printf("Invalid token claims") return false, errors.New("Invalid token claims") } } -// Method to refresh the token -func (tm *TokenManager) refreshAccessToken() error { - formData := url.Values{ - "grant_type": {"refresh_token"}, - "client_id": {tm.clientId}, - "refresh_token": {tm.refreshToken}, - } - - tm.getTokens(formData) - return nil -} - // Method to perform an HTTP request with token management -func (tm *TokenManager) DoRequest(req *http.Request) (*http.Response, error) { - token, err := tm.getToken() +func (rm *RequestHandler) DoRequest(req *http.Request) (*http.Response, error) { + accessToken, refreshToken, err := rm.GetToken() + if err != nil { + log.Printf("Error in dorequest") return nil, err } - req.Header.Set("Authorization", "Bearer "+token) + rm.AccessToken = accessToken + rm.RefreshToken = refreshToken + + req.Header.Set("Authorization", "Bearer "+accessToken) client := &http.Client{} resp, err := client.Do(req) if err != nil { return nil, err } - - if resp.StatusCode == http.StatusUnauthorized { - // Try to refresh the token and retry the request - err := tm.refreshAccessToken() - if err != nil { - return nil, err - } - - // Update the token in the request and retry - req.Header.Set("Authorization", "Bearer "+tm.token) - resp, err = client.Do(req) - } - return resp, err } diff --git a/aggregator/keycloak/token_manager_test.go b/aggregator/keycloak/token_manager_test.go new file mode 100644 index 0000000..73f5d7c --- /dev/null +++ b/aggregator/keycloak/token_manager_test.go @@ -0,0 +1,84 @@ +package keycloak + +import ( + "net/http" + "net/url" + "testing" + "time" +) + +type MockTokenProvider struct { + TokenRefreshed bool + NewTokenRequested bool +} + +func (tm *MockTokenProvider) ResetMock() { + tm.TokenRefreshed = false + tm.NewTokenRequested = false +} + +func (m *MockTokenProvider) GetTokens(data url.Values, url string) (string, string, error) { + grantType := data.Get("grant_type") + + if grantType == "refresh_token" { + m.TokenRefreshed = true + } + + if grantType == "password" { + m.NewTokenRequested = true + } + + return "", "", nil +} + +func (m *MockTokenProvider) IsTokenExpired(token string) (bool, error) { + return false, nil +} + +func TestTokenManagerHandler(t *testing.T) { + mockTokenProvider := &MockTokenProvider{} + handler := NewRequestHandler(mockTokenProvider, "", "", "", "", "") + + req, _ := http.NewRequest("GET", "http://somedomain.com", nil) + + // Make a request with expired access token and valid refresh token, expect refresh callout + handler.AccessToken = GenerateJWT(time.Now().Add(-time.Hour).Unix()) + handler.RefreshToken = GenerateJWT(time.Now().Add(time.Hour).Unix()) + handler.DoRequest(req) + + if !mockTokenProvider.TokenRefreshed { + t.Errorf("expected RefreshToken to be called, but it was not") + } + + // Make a request with expired access token and an expired refresh token, expect new token requested. + mockTokenProvider.ResetMock() + handler.AccessToken = "" + handler.RefreshToken = GenerateJWT(time.Now().Add(-time.Hour).Unix()) + handler.DoRequest(req) + + if !mockTokenProvider.NewTokenRequested { + t.Errorf("expected a new token to be requested, but it was not") + } + if mockTokenProvider.TokenRefreshed { + t.Errorf("expected only a new token to be requested, but a refresh callout was made") + } + + // Make a request with a valid access token, expect no refresh callout + mockTokenProvider.ResetMock() + handler.AccessToken = GenerateJWT(time.Now().Add(time.Hour).Unix()) + handler.DoRequest(req) + + if mockTokenProvider.TokenRefreshed || mockTokenProvider.NewTokenRequested { + t.Errorf("expected existing token to be used, but a new token was requested") + } + + // Make a request with no tokens, expect new token callout + mockTokenProvider.ResetMock() + handler.AccessToken = "" + handler.RefreshToken = "" + handler.DoRequest(req) + + if !mockTokenProvider.NewTokenRequested { + t.Errorf("expected a new token to be requested, but it was not") + } +} diff --git a/aggregator/keycloak/util.go b/aggregator/keycloak/util.go new file mode 100644 index 0000000..4fb9211 --- /dev/null +++ b/aggregator/keycloak/util.go @@ -0,0 +1,25 @@ +package keycloak + +import ( + "time" + + "github.com/golang-jwt/jwt/v5" +) + +/* Generate a JWT with a given expiration time. Useful for test cases. */ +func GenerateJWT(exp int64) string { + token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "exp": exp, + "iat": time.Now().Unix(), + "sub": "1234567890", + }) + + secretKey := []byte("secret") + tokenString, err := token.SignedString(secretKey) + + if err != nil { + return "" + } + + return tokenString +} diff --git a/aggregator/model/client_events.go b/aggregator/model/client_events.go index 1bc5b30..a053624 100644 --- a/aggregator/model/client_events.go +++ b/aggregator/model/client_events.go @@ -34,22 +34,12 @@ func deleteOldClientEvents() error { // see https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-INTERVAL-INPUT eventsQuery := "DELETE FROM client_events WHERE date < current_date - interval ?;" - - // Using same retention period for sessions stats. May need to change that in the future. - sessionsQuery := "DELETE FROM client_sessions WHERE date < current_timestamp - interval ?;" - _, eventsErr := pgdb.Query(nil, eventsQuery, retention_period) - _, sessionsErr := pgdb.Query(nil, sessionsQuery, retention_period) if eventsErr != nil { log.Println(eventsErr) return eventsErr } - - if sessionsErr != nil { - log.Println(sessionsErr) - return sessionsErr - } return nil } diff --git a/aggregator/model/client_sessions.go b/aggregator/model/client_sessions.go index b60a2cd..75e8b90 100644 --- a/aggregator/model/client_sessions.go +++ b/aggregator/model/client_sessions.go @@ -2,6 +2,8 @@ package model import ( "encoding/json" + "errors" + "fmt" "log" "net/http" "strings" @@ -12,6 +14,7 @@ import ( "sso-dashboard.bcgov.com/aggregator/config" "sso-dashboard.bcgov.com/aggregator/keycloak" "sso-dashboard.bcgov.com/aggregator/utils" + "sso-dashboard.bcgov.com/aggregator/webhooks" ) type RealmInfo struct { @@ -24,22 +27,40 @@ type SessionStats struct { OfflineSessions string `json:"offline"` } -func GetRealms(tm *keycloak.TokenManager) []string { - req, err := http.NewRequest("GET", tm.BaseUrl+"/admin/realms", nil) +var RealmErrorMessage = "Error getting realms for env %s: " +var ClientErrorMessage = "Error getting client stats for env %s: " + +type SessionInserter func(environment string, realmID string, clientID string, activeSessions string, offlineSessions string) error + +func GetRealms(rm *keycloak.RequestHandler) ([]string, error) { + req, err := http.NewRequest("GET", rm.ApiBaseUrl+"/admin/realms", nil) + if err != nil { + log.Printf("Error occurred creating request: %v", err) + return nil, err + } + + resp, err := rm.DoRequest(req) + if err != nil { - log.Fatalf("Error occurred creating request: %v", err) + log.Print("Error occurred making getting realms", err) + return nil, err } - resp, _ := tm.DoRequest(req) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + log.Printf("non 200 status code returned from realm request: %v", resp.Status) + return nil, errors.New("non 200 status code returned from realm request") + } body, err := io.ReadAll(resp.Body) if err != nil { - log.Fatalf("Error reading response body: %v", err) + log.Printf("Error reading response body: %v", err) + return nil, err } var realms []RealmInfo if err := json.Unmarshal([]byte(body), &realms); err != nil { - log.Fatalf("Error unmarshalling response body: %v", err) + log.Printf("Error unmarshalling response body: %v", err) + return nil, err } var realmNames []string @@ -48,42 +69,60 @@ func GetRealms(tm *keycloak.TokenManager) []string { realmNames = append(realmNames, realm.Realm) } - return realmNames + return realmNames, nil } -func GetClientStats(tm *keycloak.TokenManager, realms []string, env string) error { +func GetClientStats(rm *keycloak.RequestHandler, realms []string, env string) bool { + errOccured := false + + handleError := func(message string) { + log.Print(message) + errOccured = true + } + for _, realm := range realms { - req, err := http.NewRequest("GET", tm.BaseUrl+"/admin/realms/"+realm+"/client-session-stats", nil) + req, err := http.NewRequest("GET", rm.ApiBaseUrl+"/admin/realms/"+realm+"/client-session-stats", nil) if err != nil { - log.Fatalf("Error occurred creating request: %v", err) - return nil + handleError(fmt.Sprintf("Error occurred creating request: %v", err)) + continue } - resp, _ := tm.DoRequest(req) + resp, err := rm.DoRequest(req) + if err != nil { + handleError(fmt.Sprintf("Error occurred creating request: %v", err)) + continue + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + handleError(fmt.Sprintf("non 200 status code returned from realm request: %v", resp.Status)) + continue + } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - log.Fatalf("Error reading response body: %v", err) - return err + handleError(fmt.Sprintf("Error reading response body: %v", err)) + continue } var responseObjects []SessionStats err = json.Unmarshal(body, &responseObjects) if err != nil { - log.Fatalf("Error unmarshaling JSON response: %v", err) - return err + handleError(fmt.Sprintf("Error unmarshaling JSON response: %v", err)) + continue } for _, obj := range responseObjects { err := InsertActiveSessions(env, realm, obj.ClientId, obj.ActiveSessions, obj.OfflineSessions) if err != nil { - log.Fatalf("Error inserting active sessions: %v", err) + handleError(fmt.Sprintf("Error inserting active sessions: %v", err)) + continue } } } - return nil + + return errOccured } func InsertActiveSessions(environment string, realmID string, clientID string, activeSessions string, offlineSessions string) error { @@ -96,31 +135,37 @@ func InsertActiveSessions(environment string, realmID string, clientID string, a return nil } -func ActiveSessions(env string) { +func ActiveSessions(env string, baseUrl string, clientId string, username string, password string, notifier webhooks.RocketChatNotifier) { log.Println("Getting active sessions for " + env + " environment") - baseUrl := utils.GetEnv(env+"_KEYCLOAK_URL", "") - clientId := utils.GetEnv(env+"_KEYCLOAK_CLIENT_ID", "") - username := utils.GetEnv(env+"_KEYCLOAK_USERNAME", "") - password := utils.GetEnv(env+"_KEYCLOAK_PASSWORD", "") - tm := keycloak.NewTokenManager(clientId, password, username, baseUrl) + rm := keycloak.NewRequestHandler(&keycloak.RequestHandler{}, baseUrl, baseUrl, password, username, clientId) - realms := GetRealms(tm) - GetClientStats(tm, realms, strings.ToLower(env)) + realms, err := GetRealms(rm) + if err != nil { + log.Println(fmt.Sprintf(RealmErrorMessage, env), err) + notifier.NotifyRocketChat("Session Data Failure", fmt.Sprintf(RealmErrorMessage, env), err.Error()) + return + } + + hasError := GetClientStats(rm, realms, strings.ToLower(env)) + if hasError { + notifier.NotifyRocketChat("Session Data Failure", fmt.Sprintf(ClientErrorMessage, env), "One or more realm's client stats failed to be retrieved. See logs for details.") + return + } + notifier.NotifyRocketChat("Session Data Loaded Successfully", env, fmt.Sprintf("Session data for environment %s has been loaded successfully", env)) } func AllActiveSessions() { - go func() { - ActiveSessions("DEV") - }() - - go func() { - ActiveSessions("TEST") - }() - - go func() { - ActiveSessions("PROD") - }() + for _, env := range []string{"DEV", "TEST", "PROD"} { + env := env + baseUrl := utils.GetEnv(env+"_KEYCLOAK_URL", "") + clientId := utils.GetEnv(env+"_KEYCLOAK_CLIENT_ID", "") + username := utils.GetEnv(env+"_KEYCLOAK_USERNAME", "") + password := utils.GetEnv(env+"_KEYCLOAK_PASSWORD", "") + go func() { + ActiveSessions(env, baseUrl, clientId, username, password, &webhooks.RocketChat{}) + }() + } } func RunSessionsJob() { diff --git a/aggregator/model/client_sessions_test.go b/aggregator/model/client_sessions_test.go new file mode 100644 index 0000000..4193ec7 --- /dev/null +++ b/aggregator/model/client_sessions_test.go @@ -0,0 +1,183 @@ +package model + +import ( + "fmt" + "log" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "sso-dashboard.bcgov.com/aggregator/keycloak" +) + +type MockRocketChat struct { + Messages [][]string +} + +func (m *MockRocketChat) NotifyRocketChat(text string, title string, body string) { + message := []string{text, title, body} + m.Messages = append(m.Messages, message) +} + +func (m *MockRocketChat) ResetMock() { + m.Messages = [][]string{} +} + +func TestClientSessionTokenFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/realms/master/protocol/openid-connect/token" { + // Fail all token requests + w.WriteHeader(http.StatusForbidden) + } + })) + defer server.Close() + + mock := &MockRocketChat{} + ActiveSessions("DEV", server.URL, "client", "user", "pass", mock) + + if len(mock.Messages) != 1 { + t.Errorf("Expected 1 message to be sent to RocketChat") + } + rcTitle := mock.Messages[0][1] + rcError := mock.Messages[0][2] + if rcTitle != fmt.Sprintf(RealmErrorMessage, "DEV") { + t.Errorf("Expected rocket chat message to include the environment name") + } + if rcError != "non 200 status code returned from token request" { + log.Print(rcError) + t.Errorf("Expected rocket chat message to include the error message") + } +} + +func TestRealmNotifications(t *testing.T) { + var realmResponse string + var realmStatusCode int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + validToken := keycloak.GenerateJWT(time.Now().Add(time.Hour).Unix()) + if r.URL.Path == "/realms/master/protocol/openid-connect/token" { + w.WriteHeader(http.StatusAccepted) + w.Write([]byte(fmt.Sprintf(`{"access_token":"%s", "refresh_token":"%s"}`, validToken, validToken))) + } + if r.URL.Path == "/admin/realms" { + w.WriteHeader(realmStatusCode) + w.Write([]byte(realmResponse)) + } + })) + defer server.Close() + + mock := &MockRocketChat{} + + // Test when realm response is successful. Should send a successful message to RC + realmResponse = `[]` + realmStatusCode = http.StatusOK + ActiveSessions("DEV", server.URL, "client", "user", "pass", mock) + + if len(mock.Messages) != 1 { + t.Errorf("Expected 1 message to be sent to RocketChat") + } + rcText := mock.Messages[0][0] + rcTitle := mock.Messages[0][1] + + if rcText != "Session Data Loaded Successfully" { + t.Errorf("Expected successfuly rocket chat message") + } + if rcTitle != "DEV" { + t.Errorf("Expected rocket chat message to include the environment name") + } + + // When realm response is not ok, should notify rc + mock.ResetMock() + realmResponse = `[]` + realmStatusCode = http.StatusForbidden + + ActiveSessions("TEST", server.URL, "client", "user", "pass", mock) + + if len(mock.Messages) != 1 { + t.Errorf("Expected 1 message to be sent to RocketChat") + } + rcTitle = mock.Messages[0][1] + if rcTitle != fmt.Sprintf(RealmErrorMessage, "TEST") { + print(rcTitle) + t.Errorf("Expected rocket chat message to include the environment name") + } +} + +func TestClientNotifications(t *testing.T) { + var clientResponse string + var clientStatusCode int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + validToken := keycloak.GenerateJWT(time.Now().Add(time.Hour).Unix()) + if r.URL.Path == "/realms/master/protocol/openid-connect/token" { + w.WriteHeader(http.StatusAccepted) + w.Write([]byte(fmt.Sprintf(`{"access_token":"%s", "refresh_token":"%s"}`, validToken, validToken))) + } + if r.URL.Path == "/admin/realms" { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`[{"realm": "realm 1"}, {"realm": "realm 2"}]`)) + } + if strings.HasSuffix(r.URL.Path, "/client-session-stats") { + w.WriteHeader(clientStatusCode) + w.Write([]byte(clientResponse)) + } + })) + defer server.Close() + mock := &MockRocketChat{} + + // Test when realm response is successful. Should send a successful message to RC + clientResponse = `[]` + clientStatusCode = http.StatusOK + ActiveSessions("DEV", server.URL, "client", "user", "pass", mock) + + if len(mock.Messages) != 1 { + t.Errorf("Expected 1 message to be sent to RocketChat") + } + rcText := mock.Messages[0][0] + rcTitle := mock.Messages[0][1] + + if rcText != "Session Data Loaded Successfully" { + t.Errorf("Expected successfuly rocket chat message") + } + if rcTitle != "DEV" { + t.Errorf("Expected rocket chat message to include the environment name") + } + + // When client response is not ok, should notify rc + mock.ResetMock() + clientStatusCode = http.StatusForbidden + + ActiveSessions("TEST", server.URL, "client", "user", "pass", mock) + + if len(mock.Messages) != 1 { + t.Errorf("Expected 1 message to be sent to RocketChat") + } + rcTitle = mock.Messages[0][1] + if rcTitle != fmt.Sprintf(ClientErrorMessage, "TEST") { + t.Errorf("Expected rocket chat message to include the environment name") + } +} + +func TestClientContinue(t *testing.T) { + var requestCount int + var clientStatusCode int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + validToken := keycloak.GenerateJWT(time.Now().Add(time.Hour).Unix()) + if r.URL.Path == "/realms/master/protocol/openid-connect/token" { + w.WriteHeader(http.StatusAccepted) + w.Write([]byte(fmt.Sprintf(`{"access_token":"%s", "refresh_token":"%s"}`, validToken, validToken))) + } + if strings.HasSuffix(r.URL.Path, "/client-session-stats") { + requestCount++ + w.WriteHeader(clientStatusCode) + } + })) + defer server.Close() + rm := keycloak.NewRequestHandler(&keycloak.RequestHandler{}, server.URL, server.URL, "", "", "") + + clientStatusCode = http.StatusBadGateway + GetClientStats(rm, []string{"realm 1", "realm 2", "realm 3"}, "dev") + if requestCount != 3 { + t.Errorf("Expeted all realm requests to be attempted even if one fails") + } +} diff --git a/aggregator/webhooks/index.go b/aggregator/webhooks/index.go new file mode 100644 index 0000000..91f5d73 --- /dev/null +++ b/aggregator/webhooks/index.go @@ -0,0 +1,64 @@ +package webhooks + +import ( + "bytes" + "fmt" + "log" + "net/http" + + "sso-dashboard.bcgov.com/aggregator/utils" +) + +type Post struct { + Id int `json:"id"` + Title string `json:"title"` + Body string `json:"body"` + UserId int `json:"userId"` +} + +type RocketChatNotifier interface { + NotifyRocketChat(text string, title string, body string) +} + +type RocketChat struct{} + +/* +For the RC webhook, text will appear at the top, with a title and collapsible body below. +*/ +func (r *RocketChat) NotifyRocketChat(text string, title string, body string) { + // HTTP endpoint + log.Println("Sending rocket chat notification") + posturl := utils.GetEnv("RC_WEBHOOK", "") + + // JSON body + requestBodyTemplate := `{ + "text": "%s", + "attachments": [ + { + "title": "%s", + "text": "%s" + } + ] + }` + + requestBody := []byte(fmt.Sprintf(requestBodyTemplate, text, title, body)) + + // Create a HTTP post request + resp, err := http.NewRequest("POST", posturl, bytes.NewBuffer(requestBody)) + if err != nil { + log.Println("Error sending rocket chat notification", err) + } + resp.Header.Add("Content-Type", "application/json") + + client := &http.Client{} + res, err := client.Do(resp) + + if err != nil { + log.Println("Error sending rocket chat notification", err) + } + if res.StatusCode != 200 { + log.Println("Error sending rocket chat notification", res.Status, res.Body) + } + + defer res.Body.Close() +} From 6e469bc51551c4e8bf1420033d726e37a895b827 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:37:11 -0700 Subject: [PATCH 04/16] chore: workflow run go tests in workflow --- .github/actions/setup-tools/test.yaml | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/actions/setup-tools/test.yaml diff --git a/.github/actions/setup-tools/test.yaml b/.github/actions/setup-tools/test.yaml new file mode 100644 index 0000000..6b5ca22 --- /dev/null +++ b/.github/actions/setup-tools/test.yaml @@ -0,0 +1,32 @@ +name: Go CI + +on: + push: + branches: [ssoteam-1456] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ^1.21.0 + + - name: Install dependencies + run: go mod download + + - name: Run kc tests + - run: | + cd aggregator/keycloak + go test + + - name: Run session tests + - run: | + cd aggregator/keycloak + go test From f3246fa1afd92e04176b7233616b2dd66116f7f9 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:39:41 -0700 Subject: [PATCH 05/16] fix: workflow fix workflow file locaiton --- .github/{actions/setup-tools => workflows}/test.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{actions/setup-tools => workflows}/test.yaml (100%) diff --git a/.github/actions/setup-tools/test.yaml b/.github/workflows/test.yaml similarity index 100% rename from .github/actions/setup-tools/test.yaml rename to .github/workflows/test.yaml From a385f9b6270404bf962aff747eaf8bb4fb08f7b3 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:41:15 -0700 Subject: [PATCH 06/16] fix: workflow fix workflow syntax error --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6b5ca22..f7707ba 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,11 +22,11 @@ jobs: run: go mod download - name: Run kc tests - - run: | + run: | cd aggregator/keycloak go test - name: Run session tests - - run: | + run: | cd aggregator/keycloak go test From 9cf40123c57974d3ec73cfefaff85cf613bc18b9 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:45:56 -0700 Subject: [PATCH 07/16] fix: workflow fix workflow syntax error --- .github/workflows/test.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f7707ba..35c50b6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,6 +4,10 @@ on: push: branches: [ssoteam-1456] +defaults: + run: + working-directory: aggregator + jobs: build: @@ -23,10 +27,10 @@ jobs: - name: Run kc tests run: | - cd aggregator/keycloak + cd keycloak go test - name: Run session tests run: | - cd aggregator/keycloak + cd ../model go test From 7bba6dc3ce984b91d630d1d6f636f7125893e458 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:46:57 -0700 Subject: [PATCH 08/16] fix: workflow fix workflow syntax error --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 35c50b6..fb9f90a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,5 +32,5 @@ jobs: - name: Run session tests run: | - cd ../model + cd model go test From 1e396331d231f61bb15fbf88597bae5903e37a4d Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:50:25 -0700 Subject: [PATCH 09/16] fix: workflow fix workflow syntax error --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fb9f90a..cd75ee3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -28,9 +28,9 @@ jobs: - name: Run kc tests run: | cd keycloak - go test + go test -v - name: Run session tests run: | cd model - go test + go test -v From 3c3b7e6dc641556d9613ac679ddd82d5413fd58e Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:52:51 -0700 Subject: [PATCH 10/16] chore: tests fix slow running test --- aggregator/keycloak/token_manager_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/keycloak/token_manager_test.go b/aggregator/keycloak/token_manager_test.go index 73f5d7c..e388509 100644 --- a/aggregator/keycloak/token_manager_test.go +++ b/aggregator/keycloak/token_manager_test.go @@ -39,7 +39,7 @@ func TestTokenManagerHandler(t *testing.T) { mockTokenProvider := &MockTokenProvider{} handler := NewRequestHandler(mockTokenProvider, "", "", "", "", "") - req, _ := http.NewRequest("GET", "http://somedomain.com", nil) + req, _ := http.NewRequest("GET", "", nil) // Make a request with expired access token and valid refresh token, expect refresh callout handler.AccessToken = GenerateJWT(time.Now().Add(-time.Hour).Unix()) From 232b759ebd48603c5c2e42c74cba0c3baddd44a9 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:55:19 -0700 Subject: [PATCH 11/16] chore: workflow run tests on any push --- .github/workflows/test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cd75ee3..2df21a6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,7 +2,9 @@ name: Go CI on: push: - branches: [ssoteam-1456] + paths: + - aggregator/** + defaults: run: From 56a7ead8a774598bf1a182a3fb875e33d5e8fce4 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 15:56:02 -0700 Subject: [PATCH 12/16] chore: workflow run tests on push to aggregator or workflow --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2df21a6..b3220be 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,6 +4,7 @@ on: push: paths: - aggregator/** + - .github/workflows/test.yaml defaults: From 2b044fd1a3c454040a4d3debd97f4cf6893b9a49 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 16:18:15 -0700 Subject: [PATCH 13/16] chore: helm update helm chart for new secrets --- aggregator/.env.example | 16 ++++++++++ .../templates/deployment-compactor.yaml | 30 +++++++++++++++++++ helm/aggregator/values-e4ca1d-prod.yaml | 16 ++++++++++ helm/aggregator/values-eb75ad-tools.yaml | 16 ++++++++++ 4 files changed, 78 insertions(+) diff --git a/aggregator/.env.example b/aggregator/.env.example index 1e38519..443f06a 100644 --- a/aggregator/.env.example +++ b/aggregator/.env.example @@ -5,3 +5,19 @@ DB_USERNAME= DB_PASSWORD= RETENTION_PERIOD= TZ=America/Vancouver +RC_WEBHOOK= + +DEV_KEYCLOAK_URL= +DEV_KEYCLOAK_CLIENT_ID= +DEV_KEYCLOAK_USERNAME= +DEV_KEYCLOAK_PASSWORD= + +TEST_KEYCLOAK_URL= +TEST_KEYCLOAK_CLIENT_ID= +TEST_KEYCLOAK_USERNAME= +TEST_KEYCLOAK_PASSWORD= + +PROD_KEYCLOAK_URL= +PROD_KEYCLOAK_CLIENT_ID= +PROD_KEYCLOAK_USERNAME= +PROD_KEYCLOAK_PASSWORD= diff --git a/helm/aggregator/templates/deployment-compactor.yaml b/helm/aggregator/templates/deployment-compactor.yaml index 8116bb1..4f391ca 100644 --- a/helm/aggregator/templates/deployment-compactor.yaml +++ b/helm/aggregator/templates/deployment-compactor.yaml @@ -59,6 +59,36 @@ spec: value: {{ .Values.postgres.database }} - name: RETENTION_PERIOD value: {{ .Values.compactor.retentionPeriod }} + - name: RC_WEBHOOK + value: {{ .Values.compactor.webhookUrl }} + + - name: DEV_KEYCLOAK_URL + value: {{ .Values.compactor.dev.keycloakUrl }} + - name: DEV_KEYCLOAK_CLIENT_ID + value: {{ .Values.compactor.dev.keycloakClientId }} + - name: DEV_KEYCLOAK_USERNAME + value: {{ .Values.compactor.dev.keycloakUsername }} + - name: DEV_KEYCLOAK_PASSWORD + value: {{ .Values.compactor.dev.keycloakPassword }} + + - name: TEST_KEYCLOAK_URL + value: {{ .Values.compactor.test.keycloakUrl }} + - name: TEST_KEYCLOAK_CLIENT_ID + value: {{ .Values.compactor.test.keycloakClientId }} + - name: TEST_KEYCLOAK_USERNAME + value: {{ .Values.compactor.test.keycloakUsername }} + - name: TEST_KEYCLOAK_PASSWORD + value: {{ .Values.compactor.test.keycloakPassword }} + + - name: PROD_KEYCLOAK_URL + value: {{ .Values.compactor.prod.keycloakUrl }} + - name: PROD_KEYCLOAK_CLIENT_ID + value: {{ .Values.compactor.prod.keycloakClientId }} + - name: PROD_KEYCLOAK_USERNAME + value: {{ .Values.compactor.prod.keycloakUsername }} + - name: PROD_KEYCLOAK_PASSWORD + value: {{ .Values.compactor.prod.keycloakPassword }} + resources: {{- toYaml .Values.compactor.resources | nindent 12 }} {{- end }} diff --git a/helm/aggregator/values-e4ca1d-prod.yaml b/helm/aggregator/values-e4ca1d-prod.yaml index d283ca1..e9c3827 100644 --- a/helm/aggregator/values-e4ca1d-prod.yaml +++ b/helm/aggregator/values-e4ca1d-prod.yaml @@ -8,6 +8,22 @@ patroni: compactor: enabled: true retentionPeriod: '3 months' + webhookUrl: + dev: + keycloakUrl: + keycloakClientId: + keycloakUsername: + keycloakPassword: + test: + keycloakUrl: + keycloakClientId: + keycloakUsername: + keycloakPassword: + prod: + keycloakUrl: + keycloakClientId: + keycloakUsername: + keycloakPassword: autoscaling: enabled: true diff --git a/helm/aggregator/values-eb75ad-tools.yaml b/helm/aggregator/values-eb75ad-tools.yaml index ca3df3b..d9f6bad 100644 --- a/helm/aggregator/values-eb75ad-tools.yaml +++ b/helm/aggregator/values-eb75ad-tools.yaml @@ -8,6 +8,22 @@ patroni: compactor: enabled: true retentionPeriod: '6 months' + webhookUrl: + dev: + keycloakUrl: + keycloakClientId: + keycloakUsername: + keycloakPassword: + test: + keycloakUrl: + keycloakClientId: + keycloakUsername: + keycloakPassword: + prod: + keycloakUrl: + keycloakClientId: + keycloakUsername: + keycloakPassword: autoscaling: enabled: true From 82835a77f9b3a8a66d0267d827dd7fc12d5bf5e7 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 16:49:54 -0700 Subject: [PATCH 14/16] chore: cleanup small refactor, docs and cleanup --- aggregator/README.md | 3 ++- aggregator/keycloak/tokenManager.go | 24 +++++++++++++++--- aggregator/model/client_sessions.go | 32 ++---------------------- aggregator/model/client_sessions_test.go | 2 +- aggregator/webhooks/index.go | 7 ------ 5 files changed, 25 insertions(+), 43 deletions(-) diff --git a/aggregator/README.md b/aggregator/README.md index 3b8cf5f..190560d 100644 --- a/aggregator/README.md +++ b/aggregator/README.md @@ -10,7 +10,7 @@ In order to avoid the custom codebase parsing the requests, it relies on `Grafan A lightweight Go server running scheduled jobs. There are two cronjobs it controls: 1. Deleting old client events. -2. Collecting and clearing client session counts. +2. Collecting client session counts. ## Environment Variables @@ -21,6 +21,7 @@ A lightweight Go server running scheduled jobs. There are two cronjobs it contro - `DB_PASSWORD`: the password to be used for password authentication. - `RETENTION_PERIOD`: the duration of time to keep the aggregated data. - please see [Postgres Interval Input](https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-INTERVAL-INPUT) for the unit convention. +- `RC_WEBHOOK`: The url for the rocketchat webhook to use when notifying from the compactor - `DEV_KEYCLOAK_URL`: The development keycloak base URL - `DEV_KEYCLOAK_CLIENT_ID`: The development keycloak client id - `DEV_KEYCLOAK_USERNAME`: The development keycloak username diff --git a/aggregator/keycloak/tokenManager.go b/aggregator/keycloak/tokenManager.go index 33ac559..4ca813c 100644 --- a/aggregator/keycloak/tokenManager.go +++ b/aggregator/keycloak/tokenManager.go @@ -129,7 +129,7 @@ func (rm *RequestHandler) GetToken() (string, string, error) { refreshTokenExpired, err := rm.IsTokenExpired(rm.RefreshToken) var formData url.Values - + // If there is an error or expiry, must get a new set of tokens. Otherwise refresh if refreshTokenExpired || err != nil { formData = url.Values{ "grant_type": {"password"}, @@ -180,8 +180,10 @@ func (tm *RequestHandler) IsTokenExpired(token string) (bool, error) { } } -// Method to perform an HTTP request with token management -func (rm *RequestHandler) DoRequest(req *http.Request) (*http.Response, error) { +/* +Method to perform an HTTP request with token management. Returns the body or an error if network failure or non-200 status code. +*/ +func (rm *RequestHandler) DoRequest(req *http.Request) ([]byte, error) { accessToken, refreshToken, err := rm.GetToken() if err != nil { @@ -198,5 +200,19 @@ func (rm *RequestHandler) DoRequest(req *http.Request) (*http.Response, error) { if err != nil { return nil, err } - return resp, err + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + log.Printf("non 200 status code returned from realm request: %v", resp.Status) + return nil, errors.New("non 200 status code returned from realm request") + } + + body, err := io.ReadAll(resp.Body) + defer resp.Body.Close() + + if err != nil { + log.Printf("Error reading response body: %v", err) + return nil, err + } + + return body, err } diff --git a/aggregator/model/client_sessions.go b/aggregator/model/client_sessions.go index 75e8b90..698a59e 100644 --- a/aggregator/model/client_sessions.go +++ b/aggregator/model/client_sessions.go @@ -2,14 +2,11 @@ package model import ( "encoding/json" - "errors" "fmt" "log" "net/http" "strings" - "io" - "github.com/go-co-op/gocron" "sso-dashboard.bcgov.com/aggregator/config" "sso-dashboard.bcgov.com/aggregator/keycloak" @@ -30,8 +27,6 @@ type SessionStats struct { var RealmErrorMessage = "Error getting realms for env %s: " var ClientErrorMessage = "Error getting client stats for env %s: " -type SessionInserter func(environment string, realmID string, clientID string, activeSessions string, offlineSessions string) error - func GetRealms(rm *keycloak.RequestHandler) ([]string, error) { req, err := http.NewRequest("GET", rm.ApiBaseUrl+"/admin/realms", nil) if err != nil { @@ -39,19 +34,8 @@ func GetRealms(rm *keycloak.RequestHandler) ([]string, error) { return nil, err } - resp, err := rm.DoRequest(req) - - if err != nil { - log.Print("Error occurred making getting realms", err) - return nil, err - } - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - log.Printf("non 200 status code returned from realm request: %v", resp.Status) - return nil, errors.New("non 200 status code returned from realm request") - } + body, err := rm.DoRequest(req) - body, err := io.ReadAll(resp.Body) if err != nil { log.Printf("Error reading response body: %v", err) return nil, err @@ -87,19 +71,7 @@ func GetClientStats(rm *keycloak.RequestHandler, realms []string, env string) bo continue } - resp, err := rm.DoRequest(req) - if err != nil { - handleError(fmt.Sprintf("Error occurred creating request: %v", err)) - continue - } - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - handleError(fmt.Sprintf("non 200 status code returned from realm request: %v", resp.Status)) - continue - } - - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + body, err := rm.DoRequest(req) if err != nil { handleError(fmt.Sprintf("Error reading response body: %v", err)) diff --git a/aggregator/model/client_sessions_test.go b/aggregator/model/client_sessions_test.go index 4193ec7..771a9b1 100644 --- a/aggregator/model/client_sessions_test.go +++ b/aggregator/model/client_sessions_test.go @@ -25,7 +25,7 @@ func (m *MockRocketChat) ResetMock() { m.Messages = [][]string{} } -func TestClientSessionTokenFailure(t *testing.T) { +func TestTokenFailure(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/realms/master/protocol/openid-connect/token" { // Fail all token requests diff --git a/aggregator/webhooks/index.go b/aggregator/webhooks/index.go index 91f5d73..dae7306 100644 --- a/aggregator/webhooks/index.go +++ b/aggregator/webhooks/index.go @@ -9,13 +9,6 @@ import ( "sso-dashboard.bcgov.com/aggregator/utils" ) -type Post struct { - Id int `json:"id"` - Title string `json:"title"` - Body string `json:"body"` - UserId int `json:"userId"` -} - type RocketChatNotifier interface { NotifyRocketChat(text string, title string, body string) } From 3bfbdadc846b6652b3dc6c77c20e762bfe8d29df Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Fri, 24 May 2024 17:01:46 -0700 Subject: [PATCH 15/16] chore: events undo event changes since not clearing sessions --- aggregator/model/client_events.go | 10 +++++----- aggregator/model/client_sessions.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aggregator/model/client_events.go b/aggregator/model/client_events.go index a053624..28d3b93 100644 --- a/aggregator/model/client_events.go +++ b/aggregator/model/client_events.go @@ -33,12 +33,12 @@ func deleteOldClientEvents() error { retention_period := utils.GetEnv("RETENTION_PERIOD", "1 year") // see https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-INTERVAL-INPUT - eventsQuery := "DELETE FROM client_events WHERE date < current_date - interval ?;" - _, eventsErr := pgdb.Query(nil, eventsQuery, retention_period) + query := "DELETE FROM client_events WHERE date < current_date - interval ?;" + _, err := pgdb.Query(nil, query, retention_period) - if eventsErr != nil { - log.Println(eventsErr) - return eventsErr + if err != nil { + log.Println(err) + return err } return nil } diff --git a/aggregator/model/client_sessions.go b/aggregator/model/client_sessions.go index 698a59e..164c2c3 100644 --- a/aggregator/model/client_sessions.go +++ b/aggregator/model/client_sessions.go @@ -143,6 +143,6 @@ func AllActiveSessions() { func RunSessionsJob() { loc := config.LoadTimeLocation() cron := gocron.NewScheduler(loc) - cron.Every(1).Hour().Do(AllActiveSessions) + cron.Every(20).Second().Do(AllActiveSessions) cron.StartAsync() } From 597ae04462cc8db5fec18f83cc12ce30d037d645 Mon Sep 17 00:00:00 2001 From: jonathan langlois Date: Tue, 28 May 2024 09:00:35 -0700 Subject: [PATCH 16/16] chore: reduce frequency reduce cron run to every hour --- aggregator/model/client_sessions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/model/client_sessions.go b/aggregator/model/client_sessions.go index 164c2c3..698a59e 100644 --- a/aggregator/model/client_sessions.go +++ b/aggregator/model/client_sessions.go @@ -143,6 +143,6 @@ func AllActiveSessions() { func RunSessionsJob() { loc := config.LoadTimeLocation() cron := gocron.NewScheduler(loc) - cron.Every(20).Second().Do(AllActiveSessions) + cron.Every(1).Hour().Do(AllActiveSessions) cron.StartAsync() }