diff --git a/.env.dist b/.env.dist index 8a326f539..715df3296 100644 --- a/.env.dist +++ b/.env.dist @@ -67,8 +67,8 @@ RALPH_BACKENDS__DATA__ES__TEST_FORWARDING_INDEX=test-index-foo-2 # ES lrs backend # Same options as for the ES data backend, however they are prefixed with -# RALPH_BACKENDS__LRS__ES__ instead. Example: -# RALPH_BACKENDS__LRS__ES__HOSTS=http://elasticsearch:9200 +# RALPH_BACKENDS__LRS__ES__ instead. +RALPH_BACKENDS__LRS__ES__HOSTS=http://elasticsearch:9200 # MONGO data backend diff --git a/README.md b/README.md index 61d9939e3..2d8b8b7b2 100644 --- a/README.md +++ b/README.md @@ -99,14 +99,15 @@ If the database status is satisfying, you are now ready to send xAPI statements to the LRS: ```bash -gunzip data/statements.json.gz | \ +gunzip -c data/statements.json.gz | \ head -n 100 | \ jq -s . | \ curl -Lk \ --user ralph:secret \ -X POST \ -H "Content-Type: application/json" \ - http://localhost:8100/xAPI/statements/ -d @- + -d @- \ + http://localhost:8100/xAPI/statements/ ``` The command above fetches one hundred (100) example xAPI statements from our diff --git a/docker-compose-test.yml b/docker-compose-test.yml deleted file mode 100644 index 5d3cc0147..000000000 --- a/docker-compose-test.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: "3.9" - -services: - db: - image: clickhouse/clickhouse-server:23.1.1.3077-alpine - environment: - CLICKHOUSE_DB: xapi - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 - ports: - - 8123:8123 - - 9000:9000 - # ClickHouse needs to maintain a lot of open files, so they - # suggest running the container with increased limits: - # https://hub.docker.com/r/clickhouse/clickhouse-server/#! - ulimits: - nofile: - soft: 262144 - hard: 262144 - healthcheck: - test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1 - interval: 1s - retries: 60 - - lrs: - image: fundocker/ralph:master - environment: - RALPH_APP_DIR: /app/.ralph - RALPH_RUNSERVER_BACKEND: clickhouse - RALPH_BACKENDS__LRS__CLICKHOUSE__HOST: db - RALPH_BACKENDS__LRS__CLICKHOUSE__PORT: 8123 - RALPH_BACKENDS__LRS__CLICKHOUSE__DATABASE: xapi - ports: - - "8100:8100" - command: - - "uvicorn" - - "ralph.api:app" - - "--proxy-headers" - - "--host" - - "0.0.0.0" - - "--port" - - "8100" - volumes: - - .ralph:/app/.ralph \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 814bccbb9..a913c0431 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: - "${RALPH_RUNSERVER_BACKEND:-es}" volumes: - .:/app - + # -- backends elasticsearch: image: elasticsearch:8.1.0 diff --git a/docs/api/index.md b/docs/api/index.md index 45bd88109..a941d0a62 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -183,13 +183,13 @@ By default, all authenticated users have full read and write access to the serve ### Filtering results by authority (multitenancy) -In Ralph LRS, all incoming statements are assigned an `authority` (or ownership) derived from the user that makes the call. You may restrict read access to users "own" statements (thus enabling multitenancy) by setting the following environment variable: +In Ralph LRS, all incoming statements are assigned an `authority` (or ownership) derived from the user that makes the request. You may restrict read access to users "own" statements (thus enabling multitenancy) by setting the following environment variable: ```bash RALPH_LRS_RESTRICT_BY_AUTHORITY = True # Default: False ``` -**WARNING**: Two accounts with different credentials may share the same `authority` meaning they can access the same statements. It is the administrators responsability to ensure that `authority` is properly assigned. +**WARNING**: Two accounts with different credentials may share the same `authority`, meaning they can access the same statements. It is the administrator's responsibility to ensure that `authority` is properly assigned. NB: If not using "scopes", or for users with limited "scopes", using this option will make the use of option `?mine=True` implicit when fetching statement. diff --git a/docs/tutorials/development_guide.md b/docs/tutorials/development_guide.md index dd66a9603..97fbad93e 100644 --- a/docs/tutorials/development_guide.md +++ b/docs/tutorials/development_guide.md @@ -17,7 +17,7 @@ You should know that we would be glad to help you contribute to Ralph! Here's ou !!! info - In this tutorial, and even more generally in others tutorials, we tend to use Elasticsearch backend. Note that you can do the same with [another LRS backend](../backends) implemented in Ralph. + In this tutorial, and even more generally in others tutorials, we tend to use Elasticsearch backend. Note that you can do the same with [another LRS backend](../backends/index.md) implemented in Ralph. To start playing with `ralph`, you should first `bootstrap` using: @@ -279,22 +279,22 @@ curl -sLk \ -o jsonpath='{.spec.rules[0].host}')/whoami" ``` -And why not send test statements: +Let's also send some test statements: ```bash -gunzip data/statements.json.gz | \ +gunzip -c data/statements.json.gz | \ head -n 100 | \ jq -s . | \ curl -sLk \ --user foo:bar \ -X POST \ -H "Content-Type: application/json" \ + -d @- \ "https://$(\ kubectl -n development-ralph \ get \ ingress/ralph-app-current \ - -o jsonpath='{.spec.rules[0].host}')/xAPI/statements/" \ - -d @- + -o jsonpath='{.spec.rules[0].host}')/xAPI/statements/" ``` !!! tip "Install `jq`" diff --git a/docs/tutorials/lrs/authentication/basic.md b/docs/tutorials/lrs/authentication/basic.md index 73023bbba..2d362b6ae 100644 --- a/docs/tutorials/lrs/authentication/basic.md +++ b/docs/tutorials/lrs/authentication/basic.md @@ -1,6 +1,7 @@ # HTTP Basic Authentication -The default method for securing Ralph API server is with HTTP basic authentication. For this, we need to create a user in Ralph LRS. +The default method for securing the Ralph API server is HTTP Basic Authentication. +For this, we need to create a user in Ralph LRS. ## Creating user credentials @@ -10,31 +11,30 @@ To create a new user credentials, Ralph CLI provides a dedicated command: ```bash ralph auth \ + --write-to-disk \ --username janedoe \ --password supersecret \ - --scope janedoe_scope \ - --agent-ifi-mbox mailto:janedoe@example.com \ - # or --agent-ifi-mbox-sha1sum ebd31e95054c018b10727ccffd2ef2ec3a016ee9 \ - # or --agent-ifi-openid "http://jane.openid.example.org/" \ - # or --agent-ifi-account exampleAccountname http://www.exampleHomePage.com \ - -w + --scope statements/write \ + --scope statements/read \ + --agent-ifi-mbox mailto:janedoe@example.com ``` === "Docker Compose" ```bash docker compose run --rm lrs \ - bin/ralph auth \ + ralph auth \ + --write-to-disk \ --username janedoe \ --password supersecret \ - --scope janedoe_scope \ - --agent-ifi-mbox mailto:janedoe@example.com \ - # or --agent-ifi-mbox-sha1sum ebd31e95054c018b10727ccffd2ef2ec3a016ee9 \ - # or --agent-ifi-openid "http://jane.openid.example.org/" \ - # or --agent-ifi-account exampleAccountname http://www.exampleHomePage.com \ - -w + --scope statements/write \ + --scope statements/read \ + --agent-ifi-mbox mailto:janedoe@example.com ``` +!!! tip + You can either display the helper with `ralph auth --help` or check the CLI tutorial [here](../../cli.md) + This command updates your credentials file with the new `janedoe` user. Here is the file that has been created by the `ralph auth` command: @@ -104,7 +104,7 @@ and running the Ralph LRS with: docker compose up -d lrs ``` -we can try to make a request to the `whoami` endpoint again, but this time sending our username and password through Basic Auth: +we can request the `whoami` endpoint again, but this time sending our username and password through Basic Auth: === "curl" diff --git a/docs/tutorials/lrs/authentication/oidc.md b/docs/tutorials/lrs/authentication/oidc.md index 1fbb49cb0..318231624 100644 --- a/docs/tutorials/lrs/authentication/oidc.md +++ b/docs/tutorials/lrs/authentication/oidc.md @@ -17,69 +17,156 @@ It is also strongly recommended to set the optional `RALPH_RUNSERVER_AUTH_OIDC_A OpenID Connect support is currently developed and tested against [Keycloak](https://www.keycloak.org/) but may work with other identity providers that implement the specification. +## An example with Keycloak + The [Learning analytics playground](https://github.com/openfun/learning-analytics-playground/) repository contains a Docker Compose file and configuration for a demo instance of Keycloak with a `ralph` client. -## Making a GET request +First, we should stop the Ralph LRS server (if it's still running): +```bash +docker compose down +``` + +We can clone the `learning-analytics-playground` repository: +```bash +git clone git@github.com:openfun/learning-analytics-playground +``` + +Let's first create the network named `ralph`: +```bash +docker network create ralph +``` + +Then we can bootstrap the project: +```bash +cd learning-analytics-playground/ +make bootstrap +``` + +After a couple of minutes, the playground containers should be up and running. + + + +Create another docker compose file, let's call it `docker-compose.oidc.yml`, with the following content: +```yaml title="docker-compose.oidc.yml" hl_lines="9-10 26-27 29-31" +version: "3.9" + +services: + + lrs: + image: fundocker/ralph:latest + environment: + RALPH_APP_DIR: /app/.ralph + RALPH_RUNSERVER_AUTH_BACKEND: oidc + RALPH_RUNSERVER_AUTH_OIDC_ISSUER_URI: http://learning-analytics-playground_keycloak_1:8080/auth/realms/fun-mooc + RALPH_RUNSERVER_BACKEND: fs + ports: + - "8100:8100" + command: + - "uvicorn" + - "ralph.api:app" + - "--proxy-headers" + - "--workers" + - "1" + - "--host" + - "0.0.0.0" + - "--port" + - "8100" + volumes: + - .ralph:/app/.ralph + networks: + - ralph + +networks: + ralph: + external: true + +``` + +Again, we need to create the `.ralph` directory: +```bash +mkdir .ralph +``` + +Then we can start the `lrs` service: +```bash +docker compose -f docker-compose.oidc.yml up -d lrs +``` -With the Keycloak instance running, we can get the access token from Keycloak: +Now that both Keycloak and Ralph LRS server are up and running, we should be able to get the access token from Keycloak with the command: === "curl" ```bash - curl --request POST 'http://localhost:8080/auth/realms/fun-mooc/protocol/openid-connect/token' \ - --header 'Content-Type: application/x-www-form-urlencoded' \ - --data-urlencode 'client_id=ralph' \ - --data-urlencode 'client_secret=super-secret' \ - --data-urlencode 'username=ralph_admin' \ - --data-urlencode 'password=funfunfun' \ - --data-urlencode 'grant_type=password' + curl -X POST \ + -d "grant_type=password" \ + -d "client_id=ralph" \ + -d "client_secret=bcef3562-730d-4575-9e39-63e185f99bca" \ + -d "username=ralph_admin" \ + -d "password=funfunfun" \ + http://localhost:8080/auth/realms/fun-mooc/protocol/openid-connect/token ``` + ```bash + {"access_token":"","expires_in":300,"refresh_expires_in":1800,"refresh_token":"","token_type":"Bearer","not-before-policy":0,"session_state":"0889b3a5-d742-45fb-98b3-20e967960e74","scope":"email profile"} + ``` === "HTTPie" ```bash - http --form POST :8090/auth/reals/fun-mooc/protocol/openid-connect/token \ - client_id="ralph" \ - client_secret="super-secret" \ - username="ralph_admin" \ - password="funfunfun" \ - grant_type="password" + http -f POST \ + :8080/auth/realms/fun-mooc/protocol/openid-connect/token \ + grant_type=password \ + client_id=ralph \ + client_secret=bcef3562-730d-4575-9e39-63e185f99bca \ + username=ralph_admin \ + password=funfunfun ``` -which outputs (*tokens truncated for example purpose*): - -```json -{ - "access_token":"eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJSTWlLM", - "expires_in":300, - "refresh_expires_in":1800, - "refresh_token":"eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI4MDc5NjExM", - "token_type":"Bearer", - "not-before-policy":0, - "session_state":"22a36735-e35f-496b-a243-152d32ebff45", - "scope":"profile email" -} -``` + ```bash + HTTP/1.1 200 OK + ... + { + "access_token": "", + "expires_in": 300, + "not-before-policy": 0, + "refresh_expires_in": 1800, + "refresh_token": "", + "scope": "email profile", + "session_state": "1e826fa2-b4b3-42bf-837f-158fe9d5e1e5", + "token_type": "Bearer" + } + ``` -If we now send the access token to the API server as a Bearer header: +With this access token, we can now make a request to the Ralph LRS server: === "curl" + + ```bash + curl -H 'Authorization: Bearer ' \ + http://localhost:8100/whoami + ``` + ```bash - curl http://localhost:8100/whoami --A "Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJSTWlLM" + {"agent":{"openid":"http://localhost:8080/auth/realms/fun-mooc/b6e85bd0-ce6e-4b24-9f0e-6e18d8744e54"},"scopes":["email","profile"]} ``` === "HTTPie" + ```bash - http -A bearer -a eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJSTWlLM :8100/whoami + http -A bearer -a :8100/whoami ``` + ```bash + HTTP/1.1 200 OK + ... + { + "agent": { + "openid": "http://localhost:8080/auth/realms/fun-mooc/b6e85bd0-ce6e-4b24-9f0e-6e18d8744e54" + }, + "scopes": [ + "email", + "profile" + ] + } + ``` -Ralph LRS returns our authenticated user: - -``` console -HTTP/1.1 200 OK -{ - "username":"ralph_admin", - "scopes":["all"] -} -``` +Congrats, you've managed to authenticate using OpenID Connect! 🎉 diff --git a/docs/tutorials/lrs/backends.md b/docs/tutorials/lrs/backends.md index 8d45c0bcd..adca8a28e 100644 --- a/docs/tutorials/lrs/backends.md +++ b/docs/tutorials/lrs/backends.md @@ -172,7 +172,7 @@ Let's add the service of your choice to the `docker-compose.yml` file: ) ENGINE MergeTree ORDER BY (emission_time, event_id) PRIMARY KEY (emission_time, event_id)" | \ - curl "http://localhost:8123/" --data-binary @- + curl --data-binary @- "http://localhost:8123/" ``` === "HTTPie" ```bash @@ -196,21 +196,23 @@ We can finally send some xAPI statements to Ralph LRS: === "curl" ```bash - gunzip data/statements.json.gz | \ + curl -sL https://github.com/openfun/ralph/raw/master/data/statements.json.gz | \ + gunzip | \ head -n 100 | \ jq -s . | \ curl \ --user janedoe:supersecret \ -H "Content-Type: application/json" \ -X POST \ - "http://localhost:8100/xAPI/statements" \ - -d @- + -d @- \ + "http://localhost:8100/xAPI/statements" ``` === "HTTPie" ```bash - gunzip data/statements.json.gz | \ + curl -sL https://github.com/openfun/ralph/raw/master/data/statements.json.gz | \ + gunzip | \ head -n 100 | \ jq -s . | \ http -a janedoe:supersecret POST :8100/xAPI/statements diff --git a/docs/tutorials/lrs/first-steps.md b/docs/tutorials/lrs/first-steps.md index 60fee9a11..e3e02cde6 100644 --- a/docs/tutorials/lrs/first-steps.md +++ b/docs/tutorials/lrs/first-steps.md @@ -13,22 +13,33 @@ services: lrs: image: fundocker/ralph:latest environment: - - RALPH_APP_DIR: /app/.ralph - - RALPH_RUNSERVER_BACKEND: fs + RALPH_APP_DIR: /app/.ralph + RALPH_RUNSERVER_BACKEND: fs ports: - "8100:8100" command: - "uvicorn" - "ralph.api:app" - "--proxy-headers" - - "--workers 1" - - "--host 0.0.0.0" - - "--port 8100" + - "--workers" + - "1" + - "--host" + - "0.0.0.0" + - "--port" + - "8100" + volumes: + - .ralph:/app/.ralph ``` For now, we are using the `fs` (File System) backend, meaning that Ralph LRS will store learning records in local files. -We can run Ralph LRS from a terminal with the command: +First, we need to manually create the `.ralph` directory alongside the `docker-compose.yml` file with the command: + +```bash +mkdir .ralph +``` + +We can then run Ralph LRS from a terminal with the command: ```bash docker compose up -d lrs @@ -36,7 +47,7 @@ docker compose up -d lrs Ralph LRS server should be up and running! -We can try making a request to the `whoami` endpoint, to check if the user is authenticated and returns their username and permission scopes. +We can request the `whoami` endpoint to check if the user is authenticated. On success, the endpoint returns the username and permission scopes. === "curl" @@ -44,7 +55,7 @@ We can try making a request to the `whoami` endpoint, to check if the user is au curl http://localhost:8100/whoami ``` ```console - {"detail":"Could not validate credentials"}% + {"detail":"Invalid authentication credentials"}% ``` === "HTTPie" @@ -54,17 +65,20 @@ We can try making a request to the `whoami` endpoint, to check if the user is au ``` ```console HTTP/1.1 401 Unauthorized - content-length: 43 + content-length: 47 content-type: application/json date: Mon, 06 Nov 2023 15:37:32 GMT server: uvicorn www-authenticate: Basic { - "detail": "Could not validate credentials" + "detail": "Invalid authentication credentials" } ``` -We cannot access this endpoint as we are not authenticated. -Let's shutdown Ralph LRS server with the command `docker compose down` and set up authentication (we will have to restart it later to use it).. +If you've made it this far, congrats! 🎉 + +You've successfully deployed the Ralph LRS and got a response to your request! + +Let's shutdown the Ralph LRS server with the command `docker compose down` and set up authentication. diff --git a/docs/tutorials/lrs/index.md b/docs/tutorials/lrs/index.md index 1e84f8188..ba257fcb4 100644 --- a/docs/tutorials/lrs/index.md +++ b/docs/tutorials/lrs/index.md @@ -10,7 +10,7 @@ This tutorial shows you how to run Ralph LRS, step by step. Ralph LRS is based on [FastAPI](https://fastapi.tiangolo.com/). In this tutorial, we will run the server manually with [Uvicorn](https://www.uvicorn.org/), but other alternatives exists ([Hypercorn](https://pgjones.gitlab.io/hypercorn/), [Daphne](https://github.com/django/daphne)). -!!! info Prerequisites +!!! info "Prerequisites" Some tools are required to run the commands of this tutorial. Make sure they are installed first: diff --git a/docs/tutorials/lrs/multitenancy.md b/docs/tutorials/lrs/multitenancy.md index 03d11ad4b..b1644ebfc 100644 --- a/docs/tutorials/lrs/multitenancy.md +++ b/docs/tutorials/lrs/multitenancy.md @@ -5,14 +5,14 @@ By default, all authenticated users have full read and write access to the serve ## Filtering results by authority (multitenancy) -In Ralph LRS, all incoming statements are assigned an `authority` (or ownership) derived from the user that makes the call. You may restrict read access to users "own" statements (thus enabling multitenancy) by setting the following environment variable: +In Ralph LRS, all incoming statements are assigned an `authority` (or ownership) derived from the user that makes the request. You may restrict read access to users "own" statements (thus enabling multitenancy) by setting the following environment variable: ```bash title=".env" -RALPH_LRS_RESTRICT_BY_AUTHORITY = True # Default: False +RALPH_LRS_RESTRICT_BY_AUTHORITY=True # Default: False ``` !!! warning - Two accounts with different credentials may share the same `authority` meaning they can access the same statements. It is the administrators responsability to ensure that `authority` is properly assigned. + Two accounts with different credentials may share the same `authority`, meaning they can access the same statements. It is the administrator's responsibility to ensure that `authority` is properly assigned. !!! info If not using "scopes", or for users with limited "scopes", using this option will make the use of option `?mine=True` implicit when fetching statement. @@ -23,7 +23,7 @@ In Ralph, users are assigned scopes which may be used to restrict endpoint acces functionalities. You may enable this option by setting the following environment variable: ```bash title=".env" -RALPH_LRS_RESTRICT_BY_SCOPES = True # Default: False +RALPH_LRS_RESTRICT_BY_SCOPES=True # Default: False ``` Valid scopes are a slight variation on those proposed by the diff --git a/docs/tutorials/lrs/sentry.md b/docs/tutorials/lrs/sentry.md index e8154fbe4..ded54e1d9 100644 --- a/docs/tutorials/lrs/sentry.md +++ b/docs/tutorials/lrs/sentry.md @@ -9,7 +9,7 @@ RALPH_SENTRY_DSN={PROTOCOL}://{PUBLIC_KEY}:{SECRET_KEY}@{HOST}{PATH}/{PROJECT_ID RALPH_EXECUTION_ENVIRONMENT=development ``` -The Sentry DSN (Data Source Name) can be found in your project settings from Sentry application. The execution environment should reflect the environment Ralph has been deployed in (_e.g._ `production`). +The Sentry DSN (Data Source Name) can be found in your project settings from the Sentry application. The execution environment should reflect the environment Ralph has been deployed in (_e.g._ `production`). You may also want to [monitor the performance](https://develop.sentry.dev/sdk/performance/) of Ralph by configuring the CLI and LRS traces sample rates: diff --git a/mkdocs.yml b/mkdocs.yml index 4f6857257..55a7a30fa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -48,6 +48,7 @@ nav: - Backends for data storage: backends/index.md - Learning statements models: models/index.md - Tutorials: + - Ralph CLI Guide: tutorials/cli.md - Ralph LRS Guide: - tutorials/lrs/index.md - tutorials/lrs/first-steps.md @@ -60,7 +61,6 @@ nav: - Additional configurations: - tutorials/lrs/forwarding.md - tutorials/lrs/sentry.md - - Ralph CLI Guide: tutorials/cli.md - Ralph Library Guide: tutorials/library.md - Development Guide: tutorials/development_guide.md - Contributing: contribute.md