From 541e9f91dae98b8dd3a779010f9ec3d0b227b20a Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 7 Dec 2020 15:40:56 +0100 Subject: [PATCH 1/6] First draft to unify /processes and /process_graphs #310 --- .spectral.yml | 3 +- openapi.yaml | 410 ++++++++++++++++++++++++++++++++++---------------- package.json | 2 +- 3 files changed, 283 insertions(+), 132 deletions(-) diff --git a/.spectral.yml b/.spectral.yml index 79162dfb..ef0ac1b5 100644 --- a/.spectral.yml +++ b/.spectral.yml @@ -6,7 +6,7 @@ except: - oas3-valid-schema-example "openapi.yaml#/paths/~1processes/get/responses/200/content/application~1json/example": - oas3-valid-oas-content-example - "openapi.yaml#/paths/~1process_graphs/get/responses/200/content/application~1json/example": + "openapi.yaml#/paths/~1processes~1{namespace}/get/responses/200/content/application~1json/example": - oas3-valid-oas-content-example "openapi.yaml#/paths/~1service_types/get/responses/200/content/application~1json/example": - oas3-valid-oas-content-example @@ -17,6 +17,7 @@ rules: contact-properties: true tag-description: true oas3-parameter-description: true + operation-2xx-response: off # remove this line once /process_graphs/... endpoints have been removed operation-summary-formatted: description: Operation `summary` should start with upper case and not end with a dot. recommended: true diff --git a/openapi.yaml b/openapi.yaml index d8b408b3..122a2428 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -225,7 +225,7 @@ info: Back-ends and users MAY define new proprietary processes for their domain. - **Back-end providers** MUST follow the schema for predefined processes as in [`GET /processes`](/#tag/Process-Discovery) to define new processes. This includes: + **Back-end providers** MUST follow the schema for predefined processes as in [`GET /processes`](#operation/list-processes) to define new processes. This includes: * Choosing a intuitive process id, consisting of only letters (a-z), numbers and underscores. It MUST be unique across the pre-defined processes. * Defining the parameters and their exact (JSON) schemes. @@ -233,7 +233,7 @@ info: * Providing examples or compliance tests. * Trying to make the process universally usable so that other back-end providers or openEO can adopt it. - **Users** MUST follow the schema for user-defined processes as in [`GET /process_graphs`](/#tag/User-Defined-Processes) to define new processes. This includes: + **Users** MUST follow the schema for user-defined processes as in [`GET /processes/{namespace}/{process_id}`](#operation/describe-custom-process) to define new processes. This includes: * Choosing a intuitive name as process id, consisting of only letters (a-z), numbers and underscores. It MUST be unique across the user-defined processes. * Defining the algorithm as a process graph. @@ -1525,6 +1525,21 @@ paths: - returns allOf: - $ref: '#/components/schemas/process' + namespaces: + type: array + description: |- + A list of namespaces the user has access too. + + The following namespaces are usually listed: + * `backend` + * `user`: only if authenticated + * `@` + other user id: A list of namespaces of *other* users + which share at least one process accessible by the user. + The namespace with the authenticated user id MUST NOT be listed. + + For non-authenticated users public processes are taken into account. + items: + $ref: '#/components/schemas/process_namespace' links: $ref: '#/components/schemas/links_pagination' example: @@ -1617,6 +1632,187 @@ paths: $ref: '#/components/responses/client_error_auth' 5XX: $ref: '#/components/responses/server_error' + '/processes/{namespace}': + get: + summary: List all user-defined processes in a namespace + operationId: list-custom-processes + description: |- + Lists all user-defined processes (process graphs) of the + namespace that are stored at the back-end and accessible to the user. + + It is **strongly RECOMMENDED** to keep the response size small by + omitting larger optional values from the objects in `processes` + (e.g. the `exceptions`, `examples` and `links` properties). + To get the full metadata for a user-defined process clients MUST + request `GET /processes/{namespace}/{process_id}`. + tags: + - User-Defined Processes + - Process Discovery + security: + - Bearer: [] + parameters: + - $ref: '#/components/parameters/process_namespace' + - $ref: '#/components/parameters/pagination_limit' + responses: + '200': + description: JSON array with user-defined processes. + content: + application/json: + schema: + title: User-Defined Processes + type: object + required: + - processes + - links + properties: + processes: + description: Array of user-defined processes + type: array + items: + $ref: '#/components/schemas/user_defined_process_meta' + links: + $ref: '#/components/schemas/links_pagination' + example: + processes: + - id: evi + summary: Enhanced Vegetation Index + description: >- + Computes the Enhanced Vegetation Index (EVI). + It is computed with the following formula: `2.5 * (NIR - RED) / (1 + NIR + 6*RED + -7.5*BLUE)`. + parameters: + - name: red + description: Value from the red band. + schema: + type: number + - name: blue + description: Value from the blue band. + schema: + type: number + - name: nir + description: Value from the near infrared band. + schema: + type: number + returns: + description: Computed EVI. + schema: + type: number + - id: ndsi + summary: Normalized-Difference Snow Index + parameters: + - name: green + description: Value from the Visible Green (0.53 - 0.61 micrometers) band. + schema: + type: number + - name: swir + description: Value from the Short Wave Infrared (1.55 - 1.75 micrometers) band. + schema: + type: number + returns: + schema: + type: number + - id: my_custom_process + links: [] + 4XX: + $ref: '#/components/responses/client_error_auth' + 5XX: + $ref: '#/components/responses/server_error' + '/processes/{namespace}/{process_id}': + parameters: + - $ref: '#/components/parameters/process_namespace' + - name: process_id + in: path + description: Unique identifier for a user-defined process. + required: true + schema: + $ref: '#/components/schemas/process_id' + get: + summary: Full metadata for a user-defined process + operationId: describe-custom-process + description: >- + Lists all information about a user-defined process in a namespace, + including its process graph. + tags: + - User-Defined Processes + - Process Discovery + security: + - Bearer: [] + responses: + '200': + description: The user-defined process with process graph. + content: + application/json: + schema: + title: User-Defined Process + description: A user-defined process with processing instructions as process graph. + type: object + required: + - process_graph + allOf: + - $ref: '#/components/schemas/user_defined_process_meta' + examples: + evi_user_defined_process: + $ref: '#/components/examples/evi_user_defined_process' + 4XX: + $ref: '#/components/responses/client_error_auth' + 5XX: + $ref: '#/components/responses/server_error' + put: + summary: Store a user-defined process + operationId: store-custom-process + description: |- + Stores a provided user-defined process with process graph that can be + reused in other processes. If a process with the specified + `process_graph_id` exists, the process is fully replaced. The id can't + be changed for stored user-defined processes. + + The id MUST be unique for the authenticated user, including all + pre-defined processes by the back-end. + + Partially updating user-defined processes is not supported. + + To simplify exchanging user-defined processes, the property `id` can be part of + the request body. If the values don't match, the value for `id` gets + replaced with the value from the `process_graph_id` parameter in the + path. + tags: + - User-Defined Processes + security: + - Bearer: [] + responses: + '200': + description: The user-defined process has been stored successfully. + 4XX: + $ref: '#/components/responses/client_error_auth' + 5XX: + $ref: '#/components/responses/server_error' + requestBody: + required: true + description: Specifies the process graph with its meta data. + content: + application/json: + schema: + $ref: '#/components/schemas/process_graph_with_metadata' + examples: + evi_user_defined_process: + $ref: '#/components/examples/evi_user_defined_process' + delete: + summary: Delete a user-defined process + operationId: delete-custom-process + description: |- + Deletes the data related to this user-defined process, including its process graph. + + Does NOT delete jobs or services that reference this user-defined process. + tags: + - User-Defined Processes + security: + - Bearer: [] + responses: + '204': + description: The user-defined process has been successfully deleted + 4XX: + $ref: '#/components/responses/client_error_auth' + 5XX: + $ref: '#/components/responses/server_error' /udf_runtimes: get: summary: Supported UDF runtimes @@ -2032,82 +2228,29 @@ paths: /process_graphs: get: summary: List all user-defined processes - operationId: list-custom-processes + operationId: list-custom-processes-legacy + deprecated: true description: |- - Lists all user-defined processes (process graphs) of the + Redirects to all user-defined processes of the authenticated user that are stored at the back-end. - - It is **strongly RECOMMENDED** to keep the response size small by - omitting larger optional values from the objects in `processes` - (e.g. the `exceptions`, `examples` and `links` properties). - To get the full metadata for a secondary web service clients MUST - request `GET /services/{service_id}`. tags: - User-Defined Processes - - Process Discovery security: - Bearer: [] - parameters: - - $ref: '#/components/parameters/pagination_limit' responses: - '200': - description: JSON array with user-defined processes. - content: - application/json: + '308': + description: Redirects to user processes. + headers: + Location: + required: true schema: - title: User-Defined Processes - type: object - required: - - processes - - links - properties: - processes: - description: Array of user-defined processes - type: array - items: - $ref: '#/components/schemas/user_defined_process_meta' - links: - $ref: '#/components/schemas/links_pagination' - example: - processes: - - id: evi - summary: Enhanced Vegetation Index - description: >- - Computes the Enhanced Vegetation Index (EVI). - It is computed with the following formula: `2.5 * (NIR - RED) / (1 + NIR + 6*RED + -7.5*BLUE)`. - parameters: - - name: red - description: Value from the red band. - schema: - type: number - - name: blue - description: Value from the blue band. - schema: - type: number - - name: nir - description: Value from the near infrared band. - schema: - type: number - returns: - description: Computed EVI. - schema: - type: number - - id: ndsi - summary: Normalized-Difference Snow Index - parameters: - - name: green - description: Value from the Visible Green (0.53 - 0.61 micrometers) band. - schema: - type: number - - name: swir - description: Value from the Short Wave Infrared (1.55 - 1.75 micrometers) band. - schema: - type: number - returns: - schema: - type: number - - id: my_custom_process - links: [] + description: |- + Absolute URL to the list of user-defined processes. + + The URL points to the metadata endpoint `GET /processes/user`. + format: uri + type: string + example: 'https://example.openeo.org/api/processes/user' 4XX: $ref: '#/components/responses/client_error_auth' 5XX: @@ -2122,86 +2265,48 @@ paths: $ref: '#/components/schemas/process_id' get: summary: Full metadata for a user-defined process - operationId: describe-custom-process - description: Lists all information about a user-defined process, including its process graph. + operationId: describe-custom-process-legacy + deprecated: true + description: Redirects to all information about a user-defined process. tags: - User-Defined Processes - - Process Discovery security: - Bearer: [] responses: - '200': - description: The user-defined process with process graph. - content: - application/json: - schema: - title: User-Defined Process - description: A user-defined process with processing instructions as process graph. - type: object - required: - - process_graph - allOf: - - $ref: '#/components/schemas/user_defined_process_meta' - examples: - evi_user_defined_process: - $ref: '#/components/examples/evi_user_defined_process' + '308': + $ref: '#/components/responses/redirect_udp' 4XX: $ref: '#/components/responses/client_error_auth' 5XX: $ref: '#/components/responses/server_error' put: summary: Store a user-defined process - operationId: store-custom-process - description: |- - Stores a provided user-defined process with process graph that can be - reused in other processes. If a process with the specified - `process_graph_id` exists, the process is fully replaced. The id can't - be changed for stored user-defined processes. - - The id MUST be unique for the authenticated user, including all - pre-defined processes by the back-end. - - Partially updating user-defined processes is not supported. - - To simplify exchanging user-defined processes, the property `id` can be part of - the request body. If the values don't match, the value for `id` gets - replaced with the value from the `process_graph_id` parameter in the - path. + operationId: store-custom-process-legacy + deprecated: true + description: Redirects to the endpoint that stores a user-defined process. tags: - User-Defined Processes security: - Bearer: [] responses: - '200': - description: The user-defined process has been stored successfully. + '308': + $ref: '#/components/responses/redirect_udp' 4XX: $ref: '#/components/responses/client_error_auth' 5XX: $ref: '#/components/responses/server_error' - requestBody: - required: true - description: Specifies the process graph with its meta data. - content: - application/json: - schema: - $ref: '#/components/schemas/process_graph_with_metadata' - examples: - evi_user_defined_process: - $ref: '#/components/examples/evi_user_defined_process' delete: summary: Delete a user-defined process - operationId: delete-custom-process - description: |- - Deletes the data related to this user-defined process, including its process graph. - - Does NOT delete jobs or services that reference this user-defined process. + operationId: delete-custom-process-legacy + deprecated: true + description: Redirects to the endpoint that deletes a user-defined process. tags: - User-Defined Processes security: - Bearer: [] responses: - '204': - description: The user-defined process has been successfully deleted + '308': + $ref: '#/components/responses/redirect_udp' 4XX: $ref: '#/components/responses/client_error_auth' 5XX: @@ -4537,7 +4642,7 @@ components: process_id: $ref: '#/components/schemas/process_id' namespace: - $ref: '#/components/schemas/process_namespace' + $ref: '#/components/schemas/process_graph_namespace' result: type: boolean description: >- @@ -4765,7 +4870,7 @@ components: nullable: true allOf: - $ref: '#/components/schemas/process' - process_namespace: + process_graph_namespace: type: string nullable: true default: null @@ -4783,10 +4888,29 @@ components: that have a process graph included. It is RECOMMENDED to log the namespace selected by the back-end for debugging purposes. * `backend`: Uses exclusively the pre-defined processes listed at `GET /processes`. - * `user`: Uses exclusively the user-defined processes listed at `GET /process_graphs`. + * `user`: Uses exclusively the user-defined processes listed at `GET /processes/@{user_id}` + with `{user_id}` being the user id of the authenticated user. This is an alias for + directly specifying `@{user_id}` as namespace. If multiple processes with the same identifier exist, Clients SHOULD inform the user that it's recommended to select a namespace. + process_namespace: + type: string + description: |- + The namespace for a process. + + The API only defines a single type of predefined namespaces. That is for + user-defined processes stored at the back-end and defines a distinct + namespace for each individual user. They are prescribed with a leading + `@` followed by the unique identifier of a user. + + The namespace `user` is an alias for `@` + user id of the authenticated user. + It can be used to retrieve the user-defined processes of the authenticated user. + + The namespace `backend` is an alias for predefined processes. + + Back-end implementations MAY implement other namespaces that don't + conflict with any of the namespaces mentioned above. process_id: type: string description: |- @@ -5053,10 +5177,13 @@ components: example: '2017-01-01T09:36:18Z' user_id: type: string - description: >- + description: |- Unique identifier of the user. MUST match the specified pattern. - This is usually a randomly generated internal identifier from the provider - not meant for displaying purposes. + + For ease of use, it is NOT RECOMMENDED to use long randomly generated + identifiers. More readable user identifiers like `john_doe` support a + better user-experience as the user identifier is used in URIs for shared + processes, e.g. `https://example.org/api/v1.0/processes/@john_doe/my_ndvi`. pattern: '^[\w\-\.~]+$' example: john_doe description: @@ -5609,7 +5736,7 @@ components: process_id: $ref: '#/components/schemas/process_id' namespace: - $ref: '#/components/schemas/process_namespace' + $ref: '#/components/schemas/process_graph_namespace' parameter: type: string description: >- @@ -5620,6 +5747,22 @@ components: links: $ref: '#/components/schemas/log_links' responses: + redirect_udp: + description: Redirects to the endpoint for a specific user-defined process. + headers: + Location: + required: true + schema: + description: |- + Absolute URL to a specific user-defined process. + + The URL points to the endpoint `/processes/user/{process_id}` and + MUST be requested with the same HTTP method, headers and request + body of this request. `{process_id}` MUST be the value given with + this request in the parameter `{process_graph_id}`. + format: uri + type: string + example: 'https://example.openeo.org/api/processes/user/my-ndvi' logs: description: Lists the requested log entries. content: @@ -5745,6 +5888,13 @@ components: required: true schema: $ref: '#/components/schemas/job_id' + process_namespace: + name: namespace + in: path + description: A process namespace. + required: true + schema: + $ref: '#/components/schemas/process_namespace' examples: evi_user_defined_process: description: A user-defined process that computes the EVI. diff --git a/package.json b/package.json index 6e93e108..39edcdf8 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ }, "devDependencies": { "@stoplight/spectral": "5.3.0", - "redoc-cli": "^0.9.13" + "redoc-cli": "0.10.1" }, "scripts": { "start": "redoc-cli serve openapi.yaml --watch --options.expandResponses \"200,201,202,203,204\" --options.pathInMiddlePanel true", From c7ae6bb7eb425b1726c369fc1c9bbaf803f62679 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Wed, 12 May 2021 16:20:44 +0200 Subject: [PATCH 2/6] pre-defined => predefined, backend => back-end --- CHANGELOG.md | 4 ++-- errors.json | 6 +++--- openapi.yaml | 42 +++++++++++++++++++++--------------------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6ddcfb9..9580c068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,9 +83,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Additional dimensions in `cube:dimensions` can only be of type `other`. - The extents `interval` and `bbox` can have multiple entries. - Allow all STAC versions that are compatible to STAC 0.9.0. -- Process graph nodes have an additional field `namespace` to distinguish pre-defined and user-defined processes. The default behavior has not changed. [#305](https://github.com/Open-EO/openeo-api/issues/305) +- Process graph nodes have an additional field `namespace` to distinguish predefined and user-defined processes. The default behavior has not changed. [#305](https://github.com/Open-EO/openeo-api/issues/305) - Added `format: commonmark` to all properties supporting CommonMark formatting. -- `errors.json`: The pre-defined error messages have been reworked. [#272](https://github.com/Open-EO/openeo-api/issues/272), [#273](https://github.com/Open-EO/openeo-api/issues/273) +- `errors.json`: The predefined error messages have been reworked. [#272](https://github.com/Open-EO/openeo-api/issues/272), [#273](https://github.com/Open-EO/openeo-api/issues/273) - Added `FolderOperationUnsupported`, `UnsupportedApiVersion`, `PermissionsInsufficient`, `ProcessGraphIdDoesntMatch` and `PredefinedProcessExists`. - Added variable `reason` to error `FilePathInvalid` and `type` to `FileTypeInvalid` and`ServiceUnsupported`. - Replaced the following error messages. The variables in the messages may have changed, too. diff --git a/errors.json b/errors.json index c3b9e2f7..175bed0a 100644 --- a/errors.json +++ b/errors.json @@ -211,8 +211,8 @@ ] }, "PredefinedProcessExists": { - "description": "If a user wants to store a user-defined process with the id of a pre-defined process.", - "message": "A pre-defined process with the given identifier exists.", + "description": "If a user wants to store a user-defined process with the id of a predefined process.", + "message": "A predefined process with the given identifier exists.", "http": 400, "tags": [ "User-Defined Processes" @@ -227,7 +227,7 @@ ] }, "ProcessUnsupported": { - "description": "A process (pre-defined or user-defined) with the specified identifier is not available. To be used when validating or executing process graphs.", + "description": "A process (predefined or user-defined) with the specified identifier is not available. To be used when validating or executing process graphs.", "message": "Process with identifier '{process}' is not available in namespace '{namespace}'.", "http": 400, "tags": [ diff --git a/openapi.yaml b/openapi.yaml index 3f1a9803..4e58ebf6 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -214,12 +214,12 @@ info: A **process** is an operation that performs a specific task on a set of parameters and returns a result. An example is computing a statistical operation, such as mean or median, on selected EO data. A process is similar to a function or method in programming languages. In openEO, processes are used to build a chain of processes ([process graph](#section/Processes/Process-Graphs)), which can be applied to EO data to derive your own findings from the data. - A **pre-defined process** is a process provided by the *back-end*. There is a set of predefined processes by openEO to improve interoperability between back-ends. - Back-ends SHOULD follow these specifications whenever possible. Not all processes need to be implemented by all back-ends. See the **[process reference](https://processes.openeo.org)** for pre-defined processes. + A **predefined process** is a process provided by the *back-end*. There is a set of predefined processes by openEO to improve interoperability between back-ends. + Back-ends SHOULD follow these specifications whenever possible. Not all processes need to be implemented by all back-ends. See the **[process reference](https://processes.openeo.org)** for predefined processes. A **user-defined process** is a process defined by the *user*. It can directly be part of another process graph or be stored as custom process on a back-end. Internally it is a *process graph* with optional additional metadata. - A **process graph** chains specific process calls from the set of pre-defined and user-defined processes together. A process graph itself can be stored as a (user-defined) process again. Similarly to scripts in the context of programming, process graphs organize and automate the execution of one or more processes that could alternatively be executed individually. In a process graph, processes need to be specific, i.e. concrete values or "placeholders" for input parameters need to be specified. These values can be scalars, arrays, objects, references to parameters or previous computations or other process graphs. + A **process graph** chains specific process calls from the set of predefined and user-defined processes together. A process graph itself can be stored as a (user-defined) process again. Similarly to scripts in the context of programming, process graphs organize and automate the execution of one or more processes that could alternatively be executed individually. In a process graph, processes need to be specific, i.e. concrete values or "placeholders" for input parameters need to be specified. These values can be scalars, arrays, objects, references to parameters or previous computations or other process graphs. ## Defining Processes @@ -227,7 +227,7 @@ info: **Back-end providers** MUST follow the schema for predefined processes as in [`GET /processes`](#operation/list-processes) to define new processes. This includes: - * Choosing a intuitive process id, consisting of only letters (a-z), numbers and underscores. It MUST be unique across the pre-defined processes. + * Choosing a intuitive process id, consisting of only letters (a-z), numbers and underscores. It MUST be unique across the predefined processes. * Defining the parameters and their exact (JSON) schemes. * Specifying the return value of a process also with a (JSON) schema. * Providing examples or compliance tests. @@ -239,7 +239,7 @@ info: * Defining the algorithm as a process graph. * Optionally, specifying the additional metadata for processes. - If new process are potentially useful for other back-ends the openEO consortium is happily accepting [pull requests](https://github.com/Open-EO/openeo-processes/pulls) to include them in the list of pre-defined processes. + If new process are potentially useful for other back-ends the openEO consortium is happily accepting [pull requests](https://github.com/Open-EO/openeo-processes/pulls) to include them in the list of predefined processes. ### Schemas @@ -300,7 +300,7 @@ info: One of the nodes in a map of processes (the final one) MUST have the `result` flag set to `true`, all the other nodes can omit it as the default value is `false`. Having such a node is important as multiple end nodes are possible, but in most use cases it is important to exactly specify the return value to be used by other processes. Each child process graph must also specify a result node similar to the "main" process graph. - `process_id` MUST be a valid process ID in the `namespace` given. Clients SHOULD warn the user if a user-defined process is added with the same identifier as one of the pre-defined process. + `process_id` MUST be a valid process ID in the `namespace` given. Clients SHOULD warn the user if a user-defined process is added with the same identifier as one of the predefined process. ### Arguments @@ -347,7 +347,7 @@ info: consists of a single process `absolute`, but it can be arbitrary complex in general. A `` argument MUST at least consist of an object with a key `process_graph`. - Optionally, it can also be described with the same additional properties available for pre-defined processes such as an id, parameters, return values etc. + Optionally, it can also be described with the same additional properties available for predefined processes such as an id, parameters, return values etc. When embedded in a process graph, these additional properties of a user-defined process are usually not used, except for validation purposes. ``` @@ -1522,8 +1522,8 @@ paths: processes: type: array items: - title: Pre-Defined Process - description: A pre-defined process made available by the back-end. + title: Predefined Process + description: A predefined process made available by the back-end. type: object required: - id @@ -1773,7 +1773,7 @@ paths: be changed for stored user-defined processes. The id MUST be unique for the authenticated user, including all - pre-defined processes by the back-end. + predefined processes by the back-end. Partially updating user-defined processes is not supported. @@ -2026,13 +2026,13 @@ paths: List of default OpenID Connect clients that can be used by an openEO client for OpenID Connect based authentication. - A default OpenID Connect client is managed by the backend implementer. + A default OpenID Connect client is managed by the back-end implementer. It MUST be configured to be usable without a client secret, which limits its applicability to OpenID Connect grant types like "Authorization Code Grant with PKCE" and "Device Authorization Grant with PKCE" A default OpenID Connect client is provided without availability guarantees. - The backend implementer CAN revoke, reset or update it any time. + The back-end implementer CAN revoke, reset or update it any time. As such, openEO clients SHOULD NOT store or cache default OpenID Connect client information for long term usage. A default OpenID Connect client is intended to simplify authentication for novice users. @@ -3619,7 +3619,7 @@ components: description: |- The type of the UDF runtime. - Pre-defined types are: + Predefined types are: * `language` for Programming Languages and * `docker` for Docker Containers. @@ -5105,16 +5105,16 @@ components: description: |- The namespace the `process_id` is valid for. - The following options are pre-defined by the openEO API, but additional + The following options are predefined by the openEO API, but additional namespaces may be introduced by back-ends or in a future version of the API. - * `null` (default): Checks both user-defined and pre-defined processes, + * `null` (default): Checks both user-defined and predefined processes, but prefers user-defined processes if both are available. - This allows users to add missing pre-defined processes for portability, + This allows users to add missing predefined processes for portability, e.g. common processes from [processes.openeo.org](https://processes.openeo.org) that have a process graph included. It is RECOMMENDED to log the namespace selected by the back-end for debugging purposes. - * `backend`: Uses exclusively the pre-defined processes listed at `GET /processes`. + * `backend`: Uses exclusively the predefined processes listed at `GET /processes`. * `user`: Uses exclusively the user-defined processes listed at `GET /processes/@{user_id}` with `{user_id}` being the user id of the authenticated user. This is an alias for directly specifying `@{user_id}` as namespace. @@ -5142,10 +5142,10 @@ components: type: string description: |- The identifier for the process. It MUST be unique across its namespace - (e.g. pre-defined processes or user-defined processes). + (e.g. predefined processes or user-defined processes). Clients SHOULD warn the user if a user-defined process is added with the - same identifier as one of the pre-defined process. + same identifier as one of the predefined process. pattern: '^\w+$' example: ndvi process_summary: @@ -6198,9 +6198,9 @@ components: pagination MUST return all resources. If the response is paginated, the `links` array MUST be used to communicate the - links for browsing the pagination with pre-defined `rel` types. See the `links` array schema + links for browsing the pagination with predefined `rel` types. See the `links` array schema for supported `rel` types. - Backend implementations can, unless specified otherwise, use all kind of pagination techniques, + Back-end implementations can, unless specified otherwise, use all kind of pagination techniques, depending on what is supported best by their infrastructure: page-based, offset-based, token-based or something else. The clients SHOULD use whatever is specified in the links with the corresponding `rel` types. From f035421a48d7f4ee3517a47555550fb35ddd79d2 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Wed, 12 May 2021 16:33:00 +0200 Subject: [PATCH 3/6] Make the specification more consistent, don't allow the back-end namespace right now. --- openapi.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 4e58ebf6..051cb2cd 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1535,15 +1535,18 @@ paths: namespaces: type: array description: |- - A list of namespaces the user has access too. + A list of all namespaces of user-defined processes, + which the user has access too. The following namespaces are usually listed: - * `backend` * `user`: only if authenticated * `@` + other user id: A list of namespaces of *other* users which share at least one process accessible by the user. The namespace with the authenticated user id MUST NOT be listed. + The namespace `backend` for predefined processes MUST not be listed. + Predefined processes are exposed via `/processes` only. + For non-authenticated users public processes are taken into account. items: $ref: '#/components/schemas/process_namespace' From daa18ebc852fb5a06e872138efda23f816a60774 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Tue, 17 Aug 2021 14:55:30 +0200 Subject: [PATCH 4/6] Add namespace pattern, fix authentication details --- openapi.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openapi.yaml b/openapi.yaml index 805f4164..94ffc2b7 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1682,6 +1682,7 @@ paths: - User-Defined Processes - Process Discovery security: + - {} - Bearer: [] parameters: - $ref: '#/components/parameters/process_namespace' @@ -1768,6 +1769,7 @@ paths: - User-Defined Processes - Process Discovery security: + - {} - Bearer: [] responses: '200': @@ -5200,6 +5202,7 @@ components: inform the user that it's recommended to select a namespace. process_namespace: type: string + pattern: ^@?[\w\-\.~]+$ description: |- The namespace for a process. From e5ac184a6259b83aa0b36fd5e00b272d4fd5d7a9 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Tue, 17 Aug 2021 15:01:23 +0200 Subject: [PATCH 5/6] pre-defined => predefined, backend => back-end --- CHANGELOG.md | 4 ++-- errors.json | 6 +++--- openapi.yaml | 40 ++++++++++++++++++++-------------------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 235d35f2..e258cfff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,9 +105,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Additional dimensions in `cube:dimensions` can only be of type `other`. - The extents `interval` and `bbox` can have multiple entries. - Allow all STAC versions that are compatible to STAC 0.9.0. -- Process graph nodes have an additional field `namespace` to distinguish pre-defined and user-defined processes. The default behavior has not changed. [#305](https://github.com/Open-EO/openeo-api/issues/305) +- Process graph nodes have an additional field `namespace` to distinguish predefined and user-defined processes. The default behavior has not changed. [#305](https://github.com/Open-EO/openeo-api/issues/305) - Added `format: commonmark` to all properties supporting CommonMark formatting. -- `errors.json`: The pre-defined error messages have been reworked. [#272](https://github.com/Open-EO/openeo-api/issues/272), [#273](https://github.com/Open-EO/openeo-api/issues/273) +- `errors.json`: The predefined error messages have been reworked. [#272](https://github.com/Open-EO/openeo-api/issues/272), [#273](https://github.com/Open-EO/openeo-api/issues/273) - Added `FolderOperationUnsupported`, `UnsupportedApiVersion`, `PermissionsInsufficient`, `ProcessGraphIdDoesntMatch` and `PredefinedProcessExists`. - Added variable `reason` to error `FilePathInvalid` and `type` to `FileTypeInvalid` and`ServiceUnsupported`. - Replaced the following error messages. The variables in the messages may have changed, too. diff --git a/errors.json b/errors.json index 276d95a5..fa4ad5bf 100644 --- a/errors.json +++ b/errors.json @@ -225,8 +225,8 @@ ] }, "PredefinedProcessExists": { - "description": "If a user wants to store a user-defined process with the id of a pre-defined process.", - "message": "A pre-defined process with the given identifier exists.", + "description": "If a user wants to store a user-defined process with the id of a predefined process.", + "message": "A predefined process with the given identifier exists.", "http": 400, "tags": [ "User-Defined Processes" @@ -241,7 +241,7 @@ ] }, "ProcessUnsupported": { - "description": "A process (pre-defined or user-defined) with the specified identifier is not available. To be used when validating or executing process graphs.", + "description": "A process (predefined or user-defined) with the specified identifier is not available. To be used when validating or executing process graphs.", "message": "Process with identifier '{process}' is not available in namespace '{namespace}'.", "http": 400, "tags": [ diff --git a/openapi.yaml b/openapi.yaml index 0a006747..e93c5280 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -216,12 +216,12 @@ info: A **process** is an operation that performs a specific task on a set of parameters and returns a result. An example is computing a statistical operation, such as mean or median, on selected EO data. A process is similar to a function or method in programming languages. In openEO, processes are used to build a chain of processes ([process graph](#section/Processes/Process-Graphs)), which can be applied to EO data to derive your own findings from the data. - A **pre-defined process** is a process provided by the *back-end*. There is a set of predefined processes by openEO to improve interoperability between back-ends. - Back-ends SHOULD follow these specifications whenever possible. Not all processes need to be implemented by all back-ends. See the **[process reference](https://processes.openeo.org)** for pre-defined processes. + A **predefined process** is a process provided by the *back-end*. There is a set of predefined processes by openEO to improve interoperability between back-ends. + Back-ends SHOULD follow these specifications whenever possible. Not all processes need to be implemented by all back-ends. See the **[process reference](https://processes.openeo.org)** for predefined processes. A **user-defined process** is a process defined by the *user*. It can directly be part of another process graph or be stored as custom process on a back-end. Internally it is a *process graph* with optional additional metadata. - A **process graph** chains specific process calls from the set of pre-defined and user-defined processes together. A process graph itself can be stored as a (user-defined) process again. Similarly to scripts in the context of programming, process graphs organize and automate the execution of one or more processes that could alternatively be executed individually. In a process graph, processes need to be specific, i.e. concrete values or "placeholders" for input parameters need to be specified. These values can be scalars, arrays, objects, references to parameters or previous computations or other process graphs. + A **process graph** chains specific process calls from the set of predefined and user-defined processes together. A process graph itself can be stored as a (user-defined) process again. Similarly to scripts in the context of programming, process graphs organize and automate the execution of one or more processes that could alternatively be executed individually. In a process graph, processes need to be specific, i.e. concrete values or "placeholders" for input parameters need to be specified. These values can be scalars, arrays, objects, references to parameters or previous computations or other process graphs. ## Defining Processes @@ -229,7 +229,7 @@ info: **Back-end providers** MUST follow the schema for predefined processes as in [`GET /processes`](#operation/list-processes) to define new processes. This includes: - * Choosing a intuitive process id, consisting of only letters (a-z), numbers and underscores. It MUST be unique across the pre-defined processes. + * Choosing a intuitive process id, consisting of only letters (a-z), numbers and underscores. It MUST be unique across the predefined processes. * Defining the parameters and their exact (JSON) schemes. * Specifying the return value of a process also with a (JSON) schema. * Providing examples or compliance tests. @@ -241,7 +241,7 @@ info: * Defining the algorithm as a process graph. * Optionally, specifying the additional metadata for processes. - If new process are potentially useful for other back-ends the openEO consortium is happily accepting [pull requests](https://github.com/Open-EO/openeo-processes/pulls) to include them in the list of pre-defined processes. + If new process are potentially useful for other back-ends the openEO consortium is happily accepting [pull requests](https://github.com/Open-EO/openeo-processes/pulls) to include them in the list of predefined processes. ### Schemas @@ -302,7 +302,7 @@ info: One of the nodes in a map of processes (the final one) MUST have the `result` flag set to `true`, all the other nodes can omit it as the default value is `false`. Having such a node is important as multiple end nodes are possible, but in most use cases it is important to exactly specify the return value to be used by other processes. Each child process graph must also specify a result node similar to the "main" process graph. - `process_id` MUST be a valid process ID in the `namespace` given. Clients SHOULD warn the user if a user-defined process is added with the same identifier as one of the pre-defined process. + `process_id` MUST be a valid process ID in the `namespace` given. Clients SHOULD warn the user if a user-defined process is added with the same identifier as one of the predefined process. ### Arguments @@ -349,7 +349,7 @@ info: consists of a single process `absolute`, but it can be arbitrary complex in general. A `` argument MUST at least consist of an object with a key `process_graph`. - Optionally, it can also be described with the same additional properties available for pre-defined processes such as an id, parameters, return values etc. + Optionally, it can also be described with the same additional properties available for predefined processes such as an id, parameters, return values etc. When embedded in a process graph, these additional properties of a user-defined process are usually not used, except for validation purposes. ``` @@ -1546,7 +1546,7 @@ paths: type: array items: title: Pre-Defined Process - description: A pre-defined process made available by the back-end. + description: A predefined process made available by the back-end. type: object required: - id @@ -1853,13 +1853,13 @@ paths: List of default OpenID Connect clients that can be used by an openEO client for OpenID Connect based authentication. - A default OpenID Connect client is managed by the backend implementer. + A default OpenID Connect client is managed by the back-end implementer. It MUST be configured to be usable without a client secret, which limits its applicability to OpenID Connect grant types like "Authorization Code Grant with PKCE" and "Device Authorization Grant with PKCE" A default OpenID Connect client is provided without availability guarantees. - The backend implementer CAN revoke, reset or update it any time. + The back-end implementer CAN revoke, reset or update it any time. As such, openEO clients SHOULD NOT store or cache default OpenID Connect client information for long term usage. A default OpenID Connect client is intended to simplify authentication for novice users. @@ -2281,7 +2281,7 @@ paths: be changed for stored user-defined processes. The id MUST be unique for the authenticated user, including all - pre-defined processes by the back-end. + predefined processes by the back-end. Partially updating user-defined processes is not supported. @@ -3563,7 +3563,7 @@ components: description: |- The type of the UDF runtime. - Pre-defined types are: + Predefined types are: * `language` for Programming Languages and * `docker` for Docker Containers. @@ -5071,16 +5071,16 @@ components: description: |- The namespace the `process_id` is valid for. - The following options are pre-defined by the openEO API, but additional + The following options are predefined by the openEO API, but additional namespaces may be introduced by back-ends or in a future version of the API. - * `null` (default): Checks both user-defined and pre-defined processes, + * `null` (default): Checks both user-defined and predefined processes, but prefers user-defined processes if both are available. - This allows users to add missing pre-defined processes for portability, + This allows users to add missing predefined processes for portability, e.g. common processes from [processes.openeo.org](https://processes.openeo.org) that have a process graph included. It is RECOMMENDED to log the namespace selected by the back-end for debugging purposes. - * `backend`: Uses exclusively the pre-defined processes listed at `GET /processes`. + * `backend`: Uses exclusively the predefined processes listed at `GET /processes`. * `user`: Uses exclusively the user-defined processes listed at `GET /process_graphs`. If multiple processes with the same identifier exist, Clients SHOULD @@ -5089,10 +5089,10 @@ components: type: string description: |- The identifier for the process. It MUST be unique across its namespace - (e.g. pre-defined processes or user-defined processes). + (e.g. predefined processes or user-defined processes). Clients SHOULD warn the user if a user-defined process is added with the - same identifier as one of the pre-defined process. + same identifier as one of the predefined process. pattern: '^\w+$' example: ndvi process_summary: @@ -6118,9 +6118,9 @@ components: pagination MUST return all resources. If the response is paginated, the `links` array MUST be used to communicate the - links for browsing the pagination with pre-defined `rel` types. See the `links` array schema + links for browsing the pagination with predefined `rel` types. See the `links` array schema for supported `rel` types. - Backend implementations can, unless specified otherwise, use all kind of pagination techniques, + Back-end implementations can, unless specified otherwise, use all kind of pagination techniques, depending on what is supported best by their infrastructure: page-based, offset-based, token-based or something else. The clients SHOULD use whatever is specified in the links with the corresponding `rel` types. From 738e327347438a1a9f91c9ecec01f9375c430ca7 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Tue, 17 Aug 2021 15:36:22 +0200 Subject: [PATCH 6/6] Rephrased canonical link description #408 --- openapi.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 94ffc2b7..e240b925 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4399,8 +4399,9 @@ components: catalog service such as OGC CSW, a human-readable HTML version or a metadata document following another standard such as ISO 19115 or DCAT. - 6. `canonical`: This relation type points to a (permanent) URL for resources which - usually require authentication but can can be accessed without (Bearer) authentication. + 6. `canonical`: This relation type usually points to publicly accessible + and more long-lived URL for resources that often require (Bearer) + authentication with a short-lived token. This way the the exposed resources can be used by non-openEO clients without additional authentication steps. For example, a shared user-defined process or batch job results could be