From 99cc4c11fdd94099dd265db5cdfc670883edef14 Mon Sep 17 00:00:00 2001 From: Zetas Menelaos - x_menzetas Date: Wed, 17 Jul 2024 10:35:20 +0300 Subject: [PATCH] release 2.0.1 --- .gitignore | Bin 0 -> 110 bytes .gitlab-ci.yml | 10 + ...ICOS_intelligence_layer_metrics_export.iml | 11 + .idea/inspectionProfiles/Project_Default.xml | 16 + .idea/misc.xml | 6 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 64 +++ Dockerfile | 20 + ICOS_intelligence_layer_metrics_export.iml | 9 + LICENSE | 202 +++++++ NOTICE | 4 + README.md | 178 ++++++ __pycache__/metric_helpers.cpython-311.pyc | Bin 0 -> 2411 bytes .../metric_types_functions.cpython-311.pyc | Bin 0 -> 7320 bytes entrypoint.sh | 7 + .../.helmignore | 23 + .../Chart.yaml | 42 ++ .../templates/_helpers.tpl | 82 +++ .../templates/deployment.yaml | 29 + .../templates/service.yaml | 55 ++ .../templates/tests/test-connection.yaml | 15 + .../values.yaml | 131 +++++ requirements.txt | Bin 0 -> 746 bytes .../environment_variables.cpython-311.pyc | Bin 0 -> 476 bytes .../metric_helpers.cpython-311.pyc | Bin 0 -> 3122 bytes .../metric_types_functions.cpython-311.pyc | Bin 0 -> 7332 bytes .../metrics_generator.cpython-311.pyc | Bin 0 -> 20827 bytes ...step1_querry_to_premetheus.cpython-311.pyc | Bin 0 -> 3301 bytes ...p2_intelligence_layer_call.cpython-311.pyc | Bin 0 -> 2995 bytes src/environment_variables.py | 4 + src/metric_helpers.py | 62 +++ src/metric_types_functions.py | 158 ++++++ src/metrics_generator.py | 514 ++++++++++++++++++ src/step1_querry_to_premetheus.py | 77 +++ src/step2_intelligence_layer_call.py | 58 ++ 36 files changed, 1791 insertions(+) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 .idea/ICOS_intelligence_layer_metrics_export.iml create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 Dockerfile create mode 100644 ICOS_intelligence_layer_metrics_export.iml create mode 100644 LICENSE create mode 100644 NOTICE create mode 100644 README.md create mode 100644 __pycache__/metric_helpers.cpython-311.pyc create mode 100644 __pycache__/metric_types_functions.cpython-311.pyc create mode 100644 entrypoint.sh create mode 100644 icos-export-custom-metrics-to-prometheus/.helmignore create mode 100644 icos-export-custom-metrics-to-prometheus/Chart.yaml create mode 100644 icos-export-custom-metrics-to-prometheus/templates/_helpers.tpl create mode 100644 icos-export-custom-metrics-to-prometheus/templates/deployment.yaml create mode 100644 icos-export-custom-metrics-to-prometheus/templates/service.yaml create mode 100644 icos-export-custom-metrics-to-prometheus/templates/tests/test-connection.yaml create mode 100644 icos-export-custom-metrics-to-prometheus/values.yaml create mode 100644 requirements.txt create mode 100644 src/__pycache__/environment_variables.cpython-311.pyc create mode 100644 src/__pycache__/metric_helpers.cpython-311.pyc create mode 100644 src/__pycache__/metric_types_functions.cpython-311.pyc create mode 100644 src/__pycache__/metrics_generator.cpython-311.pyc create mode 100644 src/__pycache__/step1_querry_to_premetheus.cpython-311.pyc create mode 100644 src/__pycache__/step2_intelligence_layer_call.cpython-311.pyc create mode 100644 src/environment_variables.py create mode 100644 src/metric_helpers.py create mode 100644 src/metric_types_functions.py create mode 100644 src/metrics_generator.py create mode 100644 src/step1_querry_to_premetheus.py create mode 100644 src/step2_intelligence_layer_call.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..ad5e2a3cf431f56fc270199530b259c5baf158cd GIT binary patch literal 110 zcmezWPmdv!A%!88A(26!ftP`c0hzDGpam9-XNYGgV5nqB2C7PC$N;Jai9pnA0d+v+ TK)Ur&^g~3-fa>!Y%7AhJ6uuD+ literal 0 HcmV?d00001 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..3b2141e --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,10 @@ +include: + - project: 'resengit/gitlab/pipeline-helpers' + ref: 'main' + file: '/pipelines/helm-chart.yaml' + - project: 'resengit/gitlab/pipeline-helpers' + ref: main + file: '/pipelines/docker-image.yaml' + +variables: + PH_HELM_CHART_FOLDER: icos-export-custom-metrics-to-prometheus/ diff --git a/.idea/ICOS_intelligence_layer_metrics_export.iml b/.idea/ICOS_intelligence_layer_metrics_export.iml new file mode 100644 index 0000000..2cdb1e3 --- /dev/null +++ b/.idea/ICOS_intelligence_layer_metrics_export.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..662addd --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,16 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..66b6536 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..419c999 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..4e928e7 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + { + "keyToString": { + "RunOnceActivity.OpenProjectViewOnStart": "true", + "RunOnceActivity.ShowReadmeOnStart": "true", + "project.structure.last.edited": "Project", + "project.structure.proportion": "0.0", + "project.structure.side.proportion": "0.0" + } +} + + + + + 1721164033623 + + + 1721164913284 + + + + + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e49cc86 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +# Use the official Python 3.11 image as the base image +FROM python:3.11 +USER root + +# Set the working directory in the container +WORKDIR /usr/src/ + +# Copy the dependencies file to the container +COPY requirements.txt . + +# Copy the current directory contents into the container at /app +COPY . . + +# Install any needed packages specified in requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +# Use an entrypoint script to start Gunicorn +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/ICOS_intelligence_layer_metrics_export.iml b/ICOS_intelligence_layer_metrics_export.iml new file mode 100644 index 0000000..26de821 --- /dev/null +++ b/ICOS_intelligence_layer_metrics_export.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1d3c5da --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022-2024 National and Kapodistrian University of Athens + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..f21ae90 --- /dev/null +++ b/NOTICE @@ -0,0 +1,4 @@ +ICOS Metrics Export to Prometheus +Copyright © 2022-2024 National and Kapodistrian University of Athens + +🇪🇺 This work has received funding from the European Union's HORIZON research and innovation programme under grant agreement No. 101070177. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2c798d3 --- /dev/null +++ b/README.md @@ -0,0 +1,178 @@ +# ICOS Metrics Export to Prometheus + +## Introduction +This project provides a metrics export layer for ICOS, facilitating the monitoring and analysis of various +metrics. Built on top of the `prometheus_client` library, it offers a lightweight and efficient way to expose metrics to +Prometheus. + +## Prerequisites +Before you start, ensure you have Python installed on your system. This project uses the `prometheus_client` library to +expose metrics to Prometheus, so make sure to install it using pip: + +```bash +pip install prometheus_client +``` + +or by installing all the requirements: + +```bash +pip install -r requirements.txt +``` + +## Quick Start +To get started with the ICOS Metrics Export to Prometheus, simply run the metrics_generator.py script. The script +provides three routes for metrics exposure and removal: + +1) `/metrics`: This route will be used from Prometheus to scrape metrics. It exposes all the collected metrics in a format +that Prometheus can understand and collect. + +2) `/unregister_metric`: This route can be used to delete/unregister a metric created. It accepts a json payload that must contain: + 1) `metric_name` (mandatory): The name of the metric to be deleted/unregistered. + +3) `/create_metric`: This route can be configured to create and update metrics, tailored to specific monitoring +needs (type of metrics). It accepts a json payload that must contain: + 1) `type` (mandatory): The metric type: + - Counter = 1 + - Gauge = 2 + - Info = 3 + - Enum = 4 + 2) `metric_name`(mandatory): The name of the metric to be created or retrieved. + 3) `metric_info` (optional): The info of the metric to be created or retrieved. + 4) `value` (mandatory): The value that will be passed to the metric. + 5) `labels` (optional): The dictionary of labels that will be set for the metric. + 6) `states` (optional): The list of states if an Enum metric is being set for the first time. + + After getting the properties it creates the specific metric asked and registers it to the internal registry. According to the metric type value: + - Counter = 1 + Counter expects: + - metric_name (mandatory) -> string. If there is a suffix of _total on the metric name, it will be removed. + When exposing the time series for counter, a _total suffix will be added. This is for compatibility between + OpenMetrics and the Prometheus text format, as OpenMetrics requires the _total suffix. + - metric_info (optional) -> string | None. + - value (mandatory): the previous stored value will be incremented with that value -> positive number. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - Gauge = 2 + Gauge expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - value (mandatory): the new value that will be set -> Union[float, str] (must be a parsable to float + string.). + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - Info = 3 + Info expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - value (mandatory): the new value that will be set -> Dict[str, str | float]. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored), + - Enum = 4 + Enum expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - value (mandatory): the state that will be set. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (mandatory at creation of metric): the states that will be the available choice to set the state + (passed only the first time) + +4) `/create_model_metric`: This route will receive a json payload to create a metric based on specific telemetry data + that will be retrieved and a model that must exist at Intelligence layer. The metric created will be + tailored to specific monitoring needs (type of metrics). It accepts a json payload that must contain: + 1) `type` (mandatory): The metric type: + - Counter = 1 + - Gauge = 2 + - Info = 3 + - Enum = 4 + 2) `metric_name`(mandatory): The name of the metric to be created or retrieved. + 3) `metric_info` (optional): The info of the metric to be created or retrieved. + 4) `labels` (optional): The dictionary of labels that will be set for the metric. + 5) `states` (optional): The list of states if an Enum metric is being set for the first time. + 6) `telemetry_metric` (mandatory): The query of the telemetry metric from witch data will be retrieved. + 7) `model_route` (mandatory): The route of the model where it can be inferred from Intelligence API. + 8) `model_name` (mandatory): The name of the model where the retrieved telemetry data will be sent. + 9) `model_type` (mandatory): The type of the model where the retrieved telemetry data will be sent. + 10) `step_in_seconds` (mandatory): The time distance between each sample at telemetry metric. + 11) `sequence_size` (mandatory): The amount of samples that will be used. + + After getting the properties it creates the specific metric asked and registers it to the internal registry. According to the metric type value: + - Counter = 1 + Counter expects: + - metric_name (mandatory) -> string. If there is a suffix of _total on the metric name, it will be removed. + When exposing the time series for counter, a _total suffix will be added. This is for compatibility between + OpenMetrics and the Prometheus text format, as OpenMetrics requires the _total suffix. + - metric_info (optional) -> string | None. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - telemetry_metric (mandatory) -> string. + - model_route (mandatory) -> string. + - model_name (mandatory) -> string. + - model_type (mandatory) -> string. + - step_in_seconds (mandatory) -> int. + - sequence_size (mandatory) -> int. + - Gauge = 2 + Gauge expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - telemetry_metric (mandatory) -> string. + - model_route (mandatory) -> string. + - model_name (mandatory) -> string. + - model_type (mandatory) -> string. + - step_in_seconds (mandatory) -> int. + - sequence_size (mandatory) -> int. + - Info = 3 + Info expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - telemetry_metric (mandatory) -> string. + - model_route (mandatory) -> string. + - model_name (mandatory) -> string. + - model_type (mandatory) -> string. + - step_in_seconds (mandatory) -> int. + - sequence_size (mandatory) -> int. + - Enum = 4 + Enum expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - value (mandatory): the state that will be set. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (mandatory at creation of metric): the states that will be the available choice to set the state + (passed only the first time). + - telemetry_metric (mandatory) -> string. + - model_route (mandatory) -> string. + - model_name (mandatory) -> string. + - model_type (mandatory) -> string. + - step_in_seconds (mandatory) -> int. + - sequence_size (mandatory) -> int. + +5) `stop_model_metrics` This route will receive a json payload to stop the metric creation(s) based on specific telemetry + data. The json passed will contain: + 1) `metric_names` (mandatory): A list of strings with the names of the metrics to be stopped. + +## Usage +To start the metrics_generator either: +- create a docker image of it with the Dockerfile provided and deploy it. +- create a helm release from the helm provided at 'icos-export-custom-metrics-to-prometheus' folder. +- run it locally with + ```bash + uvicorn metrics_generator:app --reload --host 0.0.0.0 --port 8000 + ``` +- the application needs to have two environmental variables defined: + - `PROMETHEUS_BASE_URL`: The url which the create_model_metric route will use to retrieve/query telemetry data. + - `INTELLIGENCE_API_BASE_URL`: The url which the create_model_metric route will use to infer a model. + +After the application is up, visiting `\docs` will show the swagger of the app. + +## Contributing +Contributions to 'ICOS Metrics Export to Prometheus' are welcome. If you have suggestions for improvements or bug +fixes, please open an issue or submit a pull request. + +# Legal +The ICOS Metrics Export to Prometheus is released under the Apache 2.0 license. +Copyright © 2022-2024 National and Kapodistrian University of Athens. All rights reserved. + +🇪🇺 This work has received funding from the European Union's HORIZON research and innovation programme under grant agreement No. 101070177. diff --git a/__pycache__/metric_helpers.cpython-311.pyc b/__pycache__/metric_helpers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ffed6d554fc8e3fdd76f79c9b4a7444b168f3636 GIT binary patch literal 2411 zcmZ`)O>7fK6rT0|_-7qE4vB#v$iIq9ODPFNs!0_Mo6Ji9YnpTY9eMt)&Aqq2i`cOh6 zNYlRK`?~uYQL1)U(~_ZrcE67-5F>n_7=2WHBEmOecuk9OSp;NJD)-0+xU3&!u|u*W zhJS@526ph6gy6>J@}@~scAhQMjf`cp5-dvBbHyyYgP$fXdLwU8GaCJq09;~#Fs$Epjp_?a!Wp~f94z6UbmDrWL)YFa4)XRQXg=efBI47f?yxP3sr zNva2CZ6!jw^XdugdA`5zo{;lxw1XrpC48=A0${H6EiG9PsTdsNs-7)6uC*e@A}ENU zgIvgEEDnqPFsu)Nl*wy#S`^-VoTjpU&oKNeP*bY zg0bJ)7Ve#brH6zPO)!Y&GXzaEe41p4nruj#V#oll;)ZVUd|53O=niV`Ag()@5<3{Q zrG(!N=scTr-3{q_Hg6P73Vc}C?-Y|J<{8ZVbNM3R%v^aPS=^wmw1}wT39JtZTFE_U z2p}gXfU(Abl=q)TBwAi*g!{_Z8)~>b&q;Z{JnyNaW|yaMcX`|U3tk;h*9aAZfw&R)a+1djd2F`$8~e8tknIX|YQ~MvV;7R5&ZlHM~l4NUuvvL9Gp< zML-McLr2!(z#6}2C*TrrV;(MxHq9>6J4K3_Wdlh31}^a{=*r(E%_8+OO|k5xO|3*f zI|-j+h&>xYf>Cp$x}MDC@~{P}gavqsP+IsQEP$J^7Q1rV%qQV922>P&D8m4|B3xkh z9_~7VqzBGPTtk9LTE774zzO#>H0Vg!YtmpNc7Pw7Du1zktTOT7q$5p1uSru4wSRl4 zB34$b6IJW!+T$Ob@$>bQ(@t!hC|ok=knB?=@_Cbz*6=lJpjfIme^XRSXDX9aa*Nf?wgTGzk?5hd>>Lp>naI`X06{{;xryt*X zF;ttn=!{;hpO}3)Svzs9NkIH+=1=jDPdM94;*x-6`ViS3+{M*59Z^IPrkdn1sRVa% Z^-V`KAP7TEa+p*yySV!19U`6){{b?`Ckp@o literal 0 HcmV?d00001 diff --git a/__pycache__/metric_types_functions.cpython-311.pyc b/__pycache__/metric_types_functions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2012fc3716f760c5f3a7c9d60565615f1b95cfe4 GIT binary patch literal 7320 zcmd^ETWl298J^kMyT|s}V1n&n24B=}Z7;S{2mx}T#tXYGO6o#o7| z!O2pgDi1C~4UHVMA6u6KPQ zScz1r8J|6K=FIv3|J>&Lzw`Ugjg3(TF5h3jLiait<{#vtay5E|XMblI<_04(GOIFa zHo>NS316B^aA|+S&ysqd8b}8d!E`7QN{18SbR-d>^_&__HzXQp*{|~H#zbSfDbbW} zPBcSZKn|jYk3$S2hmatL;TNP&5yT=N65y|eJfl{Ur+n>|elaU0If`0SQMut$e`3pb zq>n?bNT{_2`gzd5^*hq}Cz)B-nY5PvBG$B6{YG$b!M#Rek=-(y>Czx2vKgnFi z{7(3I)=;#Jq&oh?O44uwCo}Nq_}^1>16~J?O1Wv|ghsVo#z5Hdk7rVv;~&f9(lOSF zjB2Wil7@z3oYRn=o5E-sI0PxQaRlT%t~qwB?HxMz%W55bMSli1>K!@&SX`@~%7nn|_dipL<;n`j~w1T=iKF zDNLUG6Z;#sSfi)Xnql((JArzu@HO}*nAKM6YzA*azi(IjemR&AZccli!#&VX=G*r( z&uq?eEG*5!B1A>Fyse~*)0&ve4v11l7BPbDAzqw88PS+QVis#jr0b;Gkg%c0qf~M{ zMc!g0D!M3%dM=exE{R%7oH8^+Qbo-bcDIp8w$Xs7fL29S#j^;MrL|ck$Hmjc47!xn zbl85>o}r|Xs3WW(T})|MOuD;$fLfI*`aOdceUdB_Z6C})Zy>pH>8xZZXBAa3=D?-tCZ(8a_e!^p_t z%ziPGQ&qR>mJbW?>ERa0j@@E$E8K7S9X|z{Mp;Ydh$>h&z`ZE#C3&Me;R>1o(gh`I z{1%6|6IBFRaVOEk#DbW(N!&!TFv)n55hD%<9gT%>GpVR{fxg09$a~kC3y5q|2Pemn zF9fcyCr9^Bo`fx_Pac*APb-t-GLm4^oCCDgC&x#RPXMeMNL7_-lu05$);WYdDAlLP z=ElaPx1Ppa7U@%|Tqa3|sK>K&PRN@x+yf>_r*#JK?+WwpZe~mO!ufljn5`35>xA7p zapm17p>1ZU>uFoh!d|m&#A+L{+eYT2Pa4|FOdujY7G5fRbhp7gk$8x*rf|U$F4)3_ z`9mds>-?c>Z<~AXSxHY_H4o04Lc$UfwvZqy8b2Gm zI#y_#AG7$KHow!Xq;inqTgwa=iI@29`QtX${f%OwSGGh1|<0|Nrh~J=gU=buJlL9*Sg? zuIOS?%DC=@0-Vm`H&6enXf^mev|0o}0CH=%U9`gEB2dB^2I9X0jm11pD%dNjzxyMrKm zt&>u1vhv=w4%G$<3sgH&X#vETQ0<^WkECwx8mb)v3L83CCF3skgK4ip-3Vo)j-MW~IRJ5&n3Bz21HgV_#o@AAHWuEP zo<``35_;+(vZ9ksN(m}t6p}H)VVWI-f}VoSu>l~rb=*jNp7`zL?Kw8=hvozGiWS~&hPOYLcXmA%wifyqKUzvWJexL!j3s1jA+sTZ_bl^0 z3qS{ZEIw}YakG-1x>_C_Fon~WaM~75y9j>d>XGZY`6Cv;!{&FGmE-}rFEUWt(f!B7 z7m43b{(%gpae!vmBPC(mjp*mm!fQp@68deS-=yhDLkl6R0~^8r@G?KV2nauH@ds`G zpjk;2{>5cpES@xZ(c=4UzTd1Q3ja(lD8H=p1sotWShx*8_acD_e9)-D6&Ka>exnBCAOHfP zk=xbq$YmGTtIu+{8bYZZRmBL5=T5NRE`(4x2%)UDTW>YADsV%PeN*}YscS4$y91s} z=2jzh6{o-Tr~s!o)yr5!`U@-w<-{adSHbCvGfHxXKn=uCfEhx;-9VyF;}Ikv0PwfAhn?st z5z=^CNfRBr#&~Z`%BuW+hRi-?hx|B2H@8 zDknveKK7Cmgg_2_pqPL<_LGdj3MRPVv_Qxd!aXFQIF(ct_;@ivDK{WRoj+M*r(;dC zcqPb%8%TpDFJ?4@)GP#_++DE}-x?roAgTrbC!nW98b6}PF_#CB1*dzB_L&ldufqq^ znw(S78~Am2OT2{M0p}A~F_tYcar2wf6K0?JO{v5Tnf0l}95pvi1WBbtiw1P_2WItj1*nvWCvEAzUQJI1A-7WOC^osejllS-2a;c@g z&|A=pqYFJ>jxPqR-aY2dJ$A=lyXDm~2O_1n4&R$t>M>88yx(t*owVONMXRZ7O?HCq pWp`IiL7A9>Th$blpKFS|KQwkP`M35mN6Hgm2kunKdKuk2{{_on8N2`h literal 0 HcmV?d00001 diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..3e7fc43 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Set default port if not specified +PORT=${PORT:-8000} + +# Start Gunicorn +exec gunicorn "src.metrics_generator:app" -b "0.0.0.0:${PORT}" -k uvicorn.workers.UvicornWorker diff --git a/icos-export-custom-metrics-to-prometheus/.helmignore b/icos-export-custom-metrics-to-prometheus/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/icos-export-custom-metrics-to-prometheus/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/icos-export-custom-metrics-to-prometheus/Chart.yaml b/icos-export-custom-metrics-to-prometheus/Chart.yaml new file mode 100644 index 0000000..8c04ddd --- /dev/null +++ b/icos-export-custom-metrics-to-prometheus/Chart.yaml @@ -0,0 +1,42 @@ +# ICOS Export Metrics +# Copyright © 2022-2024 National and Kapodistrian University of Athens +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This work has received funding from the European Union's HORIZON research +# and innovation programme under grant agreement No. 101070177. + +apiVersion: v2 +name: icos-export-custom-metrics-to-prometheus +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 2.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/icos-export-custom-metrics-to-prometheus/templates/_helpers.tpl b/icos-export-custom-metrics-to-prometheus/templates/_helpers.tpl new file mode 100644 index 0000000..58a6fdc --- /dev/null +++ b/icos-export-custom-metrics-to-prometheus/templates/_helpers.tpl @@ -0,0 +1,82 @@ +{{- /* + * ICOS Metrics Export to Prometheus + * Copyright © 2022-2024 National and Kapodistrian University of Athens + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This work has received funding from the European Union's HORIZON research + * and innovation programme under grant agreement No. 101070177. + */ -}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "ICOS-metrics-export.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ICOS-metrics-export.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ICOS-metrics-export.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ICOS-metrics-export.labels" -}} +helm.sh/chart: {{ include "ICOS-metrics-export.chart" . }} +{{ include "ICOS-metrics-export.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ICOS-metrics-export.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ICOS-metrics-export.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ICOS-metrics-export.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ICOS-metrics-export.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/icos-export-custom-metrics-to-prometheus/templates/deployment.yaml b/icos-export-custom-metrics-to-prometheus/templates/deployment.yaml new file mode 100644 index 0000000..214dd4a --- /dev/null +++ b/icos-export-custom-metrics-to-prometheus/templates/deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ .Values.name }} + template: + metadata: + labels: + app: {{ .Values.name }} + # enable/disable the collection of logs from this pod + # Default: false + logs.icos.eu/scrape: true + spec: +# nodeSelector: +# tier: "controller" + containers: + - name: {{ .Values.name }} + image: "{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + ports: + - name: http + containerPort: {{ .Values.port }} + protocol: TCP + env: + - name: PORT + value: "{{ .Values.port }}" diff --git a/icos-export-custom-metrics-to-prometheus/templates/service.yaml b/icos-export-custom-metrics-to-prometheus/templates/service.yaml new file mode 100644 index 0000000..a04ac1f --- /dev/null +++ b/icos-export-custom-metrics-to-prometheus/templates/service.yaml @@ -0,0 +1,55 @@ +{{- /* + * ICOS Metrics Export to Prometheus + * Copyright © 2022-2024 National and Kapodistrian University of Athens + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This work has received funding from the European Union's HORIZON research + * and innovation programme under grant agreement No. 101070177. + */ -}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.name }} + annotations: + # enable/disable the scraping of metrics from this service/pod + # Default: false + telemetry.icos.eu/scrape: "true" + # the interval of time at which the endponint will be scraped + # Default: 60s + telemetry.icos.eu/interval: "60s" + # the timeout for the scraping call + # Default: 2s + telemetry.icos.eu/timeout: "10s" + # the protocol to use (e.g, http, https) + # Default: http + telemetry.icos.eu/scheme: http + # the path at which scrape + # Default: / + telemetry.icos.eu/path: /metrics + # the port at which scrape + # Default: 80 + telemetry.icos.eu/port: "{{ .Values.port }}" + +spec: + type: {{ .Values.service.type }} + ports: + {{- range .Values.service.ports }} + - port: {{ .port }} + targetPort: {{ .targetPort }} + nodePort: {{ .nodePort }} + protocol: TCP + {{- end }} + selector: + app: {{ .Values.name }} diff --git a/icos-export-custom-metrics-to-prometheus/templates/tests/test-connection.yaml b/icos-export-custom-metrics-to-prometheus/templates/tests/test-connection.yaml new file mode 100644 index 0000000..bf052f8 --- /dev/null +++ b/icos-export-custom-metrics-to-prometheus/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "ICOS-metrics-export.fullname" . }}-test-connection" + labels: + {{- include "ICOS-metrics-export.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "ICOS-metrics-export.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/icos-export-custom-metrics-to-prometheus/values.yaml b/icos-export-custom-metrics-to-prometheus/values.yaml new file mode 100644 index 0000000..dd4f046 --- /dev/null +++ b/icos-export-custom-metrics-to-prometheus/values.yaml @@ -0,0 +1,131 @@ +# ICOS Metrics Export to Prometheus +# Copyright © 2022-2024 National and Kapodistrian University of Athens +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This work has received funding from the European Union's HORIZON research +# and innovation programme under grant agreement No. 101070177. + +# Default values for ICOS-metrics-export. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 +name: icos-export-custom-metrics-to-prometheus +port: 9600 + +image: + name: menelaoszetas/icos_export_custom_metrics_to_prometheus + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +service: + type: NodePort + ports: + - port: 9600 + targetPort: 9600 + nodePort: 30600 + +#imagePullSecrets: [] +#nameOverride: "" +#fullnameOverride: "" + +#serviceAccount: + # Specifies whether a service account should be created +# create: true + # Automatically mount a ServiceAccount's API credentials? +# automount: true + # Annotations to add to the service account +# annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template +# name: "" + +podAnnotations: {} +podLabels: { + app: icos-export-custom-metrics-to-prometheus +} + +#podSecurityContext: {} + # fsGroup: 2000 + +#securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +#ingress: +# enabled: false +# className: "" +# annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" +# hosts: +# - host: chart-example.local +# paths: +# - path: / +# pathType: ImplementationSpecific +# tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +#resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +#livenessProbe: +# httpGet: +# path: / +# port: http +#readinessProbe: +# httpGet: +# path: / +# port: http + +#autoscaling: +# enabled: false +# minReplicas: 1 +# maxReplicas: 100 +# targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +#volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +#volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +#nodeSelector: {} + +#tolerations: [] + +#affinity: {} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..977c2d2234a878d7de9b63a1b35b243db7cad926 GIT binary patch literal 746 zcmZ9K?@oh25XAShiH}lKplSW$vouI)Ayz>C81>=R-|PZ5=5idl-I<-;xu5Uc8dhtg zHH(tHBSMIK$zMMl?@oJ|tO3}hzocEFYgngJ}E!Y;$+<6w&Qn&YvGm(L6IempP zb(OGmMrRkB(4rb*YG(balCC+GV;26zI}3EpQ72#Gbxii|bNF*ECZ=~1bN&UnMQyPF literal 0 HcmV?d00001 diff --git a/src/__pycache__/environment_variables.cpython-311.pyc b/src/__pycache__/environment_variables.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97f58250854d70463f6eb2fea6813406ac8705dc GIT binary patch literal 476 zcmZ3^%ge<81Sk8ZrCkHkk3k$5V1Y6|2LTz=8B!Qh7;_k+7?>DR8L}8*ic*+Tn3pjz zFsue*2#8`rQ^AtLil%}&g)x{xlkFu)gWoOTfFOTg*ANfa(BOC{$6(j^&>){Gy^NBQ z0xNxeOG77BXcWb0|NsK{ltPy{W3%S!qU{D%J`zhy!6ytlAeAcu0B4V z?yi2$uJMimo(Q8Pp+*@R=oy+B=m8BfH3S=^ugP+YDZls@TY73qYF=3pGtfCjAYZR! z_zZH|uXtyxn9$_xji((m9#4;{02;Y!V eo#8ac^Mab=1?RYnGVxbr;x8~rLQoM0&=3Gy&46nF literal 0 HcmV?d00001 diff --git a/src/__pycache__/metric_helpers.cpython-311.pyc b/src/__pycache__/metric_helpers.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c1f9ab7766dafe8c96a4856a819ec6d72e9fe1c9 GIT binary patch literal 3122 zcmbsrOKclObk@7;-};lUn-6XNqNdQ6CT-O3fuw{8H9<`y=ql36c*m(>ZKty~!XcH6 zRG}OzRH-RK!aY!m%AwqPEQkXK4r?ig)t)LL4!Jobr=ECkHnC&76+(<>-oBYPZ|2SW z`iF2hNTB`l_tz;)Cge|?>^JWovePFJa*yal7YvdSR6#)8V|X&6Dso&jq>QY}9G47l z#;5u+e$}4|r~!d^2o)Z}QIGe}C2CNYsiJ#f_*Q*{=zglCyn5iaq=xh$g&F+9NTCsg zDNk>Wx5g2rN_{mY=?ZYSdB_aWL$`?@rs6{p?u6kswVl%pp>=Xv z2hciOXkEH*fyBGE@QS$Lv`^&>gC_Gic9kxt%{(i@qI5Z1$iO%7PQs*Daym8Qq9ZS6 z)48l8O{bH2$3MT4hgrgKlnl+YbW+PEGPE{HXH&U3$IEEGz_Llt-h1$aI6iQv3&1_H zCY0ngFw!c_ZV77wh~}AFl?U;F9|30pATYGIiJXIjFFu+&x3p+dW-d)9-uf`TG^^9Z z2Q2pqyunx1o+Md=}1PKv*ASZGfi@{E6|7TR_Tb+v^`n~VD#zs5aE)108DWpaAKpn!)o?Peiips&HqHjl85)p-en8A&0HxZ`jzmi{b|_rBY%8JCGzX>W(zMHtoDDC9*ye5L z5BStyx=5%+krEp?WaLN_Re`fY zoWspDEqo1JUgTOPY6Q3-f7EGp45ab7a)Mp_PP=QD**wi$r8f%{Ju44J;!!xq!_bjG zOBe;}dYaeFC-T&cx3R-;6+_#z0R*TuC#q?QY&Hj5AXAuNFA>rUKZN;l6Xt41P8qob zT%$u);fFF*up>ecv)6G~CxRw(j^Y{uw50hxfVw#$PJ>;RguNnl+3g4Lp|R5YcX~EP z?jE+JG3XU(%vRd&bZ>|o*UBSh^U>miFRbBHn}^4(_VG<+!jiC8qzU`J5ygLM3;a7S$j{$x^ z_j8W$B*oP8*aRnt{w#+^Zr^5E2(P z+|n2|^K=F3ie}PeE~}eP*u)US8fB(eX*|+wIG(e(;kXI05V>)KNuEau!@SMjK=8^? zIf+yRCt#RA0cf!De+iZU8w7T&*OIVTq+YwNvov3t|8m~$?y)576{*MW#DWUF!XY58 z&!PD7Zl`^PQzO`dbGI=Y1eBbG-BS7cKhz3^Xf%hM%x4)$R=Grs~@6$L@C z$qB2m+oaoS>^3=KHFldEwO-w;UJ@O&A_JAk&}L+)q*T44aH!l_CGdGN{}gFI5SUpH2KO{&tq5%^)TT_#dGK z-09lJ)!jNN;uXfKq!nxgwsCd0PU;YZ?kZ^o<@h$P?$$~EfN-cvT0!~jHm>fzgv1@_ EA7C29F8}}l literal 0 HcmV?d00001 diff --git a/src/__pycache__/metric_types_functions.cpython-311.pyc b/src/__pycache__/metric_types_functions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..31789cfb46f36461d1dff6173a04900afb81da9f GIT binary patch literal 7332 zcmd^EUu+b|8Q=BZf3NMe!35jEUi_!+)%IaKg%BWrkTJv!1saDxxwvwc+c|q-@AjD8 z11>o#RQ16{sG+D5*Qio2Qbmp6!H+!h&<7(0Y1dkm(@IE46%T#8idL%9mwq#Qd%p7@ zf|W>>nz`B8+1dHNZ|0lX-}lXTe`#upFp%#2=VtQDPKNm#dGJzGrSkksmSJu&G9$Ap zlVam+sv+Kx;^JJ&7x%HGy+QS-0`Wj97!RgG@lYxp57Tx|jiegmjkNAl`BYQ9Db*Zr zPPN2apv^A_P~#^-hLM9vkV9|<=u;T6r~wJ^YbDR9jg-lT4sTq{N==TSwq!(Z{IxH> z*+WL2* z%c#!@9nTnwmX=h^GMNV*B%cc#49p7X+sX4xhbT$=b zo$$D(swiP-ILbMVsoV@kvp|P)PK%C=nG`axl9*8lNi%6srL#z#IgfI>(}FHTPbEED z>2(a~-R@~Ag{lvfHbCig$A3{$vnb)OZUcr1K%Il@`R88v7IOY8YkyT0ee7Ai-1DBhw1R9up;9T`t z4k}EZ`!oAnwphc*>&-BE-#vf5RQMVirkK@I>tqIQ!?^EO$9^S{4{VNmp2K}GPUhQ> zGtX>Jax^4ef<=gkkSbeA7iTpwn;86Er z%Sc#C!HnuYsC_UpBuXkqlANo)6f4?Mk{~jhmSc2^6%OvZ8KOCty3(06tk;ozoOK;Z zLtRJr!*%<(b(n>;vL+cx*O5$hN3`O~tK&dF`N~{UWLZQ*6P0uVRuhp2psK8vrwxpZ zEKcte(^*w@S>0*C0(^V86^dhbSlkBpJ3hxpk)}!35?R6v77lZuRNO@#9l~s|He4Wfr9q2%t5Ga0N^C z8M3>vF|FgobY)SETn6bg$!t17rm4p=IVV_A9PR_LWZ*gjAb6GeS3k3*cj5elPtCR| zt8L0|o4Wem)8IBU*!`@%Z()zwK4!I#+3jQVk*AIAWyT*Cp9rrMKDys%o`^q28B@4m z2^VbP!u-J!zjglL^>iyyK15!1`hTsDu+n?l?Y;xHeH}pP#V!9-Hqmy<843d|R2}!m$$HJAd5fdrf`_a`BXV|B4@3=11=PfxX4Q zW%F;DUVh4VeZ}`I^F75ZaI^Sfn;$m4{2Ua)tZ!oqpq^XIOWBxD|Nm{wdiLvoYGE{3 z9tvmFXVJxkly+?lMO|6Dl=dLvdjgmgI|*JA<4TZnmN;`UancY|S=}H;Q4*7?27nAk zPZB{bx^z}Wgv4zIUk+@fr?$1oire91Ifl1EiC=*t%2HgzJ4iz}X{ZH90&lh82*Y{- zIARgHJM3&ND&k(!=BM+(KZ1G;F8$w70BpUwN!aRmBD5FwFP>jI_c)O@1(c)636XE|1E!I2G~tGeVh4tI^L z8jc$|+^eM4$z1Jl2lK(=rnmzf4(DH1yW8R5!4-)eUX${ExuO0b!!p-9s$l!AwE*t? zrsf6ie9i;gN$~kZ151hXx(Fw`VHb3WCrCV6@HK|Ste%xrH781EwX6X+b7@W^AS2;f zWO&%}J%QwPH6+S#@&-*wI7f~%1gTw5+I7xf26i0|^j-tIVd|4QK6=*X0LESSl4 zzn?BMtbddi^Wi65-GyxNL%U~})iq*wjm-P!6)Uve3~hg*=h^*4*jgA|{Aelu_-x7) z(w306h4h9H-nY#6Er3JVXYnzckC|S6=5l$o-xN+;!f9JL?LzqBYlm-S=MP)_4x8U$ zdbxt+4dJ2Ej^01TKac-m`j2EXO+&P}5iSYaZbfcJ3U3r;OBl3;L6hdEjjhC89oh*0 zN0<50MS%EGi$7rV2TU(h{1=ybv3SztMT;M_`9agm6#tz7oRbYWvH0mdQ3V883-V!e za6K=>n%rXY{=DxZi(6{FSedN{?_aTX-Ybcttb!jJNBL!)G2j5ALBd_Q+(hCN_<&IZ zEH13)eMSw)K^z1kBzLP(k}EE-SD)ojHIh<0D^Cc_=U$**E<{o|h@`BRTQ4=ZDt<$t zeN+4as%s=vhXbZd=2k;>6{x@UtN^GtHOgpMT7(6mI6`7}9#CJJQxbCoYaomQz9CfH z4J7I`EJ5NT5Yk9wF@}RNAh+_paGhNpxWR^sKI)jD^*Q4LSxT!Z5cYWGavT73UxE2R?C8Zb^*)+SctY)AbyVF-xYGkBB~Bp?AU z_!znQYI;;(x;}ZxiJT&l)Cr6=Tob$?{?>|hplmR4us1Lclq-={F$-v|S#W#^b{*aW zopNyLZ@^}}%9OjAP}7aWX0Q|TljiMa$1baR#BLrjLnF`HcD{V*ArRhKH2~(v$tT^> zGRL+K{)2(ypQM;S_;mdcME^41zp&fn`z?Od=0^c^-CVH*OWPjDktrlCA!!Rq*BBhR zcBC*cf5hTNn->9y-3-Pc-11rETI9z5qR$Eq*r5S4G*EJ(e%D4&Keo(|-G{Tpn8m+s z^KYA8rcl3gncqpQ)Xr5_ib8$#6(<0(9QZ;pL3QjS1;G_efWc{n&?!WGNMLa$p(^n8 zVggfcT#8zMvdB*7nr<;K)`c5Mhc*Zu#VVnrIizMF4(0BQl~C6Z=?4KW_|JeI7HJrX z9?D#2fGj%QZ*<($OZYl;Af?G!6}^Sugtx?0=$&v%fmLJK5)(7ODL-ZQn%|U5%!pZ^ zOU!%b#va%VTTJpDMqcX!7>B&`+fAbG;Y3s>fujmz0}qTJ-Fi3 z!fRiAxY%wD>@meXcGv56+ukw1!;nqU<#6)#aDb(s>Sl5E+gElRd*$BtvWBTI7oh~@ao!8qa2tddNbhf+n^ zmO=$;Z)RHwyxrd34B`ROSOL46aU)qEJ;-AHV}3Y2>DlbepQ2EK2?81zSQwc6VtqHVHN}=qKvct5@%S@4LUMzw-M%41{z4`&X0iKF=`!3vcwmX^F1} z?F{oCBQOG+VdC`4##wT2i`&T69=F5QmT_bqaR-ZK>=|d)6?bJD;tk~9k>RrLxSOP% z8Bf+5_mZ?LbTXM^z|$>$(H6c3TKJJXVFjkjjo;%(XX zczd=Z-a+y`nGM-+Je=)}canQ=rYjqXN3z}VZn*co#l(ATjOY^lA2oi!GR%kYPeYuC zd}g{=zlU_6p8hCcEz2^b>_+{$xXJoF-Y*8?o1u(b++6o$i^wl;)$5tBLX)`V0~?G8 z{%L^mHj7aqC^iWpfffDp4Gc5s5?bDJ#J35pqF)G!+r_9DowRMxtH9onQVyXFN)14L zztH}uQje+c5;~xM48}TWG;K-<8?3Vu!f!Dj!1E7{GVvj(yTiCOC4|npx?N)Pq)mvt z<)~|6=ObI_UeiJlTIgBR!Z7RrJH1QqT;PRX>k7wr3teIe>h{5G_h{*jkbdfseb?=@ zR;PbW-+Kl2EEC=QC3Zd$Wh3j}K z`LoH(Vj?L|r4z}S8OUfoIh!e@pO^9}QI<#ZnT(h!>?Q=(^A(fxa z6-24xoJ!74i4{jWH<_MV@f zG|x0S5SyK0)ux7;>Fo1Ok~6>VrHHj(i(*m zM#hrJC9|S&Q@|tUlElKSluH?C3)*tRKd$0e7hv|V-jLM8w6XTf<{J9rIy5rIyn187 zR(wPYMHAw?!bn#NxDi??+N}FF?a-er%!&N4Uto#248=uEx{3+mw^ zoWUJ}ZIgKbJf7|sB$EaHYEtHPm=^^e!HkrdN?#FkJZ|ZrhcvMtAS#*F>XNl=IAt%`lO5r$S^L_D;eO$*#a-5CnrTo%oTXjp?D2&qO8x3o>V%X`t>1G z;KhtA@)P+SZFPS9BsukT{@G*!hAZ>Q9G}bpniJ;Ci>kFphGrea;FmXHi+*FPc;!N} z02na^2(01~#6mKinTUF%cG&Yu2(mToo+#wW+RGKkWpNH-0%u%;WF`^-ocXJOfE@%D zU77sU6ry8MlJb&-XFKYYy5S=AV7qRk*-E34OwblD6$s3&7BJVLyLoJ6H_N-Su| zCvkC(cALxzq?WW3N^F9E`L_@)GGDhaZQW|?&U>v-t+YN>YTc){?yE9xXM{vc?)%=h zUvE{zPklV6hDQ`H{#GNs%hA6~yq{1bd+tS^S&2MTiX2oU2bU(w;eIu|?Ou4tN_a;p zJgkO?mrj%ek>$>nz8W~FKKYTL$-?RSnUZR4f3 zakXt+X&Ntwx)h)Gs|Fa~VYb@D_?nh>OPx4D7u_ujjX?ZBaMm`0vx>|#WYDm{T4{n6 zYNri6wrrvtdR?XtqX9Ov3G8K3M{*Wf!8Q*?7XYpWyD8UUKNw5;& z&lun`dkyOrZCZOgKYpzsB~t}{5||1;4Ui2Oi9k1; zNgf`|CNmZ!W(IR@ASUcbS{KqOm|mw3!}9jb{M# zB~)Bm%{s^oBV_99se>qORyqic9)N%Oe?YXzRD(=t!@C2ECsu=PH?RErfp-Vg;Ff#A z*h(-~3J$5kp~VyT8-h2}D-AtLLr=M_cll_kZD7fZnUgCGT}ne2X8yF)w!JQsU-GU- z`tLH48wY6 zdHYHzri5bU&ffPADlJ>hzp9;S8GLx(8_>bEQKGzSg3i>!Y8YVqk1*?>J0H*7z2kh_ zDaDKIg6(Z59CefaqIX(Kc2G$W|mE#{@f&d>oAy z)<*{GkbJjZgJA>!!H=A4N&Z?_B$1L3XnQf8$>`LR(q-+W&m?6TXgpAT1QTuH zcwm%)Xr-reX7aDqFmVgOXd**ODfux3-Z`F>;?pvpO#?m7O$}hfIUb+@VUf@~F*^e^ z8iWr7@-178>RJavp3mhAJfWYoa^aGczseJewH`Jgko73e!@_1*4+gDbObgcFQu2xj z3<`|5wk}#7y<;6)Oo&&3doa)fnV8lT;u)B9TH>!HGay9iID-zQ!dU1C(f|}Wzv3bK z3-$!V0tZ}VUgZGCvv1q3FfTCgu`F}LEdUqy8+PJ1Ow?Yn4ekK4j!3i8q9Fs2Z(xnJ zBQImCJikcj^Ed7T$o^lOkq*soMuadq3lbF_YXa&F9?0ZDbdnDZ8s#!jT}CYR@Sm7} z0_30skHPQb^EdY>T|1P}4tj5n389=HKSLppPvvvKuH@hpA3g{K&Fgb#8tzKLV=BafV`|`7m2tYe z%k8f%9bNSWfBD=y&)w=M`TA5}pW^FV4ffm~y~8QNky3C(4UQm%xqeXhO>83(`r*yaN9+Qu0OSzQeP4 zgzjbIkS3hd16FC!0Ksy4k<#N?`|20aFP>X77X)`WEE1#*=DR16<#H@YMIg&fWBVik z`ekZz8So#>G#&){%n5fmcY5b4+5pAc78;&Fs%o&J<~pg$UTpZtQIBE^t$I%=l@;1* zch-{9;iBV7<_cqV{!nAh4HX@WG2jM&ptW!_g_=H*0kX(i*Ivhc6qI=qqm92Ycy{aHP{blAV>t>4F&zcG-?P#IM z73+xTg7Zf=X12iV<7Jp>|m{<_(8==F?sK`N~U zV_jUZ#-0xV?|f+B9$4cPocG;&SrmVz7ZyC<;kECCJ4Uu`zq|M6`k(V(;q;hrt)8o==tWW;J40BV1>p%H|g!*WIH|pitwR9 z6|CjK2oeyQ(x^;tPW}b(%7p#3P%>TueX$6vD6qtmm=Zy&#V7e`Igec5Tqd6c?hRB; zL^F-7%j^v5;-Ls7ggPTg}}78Wy!CJLnVDxWW1nnK)}0SB+&v34%6?0tE=#>oiHF?lqIl0GEjx z2JLa0>Kecfp)EGbEEh!-Nz6@8rX{F`Mh0!)M^Y(JK4LM;eCXDa$s4;pN=+a9L4GGW zg~){Jd79|7-*{l`5ZPVRp7XIonyfU)gNy}B2plzxHOb4flauLdI47!s;Nb+9*d7hdPb0>+K6jh^CbqkgafGsrt@CET2*;kPMpi`MQ z7e`o2gP21`Y8ehaO*kcE8S4&uZ58<n@ z%Yy2-CL@CcNz1^N;i480wZI9xC`#t;)|`19+$jwtUl{|T@)Zz{2XNP6uwdwccoOhZ z;ni2o&FJSG#B>~9ls=2`KSe_u9J*m+&1ug6(~&dU_&ygFgf(rUEO-Lp4p!oP4peQg zkO`ADcomS0W;jR!E|KKq#SCcl@^np%0aAU!ptcP3Ed0PLeGHN5L4KDp29ohFWu5B} z8a-m>6ZQiw-vC*!7(b54_aML9SU!^R<5@leV1R4@taYGAb%`;jc7@Od&{^mKY0ZVLYnW!!(IPpJL7yf7ol2oQ z0;-+n=;}?(chEwV!-0|-ZWjh?%AfgXDKZ5rfSAx!h!BocGKXqH;4i2?5h%|zqKT`Q z(x7yoNv6cX`9Y%va=0JU1{(1OziGBOSnTKDScp0+fjR{`Q9Bt%ZKN~OeqdoMPSV-@ zHmy(4cOrVw7Nds2YOKv*)FqvUnpBT<22-RV7v15ClU%>Cy|FOge+~#Uj%Pq?P$uM^ z=8z6CIp5O%a;!fa>lgU`qx<_$?(aW4*gyH@|M}w||5$0r!WJZ_#QCn1)JRF$+Pd9| zc2f%l8XK^GDLo^Rft}SyC!NETRTWq9q=AEg(SpxK#SXQI)vw}$l>s^~p=e4pY?23? zZP0iiYCUlh;v@y|i39Ox#epkCttk$~2-KM3c>#5)x<` zij+g6PdJJy=W@2MhWC7Y`R+5N@F_KXYALW3cnB^dXx@Q2>Cpo^C#7}kz1E?X)}fEN zPXeXZXVuncOWcUcjVRp6{l>PX#)lAKEY5bkHa0lh%igf!-Mrem^>)vl&85~6wRL3a z$iwfe2Va|Wj76LCHZ?r3;vG=D1FOE!@~OL>N?cTOg>ty3YG?hue_$Z`XN;;wbl>0c z>*H$Iz>0rB@k6!d<)J%oC@-ayj8yK3V6`5s)`Qi0uo{g>wf9f0__r(m?W>{4E%Cpc zD}}bHp>1#i{72Yw=TD!|YQ5@fTJE`XS$XlIa{2Xgdlz=uja_zQm)+PUjjUZ-YISrk zi&!s)^Ch_IbR$sWHmKZ&dtBcN z*QacJy2R~Qx&3!`-8p;*dicrR&dvux> z?*>QPjysvZcY03r+y6c=(tLuq{{!!YoB80-tlcA`LqkJ$bEfD>y&7$2g zEcSXCyb8``P@t?0McXtYauTdPTkk;sY9snr+nctURw?RA_P|gHGqPqepG-GXs|Io@ zC9vN`D`RFwOoI2LEQ%<40M}6)1d7yTI{P2RG;IaWD2g1D#ti^3W3Ij)ha{mnYQFcp zaom8B(rR1ysrl_EHK!wBhNxo||8jno2lhzUVI_GKxRAk-o`*((PVR&YC6Tz1(rXwI zR|HROSj5O?BJ>xq5R&q!ozU4zyTKP}S^}L;icmq@gr{^$d|1_}su-*PUyA#rt z(DP?bog6=Rbo~6;#Nm;%>Q$gti;{b;-P?sID$eJ!J($kq-Hat5MmA^PB&14S#SWd!Ma*7qYYvXaD3nZ zBmw_4EI13u^JoE#1$OWShtu$ptquvfimspiO~GV{DY~p+E;ufeR;Aw*kc-n>1l4$5 zOAQ6XY(bErfn16W){+-sRtwyMyT}Pn%|$W<1}u-+pdd69Jz(B|k{=pt0Pc=%!BMkD znbXi3vuv8NK2ChT*jcKOn{m<+#>JadP zuV7McgOc2|E_z*90iU%;>m49*J0NiXlgt{>J6~%T&^y22FS5&~)vi4))+yLo=4I$< z!T+Y8^hZ)AWF1JtyA%{zp94AZ>ZLsBUh*^OOpLN=O$e1{b0Eacpfk9DLS>!@J39}0 z?j&`@s_FQK&Rt68^6~(nD9}p+nligG^8g*-GypE!DC7~W3Ys)3#8UL9gA2a!8^WeCz3(68N=7gC%YV?FW=rK`#B}mjH^&UcF z9Tmb*7^LDB5y4Oa~u6a#mk6EmQi1n+*GWMEI!WtG|n zzylkWA_o#VY#MQhA0(av(O{(s{K{!%4OLeHxE|ow2F@a2&=BN`Utg~Tu9t+ST`7Z6 z2%?HTBjzMD21+e>>mln+z2C^({tO?XEF~d0R-7~72UL)dyj0xA%MBG5_@%>YMH?xx zX%zL>9ula?uQoIGlQZz3WzQE+9s7Kbawe&yW=buut1YiDxgT^Kx-*Aa z7gMD(DWyXwbqH#Qu;jn*?NGd%{%h#=tEK2OYV?^q$(86&l;}?$^c=dg@Oie>b3yI7 z01mDX_B^kgen~x@DxXdQ7yVQUIcJDVzG20;X*pBnn5M0_cdY~lmB3)R^AyM->-frc zZn@3A|C8H0)y~+`bN55x_YVB(K$USt8*9jF1KDpWz z{bJiQ%C=GX-4W=u)HSAdjV+yg;A>XU<#2E%ICy)Z6g;Q~50-p~RNo;*{{eH~F~NdJ zyVgMlD1Hlg3_1g~sLZw9<030u)?xEb{?#G z%UCU?;Kz&DMhit~$x8OX0*s^4f?$7=Hj6eS_V_QoUXs;(wHxfX2Op*f- z9v^xIr))q{91D&&;f(`&H4zkN@)}HrU{zhaLcp^g>pY=-R_Fiui%Ilj1d=3!Phxs9 zodQ#mjsy{cNsl#h1q{4lA|>iXXo%1SQiN#h2xdl2tOcEuS8a5%KxvQ_HSOnTaU?(h zQ}_k|p+8hnN}2{{VlO`Jw+qOPD zT^u3GO|k%lN}WqD|67*$)*)7c_XU82RqWa1HPHBFL~%yC2HD_wN3?0^SYGh}j+6@I zU8_nf76}-C(g>UU$ zo-p(YW$y-%>)o5mL4LVe4Q^C?8>=qPJ-E6d^8SXu?0mm->G%U*_pPh98%w^us&B92 z+xs=g^l!cG`fr;)iT-x`C)@9OO2fxXTTgs`y0qz(y6KeS>jlNhjpIuDK*<+VeKExs zt2RTE55EpFKG1ov&gNQF4m2%swLgL}>`+3-7}rPkj~rk=J>WSUwExb<9`@UR*T7=h z?}T{1#c-k{hn#MbMPp#F`v?~AF>YoGy!*}YUc(AzodIHjNgKRvW?_KwZGmM2VqsN{ zV8tjXnJ*infKdn%YsyiqW%9rPiQaB#6##$e0ige-1-sx}cIi3hYd!Y(9^Q9yfq&Q& z^|e+^0DmLrzA5SsXWtGDdNowI{E+fK?7i4Ux8V zQ>8)P^m$O$w%)yD9W583gI9^43HE3-Wa`_}lMGG}dM9Y>i)ve~+VHJBrgK2Jxo>p38eV|*v>|9N8jCyh%T+QUcqULKS^zjhEE#N2LLxUzh?wIh|wvB zsx;6|7Cz5lk^3n(ozPX^=%_@Uyj}nSQG3nLQ*pyfp72&0kk5+i97z)W0C9qnVARu~ zHYbV+qA51nB~j;KQC0+rJ|#gWl%TB=?F3Qh)_=R8>v2Ps{sXlAI%+7sfCOkLj@W$Y zFS4O?sc|*3S&hWL*mFjSgO2uff0&#_lQ-!9TIAGM`a_IQI(CL-XY30n>Qs|r-I`?Q#oyU1$ zG+>jc#6WfA^^um5cIMM|&*7)-pZ1P~4)3=AZnqO|qWdZh_`wL=66>6SAbsvE$aIXG80j(zcRB;S?^b7 zwky_OnR!;R{>sb_#ri8VlghXK%FHv0^;c#_lyCc$nS!$3ugpv<>;0;njm6It=#h_U zXk47SIj*?EC0AH=g%|Djnb2Z`%7kuhd4E7*wv?DHDzgO=Wp{X~U3G_V?JK!AsqRgS zE^t6wGGqU_`?`B++s(@*rcY)16sE7rwXjXM&sG_@+?gkr z&jscDYo&?T$W48;V*}f?JXdAl^6|Ml8}G<>wtn*3U7@t+ILWJz_I0u?%P&_MxZHKz zJ^T5_yDxp#sGL4udiFfYRN}8GiHnLbr6i`*_$B3Xz7)@sC-u<*FvR@4{d)US=x1GT zbyaOF>jjl6>-~eBu{G)$92V-h-l2HHCAL##JFB)48@pHO9H=swFvnQ2%P3Mk?l9_h zm)IVa?Wx+@*mk97f0co2Ey8-d{~l~l#nY(1~JVXRdxXDb<@7p0sWLi4(EuD;Dl81 zUa&0L-M&5{>mgs>m3-;o*O5A-D~BEOB~e38_)GG4zTD+>!L~@s1z2pC3}u#TurZcw z-_YcbKe_^6#gYii|&VwjekH7V1z3R*_b>KPfvsh*kaV%BB~ z3RUv*b;Z)sMVgzDSHLW`WN67|_9X^CxVkWTaPIGm8r}3JGEufoc}=rl>Qw$UD$Dtz zWy_Y9DN>E86%8Gx8k%LHOI#V%tb%Dk7$6jLw$z;j6MqkdpPFv9rED>m!{ct@&BjaBDlgR8nbgOYMy=GGlKezZEx^WJ zrRKVpv8~j??9wtEJ6nV0tg&N}Q#Li4_AZo_);0U9FTD%`}&KJR8!LlT^E#NJzgoGPOzhBmsfb`+62 zhzjP-998j<6e+6%dl87tqV1ugk~fP0sK_q}zq)}0g!F+(TvBwz+7@0Sv@(b_^M#zo z5dA31wus3mEEYdnRpguwB!jG5gxQw-*#tUduyn@bp2TmNIfUF+h|eCuj=4ej%cZ!Y zXPM$Lc14e@wxS;)5Q1HtpvthS$*sW;40(QK<*E!Q1MbWWBMo&%UQC2tiFy+(mk085 zBjDdIT4}^xH@I$@hRXvCyItBwMgznKu608zUcLaSDZre2yyf#(zDh!Hc?74|Ffwr9 zQMAj0!qIswXc!dUY>(xthG(7#hBN>)yM& z8}*@|J?yIwO+TEk51p^}pMN^^%Fe>m6DN1VPh$hy3qL+V-^2CDc#{N#=zjG?>Zu-k zttO4urO~Q1+UV`u4&%?)pW0Vt64O6(zpr}i3_LX{S(lPkDcPRq&HfQeQvgTswcdu* z{P_ZZp__asa#%~O?qYTK*zD{4chfPbzZYY(BmDO#Ih02Ps1w{`BII^M!hwD#8T-;G zSYzbP2%5fuP2?K33;!fkMj5y7h4fzDREs(7m-HQIA_7~#gR04M9M>SRs{b^|MD=A) cgY;I|uPG+D{wDmvI9Pqk4RMgiqZNzbe@IDt761SM literal 0 HcmV?d00001 diff --git a/src/__pycache__/step2_intelligence_layer_call.cpython-311.pyc b/src/__pycache__/step2_intelligence_layer_call.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..17962cd0ed2bfbcc37db6ef1a9ac1dadced9e14d GIT binary patch literal 2995 zcma)8UuYc18K2$zv#UEx3Ocv8a#wc~IX1EtNv>Uo9;&HylI?S3$FXJX9CDA%?MPZ> z?{+&gXGxd4a+DO@Hjt(itOq}+r8q81p%g-1S}^HLA9uq-#e%_5{1EzbU<$@h{bu&= zq$^I`+1>Bv&-eeE`OSUO-=9Fx?mqp^(z|^K{fiGeE!yrpxdEL!$UufFoSqy% z3G|KZ$;rvdiBi`TDCLes;Ctc&fdHfrP#w*pdobFDFKi9DE;S-d>WmSp%lBZk-JPjN z-V=*t*pQc{`Zw~{Ae}X^TbFAf(~w`FSEQJ{Tz>%{*J?z5E)RE|1g~XCUQ|B^ZYDHn z3;KJQYHS5-1kDl>S#7$a z>4fMT+M3B$gk(Ec!hES)aY!?-ObWgyhk)Ec%dm59CK^&5w*77uK9)aBLKUR1xa1$v zs2s@SlR+$WkMBfHd$yd*E?%RU(8XE(_3P$h-oW~0Vz1&7ql@|6TUU!_g<;DwmqpB$ zzJW;*%0yjjNg73b-XTlu>j6(ERK$@tvA02!H9J{{^ z#5-`>2dAU^AEmVB{9`5ICexmh@s-SG{Y&Mzs~q3Sd}s6I*4Sq$_xp3+iL?HRv)-#` zz4STvLcvQHT=n3?=Lw~c|0(BA{>U4f_Q$3l%zI;(-Lbdf_0v~CWG9(+qiNC){`3Hk z0uusw{s9}~4q`Cuv^olb!I0pQ@5+rxA7Y%h-B!B*R+j;4xTekuPnT$QsTZI=jobqb z+g&Y1tE<1m&wqQt^nW$%)n1SOKN^+*)PGWr^^pT^dtHMAai z2dzn?2|Wu*6i~6SVgkNaDaICjMV z%(eHxS4A&l4d9IedUo;2(3vhQ0+ik|4QzBEEyUpPNOQ>$Cxs4p)~JtpSTk}zIBXtaP*1@fMSg@omDBsWB};GZ4yQX3qlpYO|(^N zS3-NWmvqOm%#zNNHo@lsFynfOU--3sIU8WG!d_q4a4^g(D_A!GHG?QuRr^-rEWgci zczSmFiZ%m`_Qm;ZZ7RP9J2m)^ph0|aZ^7Hmx?#grQL&kZ*8%FrwK>j_)!uKIzlL(w zDvvIOEuq1ySyM5HqS}hXyf+KyArE&e*s6 z?61$Zm2RLh_N-Xm2wsyW9C=MhXC}m8kzKkh9_`FEaLU>H`4c3eQ zu(?7|Cr5yrUsUwpK$_@3`%&`n)~c77Y~~&f9o{*3tQCt4##;zT^IS{a7rP)mOr?MO z!>!c)<-g7T^}++@P0jmL^KR;*m%8YuF5XrjtNrf45l=npt4Ce+=tH&t_KH97s;7?l z>WHh3h>4HNzbya4{?PW+L%w>*RS)e99`FZ8wr+fK^Y=G@SNucK9X#c#r=C7c?r)(O zSnZ*4lzLnG=g{>phpxLf-tmSC{!rnz@^~P9Kl10<&)@vZn-3K4)Or8Zd3PZ14dnfS zysPG)LTe?tF6dC6_K##oZ#ZNI1#}L>Fm)rcOcSsun&&j==&4QCbDkib#^Qg9e-8*+I#s zcy>@}866rs$&KR(wdxv{08EOG`XS&!@tz{{lkw B7%>0< literal 0 HcmV?d00001 diff --git a/src/environment_variables.py b/src/environment_variables.py new file mode 100644 index 0000000..ffc51cb --- /dev/null +++ b/src/environment_variables.py @@ -0,0 +1,4 @@ +import os + +PROMETHEUS_BASE_URL = os.getenv('PROMETHEUS_BASE_URL', 'http://91.138.223.127:30008/api/v1/query_range') +INTELLIGENCE_API_BASE_URL = os.getenv('INTELLIGENCE_API_BASE_URL', 'http://10.160.3.151:3000/') \ No newline at end of file diff --git a/src/metric_helpers.py b/src/metric_helpers.py new file mode 100644 index 0000000..7ddb270 --- /dev/null +++ b/src/metric_helpers.py @@ -0,0 +1,62 @@ +from prometheus_client import CollectorRegistry +from enum import Enum +from pydantic import BaseModel +from typing import Union, Dict, Optional + +# Initialize the custom registry +my_registry = CollectorRegistry() + + +def set_metric_info(metric_name: str, metric_info: str | None) -> str: + if metric_info is None: + metric_info = metric_name + ' info' + return metric_info + + +def set_label_keys(labels: Dict[str, str | int | float] | None) -> list[str]: + # if labels exist, create the label keys + if labels: + # Extract keys and store them in a list + labels_keys = list(labels.keys()) + else: + labels_keys = [] + return labels_keys + + +# create the enums of metric types +class MetricType(Enum): + Counter = 1 + Gauge = 2 + Info = 3 + Enum = 4 + + +class MetricItemRequest(BaseModel): + type: MetricType + metric_name: str + metric_info: Optional[str] = None + value: Union[float, str, dict[str, str | float]] + labels: Optional[Dict[str, str | int | float]] = {} + states: Optional[list[str]] = [] + + +class UnregisterMetricItemRequest(BaseModel): + metric_name: str + + +class CreateModelMetricItemRequest(BaseModel): + type: MetricType + metric_name: str + metric_info: Optional[str] = None + labels: Optional[Dict[str, str | int | float]] = {} + states: Optional[list[str]] = [] + telemetry_metric: str + model_route: str + model_name: str + model_type: str + step_in_seconds: int + sequence_size: int + + +class StopModelMetricItemRequest(BaseModel): + metric_names: list[str] diff --git a/src/metric_types_functions.py b/src/metric_types_functions.py new file mode 100644 index 0000000..2c9985b --- /dev/null +++ b/src/metric_types_functions.py @@ -0,0 +1,158 @@ +from typing import Optional, Dict, Union, List + +from prometheus_client import Gauge, Counter, Info, Enum +from prometheus_client.registry import Collector +from src.metric_helpers import my_registry, set_metric_info, set_label_keys + + +def counter(existing_metric: None | Collector, metric_name: str, metric_info: str | None, + labels: Optional[Dict[str, str | int | float]], value: float): + """ + Counters go up, and reset when the process restarts. + + If there is a suffix of _total on the metric name, it will be removed. When exposing the time series for counter, + a _total suffix will be added. This is for compatibility between OpenMetrics and the Prometheus text format, + as OpenMetrics requires the _total suffix. + + :param existing_metric: The already existing metric if found. + :param metric_name: The metric name. + :param metric_info: The metric info. + :param labels: The labels that will be passed for the metric. + :param value: The amount to increment the counter. + + :return: null. + """ + # set metric info + metric_info = set_metric_info(metric_name=metric_name, metric_info=metric_info) + # if the metric does not exist, create it + if existing_metric is None: + # set label keys for the first creation of metric + label_keys = set_label_keys(labels=labels) + # Initialize a Counter metric + if label_keys: + c = Counter(name=metric_name, documentation=metric_info, labelnames=label_keys, registry=my_registry) + c.labels(**labels).inc(amount=value) + else: + c = Counter(name=metric_name, documentation=metric_info, registry=my_registry) + c.inc(amount=value) + else: + # Increase the Counter metric + if labels: + existing_metric.labels(**labels).inc(amount=value) + else: + existing_metric.inc(amount=value) + + +def gauge(existing_metric: None | Collector, metric_name: str, metric_info: str | None, + labels: Optional[Dict[str, str | int | float]], value: Union[float, str]): + """ + Gauges can go up and down. + + :param existing_metric: The already existing metric if found. + :param metric_name: The metric name. + :param metric_info: The metric info. + :param labels: The labels that will be passed for the metric. + :param value: The value to set the gauge. It must be a float or a parsable to float string. + + :return: null. + """ + # set metric info + metric_info = set_metric_info(metric_name=metric_name, metric_info=metric_info) + # if the metric does not exist, create it + if existing_metric is None: + # set label keys for the first creation of metric + label_keys = set_label_keys(labels=labels) + # Initialize a Gauge metric + if label_keys: + g = Gauge(name=metric_name, documentation=metric_info, labelnames=label_keys, registry=my_registry) + g.labels(**labels).set(value=value) + else: + g = Gauge(name=metric_name, documentation=metric_info, registry=my_registry) + g.set(value) + else: + # Set the Gauge metric value + if labels: + existing_metric.labels(**labels).set(value=value) + else: + existing_metric.set(value=value) + + +def info(existing_metric: None | Collector, metric_name: str, metric_info: str | None, + labels: Optional[Dict[str, str | int | float]], value: Dict[str, str | float]): + """ + Info tracks key-value information, usually about a whole target. + + :param existing_metric: The already existing metric if found. + :param metric_name: The metric name. + :param metric_info: The metric info. + :param labels: The labels that will be passed for the metric. + :param value: The key-value information dictionary of the info. + + :return: null. + """ + # set metric info + metric_info = set_metric_info(metric_name=metric_name, metric_info=metric_info) + # make all value properties strings + for value_key in value.keys(): + value[value_key] = str(value[value_key]) + # if the metric does not exist, create it + if existing_metric is None: + # set label keys for the first creation of metric + label_keys = set_label_keys(labels=labels) + # Initialize an Info metric + if label_keys: + i = Info(name=metric_name, documentation=metric_info, labelnames=label_keys, registry=my_registry) + i.labels(**labels).info(val=value) + else: + i = Info(name=metric_name, documentation=metric_info, registry=my_registry) + i.info(val=value) + else: + # clear labels from already passed value keys + if labels: + for value_key in value.keys(): + labels.pop(value_key, None) + # Set the Info metric value + if labels: + existing_metric.labels(**labels).info(val=value) + else: + existing_metric.info(val=value) + + +def enum(existing_metric: None | Collector, metric_name: str, metric_info: str | None, + labels: Optional[Dict[str, str | int | float]], states: List[str], state: str): + """ + Enum tracks which of a set of states something is currently in. + + :param existing_metric: The already existing metric if found. + :param metric_name: The metric name. + :param metric_info: The metric info. + :param labels: The labels that will be passed for the metric. + :param states: The states that will be available for the metric at its creation. + :param state: The state to be set. + + :return: null. + """ + # if state passed does not exist in states passed (at creation/update) return Value Error + if state not in states: + raise ValueError('state not in states.') + # set metric info + metric_info = set_metric_info(metric_name=metric_name, metric_info=metric_info) + # if the metric does not exist, create it + if existing_metric is None: + # set label keys for the first creation of metric + label_keys = set_label_keys(labels=labels) + # Initialize an Enum metric + if label_keys: + e = Enum(name=metric_name, documentation=metric_info, labelnames=label_keys, states=states, + registry=my_registry) + e.labels(**labels).state(state=state) + else: + e = Enum(name=metric_name, documentation=metric_info, states=states, registry=my_registry) + e.state(state=state) + else: + # need to pop the label with metric name as ".state(...)" sets it again + labels.pop(metric_name, None) + if labels: + existing_metric.labels(**labels).state(state=state) + else: + existing_metric.state(state=state) diff --git a/src/metrics_generator.py b/src/metrics_generator.py new file mode 100644 index 0000000..c289e87 --- /dev/null +++ b/src/metrics_generator.py @@ -0,0 +1,514 @@ +import logging +import time +import threading +from datetime import datetime +from fastapi import FastAPI, HTTPException +from prometheus_client import make_asgi_app +from prometheus_client.multiprocess import MultiProcessCollector +from prometheus_client.registry import Collector +from src.metric_helpers import my_registry, MetricType, MetricItemRequest, UnregisterMetricItemRequest +from src.metric_helpers import CreateModelMetricItemRequest, StopModelMetricItemRequest +from src.metric_types_functions import counter, gauge, info, enum +from src.step1_querry_to_premetheus import create_prometheus_range_query_url, call_prometheus_query_url_with_timeout +from src.step2_intelligence_layer_call import call_intelligence_api_model, prepare_results_for_model_input +from src.environment_variables import PROMETHEUS_BASE_URL + + +# Using multiprocess collector for registry +def make_metrics_app(custom_registry): + MultiProcessCollector(custom_registry) + return make_asgi_app(registry=custom_registry) + + +# Create app +app = FastAPI(debug=False) +# set a logger +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) +# Initialize the custom registry +registry = my_registry +# Add prometheus asgi middleware to route /metrics requests +metrics_app = make_asgi_app(registry) +app.mount("/metrics", metrics_app) +# Global dictionary to store threads and stop events +threads = {} +stop_events = {} + + +# Function to get an existing metric by name from the registry +def get_metric_by_name_and_type(metric_name: str, metric_type: MetricType) -> None | Collector: + """ + Retrieves the metric that was registered with the given name. + + :param metric_name: The name of the metric that must be found. + :param metric_type: The type of the metric to check in order to avoid the possibility that a metric with the same + name but different type exists. + + :return: The metric that was found else None + """ + for collector in registry._collector_to_names.keys(): + # Check if this collector is the one we're looking for based on its name + if metric_name in registry._collector_to_names[collector]: + collector_type = type(collector).__name__.lower() + metric_type = metric_type.name.lower() + if collector_type != metric_type: + http_err = 'Metric name matches an already registered metric with different type.' + logger.error(http_err) + raise HTTPException(status_code=400, detail=http_err) + return collector + return None + + +# Get the labels that were set at the first registration of a metric +def get_existing_metric_labels(metric: Collector) -> dict[str, str] | None: + """ + Retrieve the labels of an existing metric. + + :param metric: The metric as a Collector. Extract from it the type and actual name. + + :return: The labels registered as a dictionary or None + """ + # Assuming `metric` is a Prometheus metric object + metric_name = metric._name + metric_type = type(metric).__name__.lower() + # adjust metric name for a Counter metric + if metric_type == 'counter': + metric_name = metric_name + '_total' + # adjust metric name for an Info metric + if metric_type == 'info': + metric_name = metric_name + '_info' + # adjust Enum metric type (it is 'stateset') + if metric_type == 'enum': + metric_type = 'stateset' + for metric_registered in registry.collect(): + if metric_registered.type == metric_type: + for sample in metric_registered.samples: + if sample.name == metric_name: + return sample.labels + return None + + +def get_full_labels_set(metric_name: Collector, request_labels: dict[str, str] | None) -> dict[str, str] | None: + """ + Will check if the labels passed at the request are aligned with the labels the metric expects. + If any label is missing, then set it with empty string ''. + If labels do not match then throw error. + + :param metric_name: The name of the metric. + :param request_labels: The labels that have been passed at request. + + :return: New dictionary with labels and their values or None. + """ + metric_preset_labels = get_existing_metric_labels(metric_name) + if metric_preset_labels: + # Check if the request has only a part of the labels + new_labels = request_labels + for label in metric_preset_labels: + if label not in new_labels: + # Set missing label to '' + new_labels[label] = '' + # If the request has more or different labels than when the metric was registered + if not all(label in metric_preset_labels for label in new_labels): + http_err = 'Request contains more or different labels than the registered metric.' + logger.error(http_err) + raise HTTPException(status_code=400, detail=http_err) + else: + return new_labels + else: + return None + + +@app.get("/") +def read_root(): + return + + +@app.post('/create_metric') +def create_metric(request: MetricItemRequest): + """ + create_metric route will receive a json payload to create or update a metric. + + :param request: The json passed will contain: + + - type (mandatory): The metric type. + - metric_name (mandatory): The name of the metric to be created or retrieved. + - metric_info (optional): The info of the metric to be created or retrieved. + - value (mandatory): The value that will be passed to the metric. + - labels (optional): The dictionary of labels that will be set for the metric. + - states (optional): The list of states if an enum metric is being set for the first time. + + According to the metric type value: + + - Counter = 1 + Counter expects: + - metric_name (mandatory) -> string. If there is a suffix of _total on the metric name, it will be removed. + When exposing the time series for counter, a _total suffix will be added. This is for compatibility between + OpenMetrics and the Prometheus text format, as OpenMetrics requires the _total suffix. + - metric_info (optional) -> string | None. + - value (mandatory): the previous stored value will be incremented with that value -> positive number. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - Gauge = 2 + Gauge expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - value (mandatory): the new value that will be set -> Union[float, str] (must be a parsable to float + string.). + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - Info = 3 + Info expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - value (mandatory): the new value that will be set -> Dict[str, str | float]. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored), + - Enum = 4 + Enum expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - value (mandatory): the state that will be set. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (mandatory at creation of metric): the states that will be the available choice to set the state + (passed only the first time) + + :return: a json response with 400 if error occurs or 200 if metric is saved successfully. + """ + # get the metrics type value. + metric_type = request.type + # get the metric name + # Strip whitespace and check if it's not None or empty + metric_name = request.metric_name.strip() if request.metric_name else None + # get metric info + metric_info = request.metric_info + # get the value passed + value = request.value + # get metric labels + labels = request.labels + # get states + states = request.states + + # if metric name is not passed then return error + if not metric_name: + http_err = 'metric_name is required.' + logger.error(http_err) + raise HTTPException(status_code=400, detail=http_err) + + # if value is not passed then return error + if isinstance(value, str): + if not value.strip(): + # If value is a string, strip whitespace and check if it's empty + http_err = 'value as str is required and cannot be empty or just whitespace.' + logger.error(http_err) + raise HTTPException(status_code=400, detail=http_err) + elif value is None: # Explicitly check for None if it's not a string (assuming float) + http_err = 'value as float is required and cannot be None.' + logger.error(http_err) + raise HTTPException(status_code=400, detail=http_err) + + # check if metric already exists and has the same type + # if it already exists then just update it at the metric runs + try: + existing_metric = get_metric_by_name_and_type(metric_name, metric_type) + except HTTPException as http_exc: + # Log the exception or do additional processing + logger.error('HTTPException: {}'.format(http_exc.detail)) + # Re-raise the HTTPException for FastAPI to handle + raise http_exc + + # check that the labels passed for calling an existing metric are correct + try: + if existing_metric is not None: + labels = get_full_labels_set(existing_metric, labels) + except HTTPException as http_exc: + # Log the exception or do additional processing + logger.error('HTTPException: {}'.format(http_exc.detail)) + # Re-raise the HTTPException for FastAPI to handle + raise http_exc + + try: + # Update the appropriate metric based on the enum + if metric_type == MetricType.Counter: + counter(existing_metric=existing_metric, metric_name=metric_name, metric_info=metric_info, labels=labels, + value=value) + elif metric_type == MetricType.Gauge: + gauge(existing_metric=existing_metric, metric_name=metric_name, metric_info=metric_info, labels=labels, + value=value) + elif metric_type == MetricType.Info: + if not isinstance(value, dict): + http_err = 'value at info metric must be a dictionary.' + logger.error(http_err) + raise HTTPException(status_code=400, detail=http_err) + info(existing_metric=existing_metric, metric_name=metric_name, metric_info=metric_info, labels=labels, + value=value) + elif metric_type == MetricType.Enum: + enum(existing_metric=existing_metric, metric_name=metric_name, metric_info=metric_info, labels=labels, + states=states, state=value) + except ValueError as e: + # Log the exception + logger.error('HTTPException: {}'.format(e)) + # Raise the HTTPException for FastAPI to handle + raise HTTPException(status_code=400, detail='{}'.format(e)) + + logger.info('Time: {}, metrics name: {}, value: {}'.format( + datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3], metric_name, value)) + + return {'message': 'Metric updated successfully.'} + + +# unregister metrics that have been created +@app.post('/unregister_metric') +def unregister_metric(request: UnregisterMetricItemRequest): + """ + unregister_metric route will receive a json payload to unregister a metric. + + :param request: The json passed will contain: + + - metric_name (mandatory): The name of the metric to be unregistered. + + :return: a json response (200) if metric is unregistered successfully. + """ + # check if counter metric already exists and has the same type + # if it already exists then reset it + try: + existing_counter_metric = get_metric_by_name_and_type(request.metric_name, MetricType.Gauge) + if existing_counter_metric is not None: + registry.unregister(existing_counter_metric) + return {'message': 'Unregistered metric successfully.'} + return {'message': 'Metric not found. You can create a new one.'} + except HTTPException as http_exc: + # Log the exception or do additional processing + logger.error('HTTPException: {}'.format(http_exc.detail)) + # Re-raise the HTTPException for FastAPI to handle + raise http_exc + + +def repeated_operation(request: CreateModelMetricItemRequest, exception_list, first_cycle_done): + """ + The whole operation that will run repeatedly to get data from Prometheus/Thanos, call an intelligence api model and + post the metric. + + :param request: The request contains all the info needed (model name, query, sequence size, steps etc.). + :param exception_list: A list to store exceptions. + :param first_cycle_done: An Event to signal the completion of the first cycle. + + :return: None + """ + query = request.telemetry_metric + step_in_seconds = request.step_in_seconds + sequence_size = request.sequence_size + + try: + # create the url for the query + query_url = create_prometheus_range_query_url(PROMETHEUS_BASE_URL, query, step_in_seconds, sequence_size) + # call the query url created to get the results + query_results = call_prometheus_query_url_with_timeout(query_url, timeout=step_in_seconds-1) + # check that a result is returned and results is filled with data + if query_results is not None and len(query_results) > 0: + # prepare the input data for the model + model_input_data = prepare_results_for_model_input(query_results, sequence_size) + # run the model and save the result + model_result_status_code, model_result = call_intelligence_api_model(request, model_input_data) + # If model_result_status_code is not 200, exception must be thrown for error with intelligence API + # communication + if model_result_status_code != 200: + http_err = 'Intelligence API error or endpoint does not exist.' + raise HTTPException(status_code=400, detail=http_err) + model_result = model_result[0][0] + # post the result + data = request.dict(include={ + 'type', + 'metric_name', + 'metric_info', + 'labels', + 'states' + }) + data['value'] = model_result + create_metric(MetricItemRequest(**data)) + else: + # If result is None, exception must be thrown for empty data + http_err = 'Telemetry metric not found or returned null results.' + raise HTTPException(status_code=400, detail=http_err) + except Exception as e: + exception_list.append(e) + finally: + first_cycle_done.set() + + +def create_model_telemetry_metric(request: CreateModelMetricItemRequest, exception_list, first_cycle_done, stop_event): + """ + create_model_telemetry_metric will receive a json payload to create a metric based on specific telemetry data + that will be retrieved and a model that must exist at Intelligence layer. + + :param request: The json passed at create_model_metric route. + :param exception_list: used to catch the error that could occur at the first execution. + :param first_cycle_done: An Event to signal the completion of the first cycle. + :param stop_event: An Event to signal the alt execution. + + :return: None. + """ + try: + start_time = time.time() + # Run the first cycle + repeated_operation(request, exception_list, first_cycle_done) + if exception_list: + raise exception_list[0] + # Wait for the next time interval, taking into account the time already elapsed + time_to_next_interval = max(request.step_in_seconds - (time.time() - start_time), 0) + time.sleep(time_to_next_interval) + + # Start a loop to run the operation repeatedly + while not stop_event.is_set(): + start_time = time.time() + # Run the repeated operation + repeated_operation(request, exception_list, first_cycle_done) + # Wait for the next time interval, taking into account the time already elapsed + # time_to_next_interval = max(request.step_in_seconds - (time.time() - start_time), 0) + time_to_next_interval = max(request.step_in_seconds - (time.time() - start_time), 0) + time.sleep(time_to_next_interval) + except Exception as e: + return e + + +# create a metric based telemetry metric provided and model that will run +@app.post('/create_model_metric') +def create_model_metric_endpoint(request: CreateModelMetricItemRequest): + """ + create_model_metric route will receive a json payload to create a metric based on specific telemetry data + that will be retrieved and a model that must exist at Intelligence layer. + + :param request: The json passed will contain: + + - type (mandatory): The metric type. + - metric_name (mandatory): The name of the metric to be created or retrieved. + - metric_info (optional): The info of the metric to be created or retrieved. + - labels (optional): The dictionary of labels that will be set for the metric. + - states (optional): The list of states if an enum metric is being set for the first time. + - telemetry_metric (mandatory): The query of the telemetry metric from witch data will be retrieved. + - model_route (mandatory): The route of the model where it can be inferred from Intelligence API. + - model_name (mandatory): The name of the model where the retrieved telemetry data will be sent. + - model_type (mandatory): The type of the model where the retrieved telemetry data will be sent. + - step_in_seconds (mandatory): The time distance between each sample at telemetry metric. + - sequence_size (mandatory): The amount of samples that will be used. + + According to the metric type value: + + - Counter = 1 + Counter expects: + - metric_name (mandatory) -> string. If there is a suffix of _total on the metric name, it will be removed. + When exposing the time series for counter, a _total suffix will be added. This is for compatibility between + OpenMetrics and the Prometheus text format, as OpenMetrics requires the _total suffix. + - metric_info (optional) -> string | None. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - telemetry_metric (mandatory) -> string. + - model_route (mandatory) -> string. + - model_name (mandatory) -> string. + - model_type (mandatory) -> string. + - step_in_seconds (mandatory) -> int. + - sequence_size (mandatory) -> int. + - Gauge = 2 + Gauge expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - telemetry_metric (mandatory) -> string. + - model_route (mandatory) -> string. + - model_name (mandatory) -> string. + - model_type (mandatory) -> string. + - step_in_seconds (mandatory) -> int. + - sequence_size (mandatory) -> int. + - Info = 3 + Info expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (ignored). + - telemetry_metric (mandatory) -> string. + - model_route (mandatory) -> string. + - model_name (mandatory) -> string. + - model_type (mandatory) -> string. + - step_in_seconds (mandatory) -> int. + - sequence_size (mandatory) -> int. + - Enum = 4 + Enum expects: + - metric_name (mandatory) -> string. + - metric_info (optional) -> string | None. + - labels (optional) -> Optional[Dict[str, str | int | float]]. + - states (mandatory at creation of metric): the states that will be the available choice to set the state + (passed only the first time) + - telemetry_metric (mandatory) -> string. + - model_route (mandatory) -> string. + - model_name (mandatory) -> string. + - model_type (mandatory) -> string. + - step_in_seconds (mandatory) -> int. + - sequence_size (mandatory) -> int. + + :return: a json response 400 if error occurs or 200 if telemetry data are found, model inference is successful and + model results are sent to Prometheus/Thanos. + """ + try: + # Create a stop event for this specific request + stop_event = threading.Event() + # Run the first cycle and send immediate response + exception_list = [] + first_cycle_done = threading.Event() + first_cycle_thread = threading.Thread(target=create_model_telemetry_metric, args=(request, exception_list, + first_cycle_done, stop_event)) + first_cycle_thread.start() + first_cycle_done.wait() # Wait for the first cycle to complete + if exception_list: + raise exception_list[0] + + # Store the thread and stop event in the global dictionaries + threads[request.metric_name] = first_cycle_thread + stop_events[request.metric_name] = stop_event + + return {'message': 'First cycle completed successfully. Metric creation started.'} + except Exception as e: + http_err = 'An error occurred in create_model_metric_endpoint: {}'.format(e) + logger.error(http_err) + raise HTTPException(status_code=400, detail='{}'.format(e)) + + +@app.post('/stop_model_metrics') +def stop_model_metrics(request: StopModelMetricItemRequest): + """ + stop_model_metrics route will receive a json payload to stop the metric creations based on specific telemetry data. + + :param request: The json passed will contain: + + - metric_names (mandatory): A list with the names of the metrics to be stopped. + + :return: a json response 200 if the metric creations are stopped successfully even if metric may not exist. + """ + try: + for request_metric_name in request.metric_names: + # Get the metric name + metric_name = request_metric_name.strip() if request_metric_name else None + if not metric_name: + raise HTTPException(status_code=400, detail='metric_name is required.') + + # Stop the corresponding thread + if metric_name in stop_events: + stop_events[metric_name].set() + threads[metric_name].join() + # Clean up the global dictionaries + del threads[metric_name] + del stop_events[metric_name] + # else: + # raise HTTPException(status_code=400, detail='Metric not found.') + return {'message': 'Metric creation(s) stopped successfully.'} + except Exception as e: + http_err = 'An error occurred in stop_model_metric: {}'.format(e) + logger.error(http_err) + raise HTTPException(status_code=400, detail='{}'.format(e)) + + +@app.on_event("shutdown") +def shutdown_event(): + for event in stop_events.values(): + event.set() + for thread in threads.values(): + thread.join() diff --git a/src/step1_querry_to_premetheus.py b/src/step1_querry_to_premetheus.py new file mode 100644 index 0000000..bf60b29 --- /dev/null +++ b/src/step1_querry_to_premetheus.py @@ -0,0 +1,77 @@ +import urllib.parse +from datetime import datetime, timedelta +import requests + + +def create_prometheus_range_query_url(base_url, query, step_in_seconds, sequence_size, end_time=None): + """ + Creates the URL that will ask prometheus/Thanos for the specific metric. + + :param base_url: The base url of prometheus/Thanos API. + :param query: The query to get the specific metric. Query must secure the uniqueness of the response results. + :param step_in_seconds: The step at witch the query will get the past values from prometheus/Thanos. + :param sequence_size: How many past values are ideally wanted. + :param end_time: Till what time to query. Default will be the current time that the function is called. + + :return: The url with all the info. + """ + + # Get the current time in ISO 8601 format with 'Z' to indicate UTC time + end_timestamp = end_time + if end_timestamp is None: + end_timestamp = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + + # Convert string to datetime object + start_timestamp = datetime.strptime(end_timestamp, '%Y-%m-%dT%H:%M:%SZ') + # Subtract the sequence_size * step_in_seconds + start_timestamp = start_timestamp - timedelta(seconds=(sequence_size+1)*step_in_seconds) + # Convert back to ISO 8601 format string with 'Z' + start_timestamp = start_timestamp.strftime('%Y-%m-%dT%H:%M:%SZ') + + # set step to a format of seconds that prometheus/Thanos understand + step = str(step_in_seconds) + 's' + + # Construct the parameters + params = { + 'query': query, + 'start': start_timestamp, + 'end': end_timestamp, + 'step': step + } + + # URL encode the parameters + encoded_params = urllib.parse.urlencode(params) + + # Combine with the base URL + full_url = f"{base_url}?{encoded_params}" + return full_url + + +def call_prometheus_query_url_with_timeout(url, timeout): + """ + Call the url provided for querying prometheus/Thanos + + :param url: prometheus/Thanos query url. + :param timeout: The amount of time in seconds to wait till the call is completed. + + :return: The results of the query in a list or tuples [(timestamp, value)] + """ + try: + response = requests.get(url, timeout=timeout) + if response.status_code == 200: + res = response.json() + if res['data']: + if res['data']['result'] and len(res['data']['result']) > 0: + return res['data']['result'][0]['values'] + else: + return [] + else: + return [] + else: + return [] + except requests.exceptions.Timeout: + print("Request timed out.") + return [] + except requests.exceptions.RequestException as e: + print(f"An HTTP error occurred: {e}") + return [] diff --git a/src/step2_intelligence_layer_call.py b/src/step2_intelligence_layer_call.py new file mode 100644 index 0000000..1c30c21 --- /dev/null +++ b/src/step2_intelligence_layer_call.py @@ -0,0 +1,58 @@ +import requests +import json +from fastapi import HTTPException +from src.environment_variables import INTELLIGENCE_API_BASE_URL +from src.metric_helpers import CreateModelMetricItemRequest + + +def prepare_results_for_model_input(results, sequence_size): + """ + Takes the results from the prometheus/Thanos query and prepares them as an input for the model call. + + :param results: The results returned from prometheus/Thanos query, a list of tuples. + :param sequence_size: The amount of past values that the model will take as input. + + :return: An array with the results + """ + # Extract the second value from each tuple that is the metric value + extracted_values = [value[1] for value in results] + # Desired size of the new list is the sequence size provided + desired_size = sequence_size + # Prepend zeros if the extracted list is smaller than the desired size + if len(extracted_values) < desired_size: + extracted_values = [0] * (desired_size - len(extracted_values)) + extracted_values + if len(extracted_values) > desired_size: + extracted_values = extracted_values[len(extracted_values) - desired_size:] + return extracted_values + + +def call_intelligence_api_model(request: CreateModelMetricItemRequest, input_data): + """ + This function will call the intelligence api endpoint that corresponds to the model name passed with the data + provided. + + :param request: The request from which information to call Intelligence API for the model inference will be + retrieved. + :param input_data: The data to pass to the model. + + :return: Response status code and response data as a json. + """ + url = INTELLIGENCE_API_BASE_URL + request.model_route + headers = { + 'accept': 'application/json', + 'Content-Type': 'application/json', + } + data = json.dumps({ + "model_tag": request.model_name, + "model_type": request.model_type, + "input_series": input_data + }) + try: + response = requests.post(url, headers=headers, data=data) + return response.status_code, response.json() + except Exception as e: + # If model_result_status_code is not 200, exception must be thrown for error with intelligence API + # communication + message = 'Intelligence API error or endpoint does not exist. Error: {}'.format(e) + # Raise the HTTPException for FastAPI to handle + raise HTTPException(status_code=400, detail='{}'.format(message))