Prometheus ec2 monitoring

ansible/deploy-clickhouse-proxy.yml

@@ -5,12 +5,17 @@
   become: true
   roles:
     - role: bootstrap
+    - role: dehydrated
+      vars: 
+        ssl_domains: 
+          - clickhouseproxy.dev.ooni.io
     - role: nginx
       tags: nginx
     - role: clickhouse_proxy
       vars: 
         clickhouse_url: "clickhouse3.prod.ooni.io"
         clickhouse_port: 9000
+        clickhouse_proxy_public_fqdn: "clickhouseproxy.dev.ooni.io"
     - role: dehydrated
       vars:
         ssl_domains: "clickhouseproxy.dev.ooni.io"

ansible/roles/clickhouse_proxy/defaults/main.yml

@@ -0,0 +1 @@
+tls_cert_dir: /var/lib/dehydrated/certs

ansible/roles/clickhouse_proxy/tasks/main.yml

@@ -9,6 +9,17 @@
   notify:
     - reload nftables  
 
+# For prometheus scrape requests
+- name: Allow traffic on port 9200
+  tags: prometheus-proxy
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/9200.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 9200 counter accept comment "prometheus"
+  notify:
+    - reload nftables  
+
 - name: Create the modules-enabled directory if not exists
   tags: webserv
   ansible.builtin.file:
@@ -28,3 +39,14 @@
   notify:
     - reload nginx
     - restart nginx
+
+- name: Add prometheus proxy nginx config
+  tags: webserv
+  template:
+    src: templates/prometheus-proxy.conf
+    dest: /etc/nginx/conf.d/prometheus-proxy.conf
+    mode: 0755
+    owner: root
+  notify:
+    - reload nginx
+    - restart nginx

ansible/roles/clickhouse_proxy/templates/prometheus-proxy.conf

@@ -0,0 +1,16 @@
+server {
+    listen 9200 ssl;
+
+    server_name {{ clickhouse_proxy_public_fqdn }};
+
+    include /etc/nginx/ssl_intermediate.conf;
+
+    ssl_certificate {{tls_cert_dir}}/{{inventory_hostname}}/fullchain.pem;
+    ssl_certificate_key {{tls_cert_dir}}/{{inventory_hostname}}/privkey.pem;
+    ssl_trusted_certificate {{tls_cert_dir}}/{{inventory_hostname}}/chain.pem;
+
+    proxy_ssl_server_name on;
+    location ~ /([a-zA-Z0-9_\.]+)/(.*) {
+        proxy_pass http://$1:9100/$2$is_args$args;
+    }
+}

ansible/roles/prometheus/templates/prometheus.yml

@@ -214,4 +214,38 @@ scrape_configs:
     static_configs:
       - targets:
         - backend-hel.ooni.org:444
+
+  # EC2 instances monitoring: 
+  - job_name: 'ooni-api-ec2'
+    scrape_interval: 5s
+    scheme: https 
+    metrics_path: "/metrics" 
+
+    # Node level metrics for cluster nodes
+    ec2_sd_configs:
+      - access_key: "{{prometheus_aws_access_key}}" 
+        secret_key: "{{prometheus_aws_secret_key}}"
+        region: "eu-central-1"
+        port: 9100 # should be the proxy 
+    relabel_configs: # Change the host to the proxy host with relabeling
+      - source_labels: [__address__]
+        regex: "([0-9\\.]+):([0-9]+)" # <ip>:<port>"
+        replacement: "$1"
+        target_label: "ec2_host"
+        action: "replace"
+      - source_labels: [__address__]
+        regex: "([0-9\\.]+):([0-9]+)" # <ip>:<port>
+        replacement: "{{clickhouse_proxy_host_dev}}:9200/${1}/metrics"
+        target_label: "proxy_host"
+        action: "replace"
+      - source_labels: [proxy_host]
+        regex: "([^/]*)/(.*)"
+        replacement: "$1"
+        target_label: "__address__"
+        action: "replace"
+      - source_labels: [proxy_host]
+        regex: "([^/]*)/(.*)"
+        replacement: "/$2"
+        target_label: "__metrics_path__"
+        action: "replace"
 ...

ansible/roles/prometheus/vars/main.yml

@@ -150,3 +150,9 @@ blackbox_jobs:
   - name: icmp
     module: icmp
     targets: "{{ dom0_hosts | list }}"
+
+prometheus_aws_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_monitoring/access_key', profile='oonidevops_user_dev') }}"
+prometheus_aws_secret_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_monitoring/secret_key', profile='oonidevops_user_dev') }}"
+
+clickhouse_proxy_host_dev: "clickhouseproxy.dev.ooni.io"
+clickhouse_proxy_host_prod: "clickhouseproxy.dev.ooni.io" # TODO Change for prod

tf/environments/dev/main.tf

@@ -301,6 +301,13 @@ module "ooniapi_cluster" {
 
   instance_type = "t3a.micro"
 
+  monitoring_sg_ids = [
+    # The clickhouse proxy has an nginx configuration
+    # to proxy requests from the monitoring server
+    # to the cluster instances
+    module.ooni_clickhouse_proxy.ec2_sg_id
+  ]
+
   tags = merge(
     local.tags,
     { Name = "ooni-tier0-api-ecs-cluster" }
@@ -411,6 +418,10 @@ module "ooniapi_reverseproxy" {
   )
 }
 
+data "dns_a_record_set" "monitoring_host" {
+  host = "monitoring.ooni.org"
+}
+
 module "ooni_clickhouse_proxy" {
   source = "../../modules/ec2"
 
@@ -426,31 +437,37 @@ module "ooni_clickhouse_proxy" {
 
   name = "oonickprx"
   ingress_rules = [{
-    from_port = 22,
-    to_port = 22,
-    protocol = "tcp",
+    from_port   = 22,
+    to_port     = 22,
+    protocol    = "tcp",
     cidr_blocks = ["0.0.0.0/0"],
-  }, {
-    from_port = 80,
-    to_port = 80,
-    protocol = "tcp",
+    }, {
+    from_port   = 80,
+    to_port     = 80,
+    protocol    = "tcp",
     cidr_blocks = ["0.0.0.0/0"],
-  }, {
-    from_port = 9000,
-    to_port = 9000,
-    protocol = "tcp",
+    }, {
+    from_port   = 9000,
+    to_port     = 9000,
+    protocol    = "tcp",
     cidr_blocks = module.network.vpc_subnet_private[*].cidr_block,
+    }, {
+    // For the prometheus proxy:
+    from_port   = 9200,
+    to_port     = 9200,
+    protocol    = "tcp"
+    cidr_blocks = [for ip in flatten(data.dns_a_record_set.monitoring_host.*.addrs) : "${tostring(ip)}/32"]
   }]
 
   egress_rules = [{
-    from_port = 0,
-    to_port = 0,
-    protocol = "-1",
+    from_port   = 0,
+    to_port     = 0,
+    protocol    = "-1",
     cidr_blocks = ["0.0.0.0/0"],
-  }, {
-    from_port = 0,
-    to_port = 0,
-    protocol = "-1",
+    }, {
+    from_port        = 0,
+    to_port          = 0,
+    protocol         = "-1",
     ipv6_cidr_blocks = ["::/0"]
   }]
 
@@ -792,6 +809,9 @@ resource "aws_acm_certificate_validation" "ooniapi_frontend" {
 ### Ooni monitoring
 
 module "ooni_monitoring" {
-  source = "../../modules/ooni_monitoring"
-  tags = local.tags  
-}
+  source      = "../../modules/ooni_monitoring"
+  environment = local.environment
+  aws_region  = var.aws_region
+
+  tags = local.tags
+}

tf/modules/ec2/outputs.tf

@@ -5,3 +5,7 @@ output "aws_instance_id" {
 output "aws_instance_public_dns" {
     value = aws_instance.ooni_ec2.public_dns
 }
+
+output "ec2_sg_id" {
+  value = aws_security_group.ec2_sg.id
+}

tf/modules/ecs_cluster/main.tf

@@ -120,6 +120,14 @@ resource "aws_security_group" "container_host" {
     ]
   }
 
+  ingress {
+    protocol = "tcp"
+    from_port = 9100
+    to_port = 9100
+
+    security_groups = var.monitoring_sg_ids
+  }
+
   egress {
     from_port        = 0
     to_port          = 0

tf/modules/ecs_cluster/variables.tf

@@ -62,6 +62,10 @@ variable "instance_volume_size" {
   default = "5"
 }
 
+variable "monitoring_sg_ids" {
+  default = []
+}
+
 variable "node_exporter_port" {
   default = "9100"
 }

tf/modules/ooni_monitoring/main.tf

@@ -1,3 +1,11 @@
+locals {
+  name = "ecs-service-discovery-${var.environment}"
+
+  tags = {
+    Name = local.name
+    Environment = var.environment
+  }
+}
 resource "aws_iam_user" "ooni_monitoring" {
   name = "oonidevops-monitoring"
 }
@@ -34,4 +42,4 @@ resource "aws_ssm_parameter" "ooni_monitoring_secret_key" {
   name = "/oonidevops/secrets/ooni_monitoring/secret_key"
   type = "SecureString"
   value = aws_iam_access_key.ooni_monitoring.secret
-}
+}

tf/modules/ooni_monitoring/variables.tf

@@ -2,4 +2,19 @@ variable "tags" {
   description = "tags to apply to the resources"
   default     = {}
   type        = map(string)
-}
+}
+
+variable "environment" {
+  type = string
+}
+
+variable "task_memory" {
+  description = "How much memory to allocate for this task"
+  type = number
+  default = 64
+}
+
+variable "aws_region" {
+  description = "AWS region"
+  type = string
+}