Merge branch 'main' into refactor/pg-pass-prod

ansible/deploy-clickhouse.yml

@@ -1,7 +1,7 @@
 ---
 - name: Deploy oonidata clickhouse hosts
   hosts:
-    - notebook.ooni.org
+    - notebook1.htz-fsn.prod.ooni.nu
     - data1.htz-fsn.prod.ooni.nu
     # - data2.htz-fsn.prod.ooni.nu
     - data3.htz-fsn.prod.ooni.nu

ansible/deploy-notebook.yml

@@ -1,10 +1,13 @@
 ---
 - name: Deploy notebook host
-  hosts: notebook.ooni.org
+  hosts: notebook1.htz-fsn.prod.ooni.nu
   become: true
   tags:
     - notebook
   vars:
-    enable_oonipipeline_worker: false
+    notebook_domain: "notebook.ooni.org"
+    ssl_domains:
+      - "{{ inventory_hostname }}"
+      - "notebook.ooni.org"
   roles:
-    - oonidata
+    - notebook

ansible/group_vars/clickhouse/vars.yml

@@ -5,7 +5,7 @@ nftables_clickhouse_allow:
     ip: 88.198.54.12
   - fqdn: data3.htz-fsn.prod.ooni.nu
     ip: 168.119.7.188
-  - fqdn: notebook.ooni.org
+  - fqdn: notebook1.htz-fsn.prod.ooni.nu
     ip: 138.201.19.39
   - fqdn: backend-hel.ooni.org
     ip: 65.108.192.151
@@ -19,7 +19,7 @@ nftables_zookeeper_allow:
     ip: 88.198.54.12
   - fqdn: data3.htz-fsn.prod.ooni.nu
     ip: 168.119.7.188
-  - fqdn: notebook.ooni.org
+  - fqdn: notebook1.htz-fsn.prod.ooni.nu
     ip: 138.201.19.39
 
 clickhouse_version: 24.8.6.70
@@ -94,9 +94,9 @@ clickhouse_keeper:
     port: 9234
 
   - keeper_server:
-    server: notebook.ooni.org
+    server: notebook1.htz-fsn.prod.ooni.nu
     id: 4
-    hostname: notebook.ooni.org
+    hostname: notebook1.htz-fsn.prod.ooni.nu
     port: 9234
 
 clickhouse_zookeeper:
@@ -107,7 +107,7 @@ clickhouse_zookeeper:
     host: clickhouse3.prod.ooni.io
     port: 9181
   - node:
-    host: notebook.ooni.org
+    host: notebook1.htz-fsn.prod.ooni.nu
     port: 9181
 
 clickhouse_remote_servers:

ansible/inventory

@@ -5,7 +5,7 @@ ghs_ams
 ## Role tags
 
 [clickhouse]
-notebook.ooni.org
+notebook1.htz-fsn.prod.ooni.nu
 data1.htz-fsn.prod.ooni.nu
 data3.htz-fsn.prod.ooni.nu
 
@@ -16,7 +16,7 @@ data1.htz-fsn.prod.ooni.nu
 
 [htz_fsn]
 monitoring.ooni.org
-notebook.ooni.org
+notebook1.htz-fsn.prod.ooni.nu
 data1.htz-fsn.prod.ooni.nu
 data3.htz-fsn.prod.ooni.nu
 #backend-fsn.ooni.org

ansible/password-pipe

@@ -1,2 +1,2 @@
 #!/bin/sh
-exec gpg --quiet --decrypt --batch <~/.ssh/ooni-sysadmin.vaultpw.gpg
+exec aws ssm get-parameter --name /oonidevops/secrets/devops_vault_password --profile oonidevops_user_prod --query "Parameter.Value" --with-decryption --output text

ansible/roles/monitoring/tasks/main.yml

@@ -1,6 +1,14 @@
 ---
 # # monitoring host # #
 
+- name: Create Grafana repo GPG pubkey
+  tags: apt
+  template:
+    src: templates/grafana.asc
+    dest: /etc/apt/grafana.asc
+    mode: 0644
+    owner: root
+
 - name: Set grafana apt repo
   tags: monitoring, grafana
   template:
@@ -9,6 +17,14 @@
     mode: 0644
     owner: root
 
+- name: Create Grafana sources list
+  tags: apt
+  template:
+    src: templates/grafana.sources
+    dest: /etc/apt/sources.list.d/grafana.sources
+    mode: 0644
+    owner: root
+
 - name: Installs packages
   tags: monitoring, prometheus
   apt:
@@ -37,22 +53,6 @@
     mode: 0644
     owner: root
 
-- name: Create Grafana repo GPG pubkey
-  tags: apt
-  template:
-    src: templates/grafana.gpg
-    dest: /etc/apt/grafana.asc
-    mode: 0644
-    owner: root
-
-- name: Create Grafana sources list
-  tags: apt
-  template:
-    src: templates/grafana.sources
-    dest: /etc/apt/sources.list.d/grafana.sources
-    mode: 0644
-    owner: root
-
 - name: Installs grafana
   tags: monitoring, grafana
   apt:
@@ -65,7 +65,7 @@
   tags: monitoring, grafana
   lineinfile:
     path: /etc/grafana/grafana.ini
-    regexp: '^;?domain = '
+    regexp: "^;?domain = "
     line: domain = grafana.ooni.org
 
 - name: Autoremove
@@ -188,7 +188,7 @@
   tags: fail2ban
   lineinfile:
     path: /etc/fail2ban/jail.conf
-    regexp: '^backend '
+    regexp: "^backend "
     line: backend = systemd
 
 - name: Configure fail2ban

ansible/roles/monitoring/templates/grafana.list

@@ -1 +1 @@
-deb https://packages.grafana.com/oss/deb stable main
+deb [signed-by=/etc/apt/grafana.asc] https://apt.grafana.com stable main

ansible/roles/oonidata/defaults/main.yml → ansible/roles/notebook/defaults/main.yml

@@ -2,12 +2,11 @@ miniconda_install_dir: /opt/miniconda
 jupyterhub_config_dir: /etc/jupyterhub
 jupyterhub_runtime_dir: /srv/jupyterhub
 oonipipeline_runtime_dir: /srv/oonipipeline
-tls_cert_dir: /etc/letsencrypt/live
+tls_cert_dir: /var/lib/dehydrated/certs
 admin_group_name: admin
-enable_oonipipeline_worker: true
-enable_jupyterhub: true
 clickhouse_url: "clickhouse://localhost"
-certbot_domains:
+notebook_domain: "{{ inventory_hostname }}"
+ssl_domains:
   - "{{ inventory_hostname }}"
 conda_forge_packages:
   - seaborn

ansible/roles/notebook/handlers/main.yml

@@ -0,0 +1,10 @@
+- name: Restart jupyterhub
+  ansible.builtin.systemd_service:
+    name: jupyterhub
+    state: restarted
+    daemon_reload: true
+
+- name: Reload nginx
+  ansible.builtin.systemd_service:
+    name: nginx
+    state: reloaded

ansible/roles/oonidata/tasks/jupyterhub.yml → ansible/roles/notebook/tasks/jupyterhub.yml

@@ -113,15 +113,7 @@
     - nginx
 
 - ansible.builtin.include_role:
-    name: geerlingguy.certbot
+    name: dehydrated
   tags:
     - oonidata
-    - certbot
-  vars:
-    certbot_admin_email: [email protected]
-    certbot_create_extra_args: ""
-    certbot_create_if_missing: true
-    certbot_create_standalone_stop_services:
-      - nginx
-    certbot_certs:
-      - domains: "{{ certbot_domains }}"
+    - dehydrated

ansible/roles/oonidata/tasks/main.yml → ansible/roles/notebook/tasks/main.yml

@@ -17,7 +17,6 @@
     - conda
 
 - ansible.builtin.import_tasks: jupyterhub.yml
-  when: enable_jupyterhub
   tags:
     - oonidata
     - jupyterhub
@@ -61,9 +60,3 @@
     - oonidata
     - oonipipeline
     - packages
-
-- ansible.builtin.import_tasks: oonipipeline-worker.yml
-  when: enable_oonipipeline_worker
-  tags:
-    - oonidata
-    - oonipipeline

ansible/roles/oonidata/templates/nginx-jupyterhub.j2 → ansible/roles/notebook/templates/nginx-jupyterhub.j2

@@ -5,6 +5,12 @@ map $http_upgrade $connection_upgrade {
     ''      close;
 }
 
+server {
+    listen 80;
+    server_name {{ notebook_domain }};
+    return 301 https://$server_name$request_uri;
+}
+
 server {
     listen 443 ssl http2;
 
@@ -14,9 +20,7 @@ server {
     ssl_certificate_key {{ tls_cert_dir }}/{{ inventory_hostname }}/privkey.pem;
     ssl_trusted_certificate {{ tls_cert_dir }}/{{ inventory_hostname }}/chain.pem;
 
-    server_name _;
-    access_log  /var/log/nginx/{{ inventory_hostname }}.access.log;
-    error_log   /var/log/nginx/{{ inventory_hostname }}.log warn;
+    server_name {{ notebook_domain }};
 
     add_header Access-Control-Allow-Origin *;
 

ansible/roles/oonidata_airflow/defaults/main.yml

@@ -1,2 +1 @@
 tls_cert_dir: /var/lib/dehydrated/certs
-certbot_domains_extra: []