refactor(jupyter): split jupyter into hub and lab roles

- hub role configure and start JupyterHub
- lab role configure JupyterLab and kernels on YARN NM filesystem

The JupyterLab venv is installed directly on YARN NodeManager machine
instead of putting a venv containing JupyterLab in HDFS. It allows faster
startup when a user spawn a JupyterLab and kernels configuration can be
done on worker directly without respawning a JupyterLab.

playbooks/jupyterhub_server_config.yml → playbooks/jupyter_hub_config.yml

@@ -2,13 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: Configure Jupyterhub Server
-  hosts: jupyterhub_server
+- name: Configure Jupyterhub
+  hosts: jupyter_hub
   tasks:
     - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: jupyterhub_server
-    - name: Configure Jupyterhub Server
+        node_name: jupyter_hub
+    - name: Configure Jupyterhub
       ansible.builtin.import_role:
-        name: tosit.tdp_extra.jupyterhub.server
+        name: tosit.tdp_extra.jupyter.hub
         tasks_from: config
     - ansible.builtin.meta: clear_facts # noqa unnamed-task

playbooks/jupyterhub_server_install.yml → playbooks/jupyter_hub_install.yml

@@ -2,13 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: Install Jupyterhub Server
-  hosts: jupyterhub_server
+- name: Install Jupyterhub
+  hosts: jupyter_hub
   tasks:
     - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: jupyterhub_server
-    - name: Install Jupyterhub Server
+        node_name: jupyter_hub
+    - name: Install Jupyterhub
       ansible.builtin.import_role:
-        name: tosit.tdp_extra.jupyterhub.server
+        name: tosit.tdp_extra.jupyter.hub
         tasks_from: install
     - ansible.builtin.meta: clear_facts # noqa unnamed-task

playbooks/jupyterhub_server_restart.yml → playbooks/jupyter_hub_restart.yml

@@ -2,13 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: Restart Jupyterhub Server
-  hosts: jupyterhub_server
+- name: Restart Jupyterhub
+  hosts: jupyter_hub
   tasks:
     - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: jupyterhub_server
-    - name: Restart Jupyterhub Server
+        node_name: jupyter_hub
+    - name: Restart Jupyterhub
       ansible.builtin.import_role:
-        name: tosit.tdp_extra.jupyterhub.server
+        name: tosit.tdp_extra.jupyter.hub
         tasks_from: restart
     - ansible.builtin.meta: clear_facts # noqa unnamed-task

playbooks/jupyterhub_server_start.yml → playbooks/jupyter_hub_start.yml

@@ -2,13 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: Start Jupyterhub Server
-  hosts: jupyterhub_server
+- name: Start Jupyterhub
+  hosts: jupyter_hub
   tasks:
     - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: jupyterhub_server
-    - name: Start Jupyterhub Server
+        node_name: jupyter_hub
+    - name: Start Jupyterhub
       ansible.builtin.import_role:
-        name: tosit.tdp_extra.jupyterhub.server
+        name: tosit.tdp_extra.jupyter.hub
         tasks_from: start
     - ansible.builtin.meta: clear_facts # noqa unnamed-task

playbooks/jupyterhub_server_stop.yml → playbooks/jupyter_hub_stop.yml

@@ -2,13 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: Stop Jupyterhub Server
-  hosts: jupyterhub_server
+- name: Stop Jupyterhub
+  hosts: jupyter_hub
   tasks:
     - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: jupyterhub_server
-    - name: Stop Jupyterhub Server
+        node_name: jupyter_hub
+    - name: Stop Jupyterhub
       ansible.builtin.import_role:
-        name: tosit.tdp_extra.jupyterhub.server
+        name: tosit.tdp_extra.jupyter.hub
         tasks_from: stop
     - ansible.builtin.meta: clear_facts # noqa unnamed-task

playbooks/jupyterhub_kerberos_install.yml → playbooks/jupyter_kerberos_install.yml

@@ -2,14 +2,14 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: Kerberos Jupyterhub Server install
-  hosts: jupyterhub_server
+- name: Kerberos Jupyterhub install
+  hosts: jupyter_hub
   strategy: linear
   tasks:
     - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: jupyterhub_kerberos
-    - name: Install Jupyterhub Server Kerberos
+        node_name: jupyter_kerberos
+    - name: Install Jupyterhub Kerberos
       ansible.builtin.import_role:
-        name: tosit.tdp_extra.jupyterhub.server
+        name: tosit.tdp_extra.jupyter.hub
         tasks_from: kerberos
     - ansible.builtin.meta: clear_facts # noqa unnamed-task

playbooks/jupyterhub_server_init.yml → playbooks/jupyter_lab_config.yml

@@ -2,13 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-- name: Init Jupyterhub Server
-  hosts: jupyterhub_server
+- name: Configure Jupyterlab
+  hosts: yarn_nm
   tasks:
     - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: jupyterhub_server
-    - name: Initialize Jupyterhub Server
+        node_name: jupyter_lab
+    - name: Configure Jupyterlab
       ansible.builtin.import_role:
-        name: tosit.tdp_extra.jupyterhub.server
-        tasks_from: init
+        name: tosit.tdp_extra.jupyter.lab
+        tasks_from: config
     - ansible.builtin.meta: clear_facts # noqa unnamed-task

playbooks/jupyter_lab_install.yml

@@ -0,0 +1,14 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Install Jupyterlab
+  hosts: yarn_nm
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: jupyter_lab
+    - name: Install Jupyterlab
+      ansible.builtin.import_role:
+        name: tosit.tdp_extra.jupyter.lab
+        tasks_from: install
+    - ansible.builtin.meta: clear_facts # noqa unnamed-task

playbooks/jupyterhub_ssl-tls_install.yml → playbooks/jupyter_ssl-tls_install.yml

@@ -3,12 +3,12 @@
 
 ---
 - name: SSL-TLS Jupyterhub Install
-  hosts: jupyterhub_server
+  hosts: jupyter_hub
   tasks:
     - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: jupyterhub_server
-    - name: Install Jupterhub Server SSL-TLS
+        node_name: jupyter_hub
+    - name: Install Jupterhub SSL-TLS
       ansible.builtin.import_role:
-        name: tosit.tdp_extra.jupyterhub.server
+        name: tosit.tdp_extra.jupyter.hub
         tasks_from: ssl-tls
     - ansible.builtin.meta: clear_facts # noqa unnamed-task

roles/jupyter/hub/tasks/config.yml

@@ -37,11 +37,3 @@
     owner: root
     group: root
     mode: "0644"
-
-- name: Template sparkmagic configuration
-  ansible.builtin.template:
-    src: sparkmagic/config.json.j2
-    dest: "{{ jupyterhub_server_conf_dir }}/sparkmagic/config.json"
-    owner: root
-    group: root
-    mode: "0644"

roles/jupyter/hub/tasks/install.yml

@@ -41,7 +41,16 @@
     remote_src: true
     creates: "{{ jupyterhub_root_dir }}/{{ jupyterhub_release }}"
 
-- name: Install Jupyterhub
+- name: Create virtual env and update pip if necessary
+  ansible.builtin.pip:
+    name: pip
+    state: latest
+    extra_args: "--no-index --find-links=file://{{ jupyterhub_root_dir }}/{{ jupyterhub_release }}/dependencies"
+    umask: "0022"
+    virtualenv: "{{ jupyterhub_root_dir }}/{{ jupyterhub_release }}"
+    virtualenv_python: "{{ jupyterhub_python_executable }}"
+
+- name: Install Jupyterhub in virtual env
   ansible.builtin.pip:
     requirements: "{{ jupyterhub_root_dir }}/{{ jupyterhub_release }}/requirements.txt"
     extra_args: "--no-index --find-links=file://{{ jupyterhub_root_dir }}/{{ jupyterhub_release }}/dependencies"
@@ -69,14 +78,6 @@
     owner: root
     mode: "0755"
 
-- name: Create sparkmagic configuration directory
-  ansible.builtin.file:
-    path: "{{ jupyterhub_server_conf_dir }}/sparkmagic"
-    state: directory
-    group: root
-    owner: root
-    mode: "0755"
-
 - name: Create certificates directory
   ansible.builtin.file:
     path: "{{ jupyterhub_server_certs_folder }}"

roles/jupyter/hub/templates/jupyterhub_config.py.j2

@@ -118,7 +118,6 @@ def pre_spawn_hook(spawner):
     # spawner.user_options holds the value parsed from the form
     # values are always list of strings
     queue = spawner.user_options.get("queue", ["default"])[0]
-    venv = spawner.user_options.get("venv", ["yarn_environment"])[0]
     cpu_limit = spawner.user_options.get("cpu", ["1"])[0]
     mem_limit = spawner.user_options.get("memory", ["2G"])[0]
     mem_limit = int(mem_limit) if mem_limit.isnumeric() else mem_limit
@@ -139,7 +138,6 @@ def pre_spawn_hook(spawner):
     spawner.cpu_limit = cpu_limit
     spawner.mem_limit = mem_limit
     spawner.queue = queue
-    spawner.localize_files = { 'environment': { 'source': ("hdfs://{{ jupyterhub_hdfs_venvs_dir }}/" + venv +".tar.gz") } , 'config.json': { 'source': '{{ jupyterhub_server_conf_dir }}/sparkmagic/config.json' } }
 
 {% for key, value in jupyterhub_properties.get('yarnspawner', {}).items() +%}
 c.YarnSpawner.{{ key }} = {% if value is string and not value.startswith('f:') %}'{{ value }}'{% else %}{{ value if value is not string else value.lstrip('f:') }}{% endif %}