From 109819e6c70271d9120fc0a5010891ad6494a0c1 Mon Sep 17 00:00:00 2001
From: Pierre Sauvage <m.sauvage.pierre@gmail.com>
Date: Tue, 18 Feb 2025 10:41:25 +0100
Subject: [PATCH] remove .env, add source.sh, use yq instead of jq

---
 .env       |  10 -----
 destroy.sh |  22 ++++------
 incus.json |  38 ++++++++++++++--
 incus.yml  |  58 ++++++++++++++++++++++++
 launch.sh  | 127 ++++++++++++++++++++++++++++++++++-------------------
 source.sh  |  78 ++++++++++++++++++++++++++++++++
 6 files changed, 262 insertions(+), 71 deletions(-)
 delete mode 100644 .env
 create mode 100644 incus.yml
 create mode 100755 source.sh

diff --git a/.env b/.env
deleted file mode 100644
index 221ee05..0000000
--- a/.env
+++ /dev/null
@@ -1,10 +0,0 @@
-#customizable variables
-network=lxdbr0
-file=incus.json
-user=incus
-iprange=192.168.56.1/24
-hostfile="$TDP_HOME/tdp-getting-started/inventory/hosts.ini"
-datadir=`pwd`/data
-storagedir=${datadir}/pool
-storagepool=tdp
-privatekey=${datadir}/incus_key
diff --git a/destroy.sh b/destroy.sh
index 9ac70f8..543633e 100755
--- a/destroy.sh
+++ b/destroy.sh
@@ -2,22 +2,16 @@
 
 cd "$(dirname "$0")"
 
-# Get customized variables
-source .env
-
-
-#Use incus project to namespace cluster
-project=$(jq ".project" $file | sed 's/"//g')
+source ./source.sh
 
 incus project switch $project
 
 # Delete the VMs
-hostnum=$(jq '.hosts | length' $file | sed 's/"//g')
-for h in $(seq 0 $((hostnum - 1))); do
-    name=$(jq ".hosts[$h].hostname" $file | sed 's/"//g')
+for h in $(seq 0 $((hosts_num - 1))); do
+    name=$(parseconf ".hosts[$h].hostname")
     if [[ "$name" == "null" ]]; then
-	role=$(jq ".hosts[$h].role" $file | sed 's/"//g');
-	qty=$(jq ".hosts[$h].quantity" $file | sed 's/"//g');
+    	role=$(parseconf ".hosts[$h].role");
+    	qty=$(parseconf ".hosts[$h].quantity");
         if [[ "qty" == "null" ]]; then
             qty=1
         fi
@@ -33,11 +27,11 @@ for h in $(seq 0 $((hostnum - 1))); do
 done
 
 # Delete storage pool and network
-incus profile device remove default enp5s0 --project tdp
-incus profile device remove default root --project tdp
+incus profile device remove default enp5s0
+incus profile device remove default root
 incus storage delete $storagepool
+incus storage delete $storage
 incus network delete $network
 
-
 # Do not drop project, it’s not empty (image)
 #incus project delete $project
diff --git a/incus.json b/incus.json
index 67a7b77..91a1670 100644
--- a/incus.json
+++ b/incus.json
@@ -1,7 +1,38 @@
 {
-  "box": "rockylinux/8/cloud",
-  "domain": "tdp",
   "project": "tdp",
+  "image": "rockylinux/8/cloud",
+  "admin_user": "incus",
+  "network": {
+    "manage": true,
+    "name": "tdp",
+    "domain": "tdp",
+    "cidr": "192.168.56.1/24"
+  },
+  "storage": {
+    "manage": true,
+    "name": "tdp",
+    "dir": "./pool",
+    "root_size": "40GB"
+  },
+  "profiles": {
+    "master": {
+      "devices": {
+        "data": {
+          "path": "/data",
+          "size": "100GB"
+        }
+      }
+    },
+    "worker": {
+      "devices": {
+        "data": {
+          "path": "/data",
+          "size": "100GB"
+        }
+      }
+    }
+  },
+  "output_dir": "../tdp-getting-started",
   "hosts": [
     {
       "role": "edge",
@@ -34,7 +65,8 @@
       "role": "worker",
       "cpus": 2,
       "memory": 3072,
-      "quantity": 3
+      "quantity": 3,
+      "profiles": ["worker"]
     }
   ]
 }
diff --git a/incus.yml b/incus.yml
new file mode 100644
index 0000000..de4d823
--- /dev/null
+++ b/incus.yml
@@ -0,0 +1,58 @@
+---
+project: tdp
+image: rockylinux/8/cloud
+admin_user: incus
+network:
+  manage: true
+  name: tdp
+  domain: tdp
+  cidr: 192.168.56.1/24
+storage:
+  manage: true
+  name: tdp
+  dir: "./pool"
+  root_size: 40GB
+profiles:
+  master:
+    devices:
+      data:
+        path: "/mnt"
+        size: 100GB
+  worker:
+    devices:
+      data:
+        path: "/mnt"
+        size: 100GB
+output_dir: "../tdp-getting-started"
+hosts:
+  - role: edge
+    cpus: 2
+    memory: 4096
+    quantity: 1
+  - role: master
+    cpus: 3
+    groups:
+      - master
+      - master1
+    hostname: master-01
+    memory: 6144
+  - role: master
+    cpus: 3
+    groups:
+      - master
+      - master2
+    hostname: master-02
+    memory: 6144
+  - role: master
+    cpus: 3
+    groups:
+      - master
+      - master3
+    hostname: master-03
+    memory: 6144
+  - role: worker
+    cpus: 2
+    memory: 3072
+    quantity: 3
+    profiles:
+      - worker
diff --git a/launch.sh b/launch.sh
index 2b6e128..7150738 100755
--- a/launch.sh
+++ b/launch.sh
@@ -1,30 +1,58 @@
 #!/usr/bin/env bash
 
+set -x
+
 cd "$(dirname "$0")"
 
-# Get customized variables
-source .env
+source ./source.sh
 
-#variables extracted from incus config file
-project=$(jq ".project" $file | sed 's/"//g')
-image=$(jq '.box' $file | sed 's/"//g')
-hostnum=$(jq '.hosts | length' $file | sed 's/"//g')
-domain=$(jq '.domain' $file | sed 's/"//g')
+if [ ! -d "${output_dir}/inventory" ]; then
+    error "${output_dir} is not a valid tdp project directory. Please initialize a TDP project at that path, or override 'output_dir'"
+fi
 
 declare -A groups;
 
 # check project exists
-if ! incus project list | grep $project ; then
+if ! incus project show $project  &>/dev/null; then
     incus project create $project
 fi
 
+
+echo $project
+
 incus project switch $project
 
-# check lxdbr0 network exists and is a bridge
-if ! incus network list | grep $network | grep bridge ; then
-    incus network create $network "ipv4.address=${iprange}" dns.domain=$domain ipv6.nat=false ipv6.address=none ipv4.nat=true
+yes | ssh-keygen -t rsa -b 4096 -f ${private_key} -N ''
+echo '#Generated by tdp-incus/launch.sh' > ${ansible_hosts_file}.tmp
+echo "" >> ${ansible_hosts_file}.tmp
+
+# check network exists and is a bridge
+if [ "${network_manage}" == "true" ]; then
+    domain=$(parseconf '.network.domain // "tdp"')
+    cidr=$(parseconf '.network.cidr // "10.10.10.1/24"')
+    if ! incus network show $network 2>&1 >/dev/null; then
+        incus network create $network "ipv4.address=${cidr}" dns.domain=$domain ipv6.nat=false ipv6.address=none ipv4.nat=true
+    fi
+else
+    if ! incus network show $network 2>&1 >/dev/null; then
+        error "Network ${network} does not exists and set to manage=false. Create it manually or set manage to true"
+    else
+        domain=$(incus network show ${network} | ${_yq} '.config["dns.domain"]')
+        cidr=$(incus network show ${network} | ${_yq} '.config["ipv4.address"]')
+    fi
 fi
 
+# unless tdp storage exists
+if [ "${storage_manage}" == "true" ]; then
+    if ! incus storage show $storage 2>&1 >/dev/null; then
+        #create local dir storage
+        storage_dir=`realpath $(parseconf '.storage.dir // "./data/pool"')`
+        mkdir -p $storage_dir
+    	incus storage create ${storage} dir source=${storage_dir}
+    fi
+fi
+
+# Default profiles (default)
 incus profile edit default <<-EOF
 config: {}
 description: Default Incus profile
@@ -39,13 +67,13 @@ devices:
     type: nic
   root:
     path: /
-    pool: tdp
-    size: 50GB
+    pool: ${storage}
+    size: ${root_size}
     type: disk
 name: default
 EOF
 
-if ! incus profile list | grep cloud ; then
+if ! incus profile show cloud  2>&1 >/dev/null; then
     incus profile create cloud
 fi
 
@@ -59,31 +87,41 @@ devices:
 name: cloud
 EOF
 
-mkdir -p $datadir
-yes | ssh-keygen -t rsa -b 4096 -f $privatekey -N ''
-echo '#Generated by tdp-incus/launch.sh' > ${hostfile}.tmp
-echo "" >> ${hostfile}.tmp
-
-# unless tdp storage exists
-if ! incus storage list | grep $storagepool; then
-    #create local dir storage
-	mkdir -p $storagedir
-	incus storage create $storagepool dir source=$storagedir
-fi
-
-#foreach hostnum
-for f in $(seq 0 $((hostnum - 1))); do
+# #foreach hosts_num
+# for profile in $(parseconf '.profiles | keys[]'); do
+#     echo toto $key;
+#     if ! incus profile show $profile; then
+#         incus profile create $profile
+#     fi
+#     filter="  .profiles[].devices |=
+#         map_values(
+#           if .type == null then
+#             .type = \"disk\" |
+#             (if .pool == null then .pool = \"${storage}\" else . end)
+#           elif .type == \"disk\" and .pool == null then
+#             .pool = \"${storage}\"
+#           else
+#             .
+#           end
+#         )"
+#     echo -e $filter
+#     ${_yq} -y "${filter} | .profiles.${profile}" incus.yml > tutu.yml
+#     # | incus profile edit $profile
+#     exit 1
+# done
+
+#foreach hosts_num
+for f in $(seq 0 $((hosts_num - 1))); do
     # parse json file to get host config
-    hostname=$(jq ".hosts[$f].hostname" $file | sed 's/"//g')
-    memory=$(jq ".hosts[$f].memory" $file)
-    cpu=$(jq ".hosts[$f].cpus" $file)
-    quantity=$(jq ".hosts[$f].quantity" $file | sed 's/"//g')
-    role=$(jq ".hosts[$f].role" $file | sed 's/"//g')
-    gr=$(jq ".hosts[$f].groups[]" $file 2>/dev/null | sed 's/"//g')
+    hostname=$(parseconf ".hosts[$f].hostname")
+    memory=$(parseconf ".hosts[$f].memory")
+    cpu=$(parseconf ".hosts[$f].cpus")
+    quantity=$(parseconf ".hosts[$f].quantity")
+    role=$(parseconf ".hosts[$f].role")
+    gr=$(parseconf ".hosts[$f].groups[]")
     echo hostname $hostname mem $memory cpu $cpu quantity $quantity role $role gr $(IFS=','; echo "${gr[*]}"; unset IFS)
     if [ -z "$memory" ] || [ -z "$cpu" ] || [ -z "$role" ]; then
-        echo 'memory, cpu, and role are mandatory vars'
-	exit 1;
+        error 'memory, cpu, and role are mandatory vars'
     fi
     if [[ "$quantity" == "null" ]]; then
        quantity=1
@@ -94,8 +132,7 @@ for f in $(seq 0 $((hostnum - 1))); do
     fi
     if [ "$quantity" -gt 1 ]; then
         if [ "$hostname" != "null" ]; then
-            echo "Cannot have both quantity > 1 ($quantity) and hostname specified ($hostname)"
-            exit 1;
+            error "Cannot have both quantity > 1 ($quantity) and hostname specified ($hostname)"
         fi
     fi
     for i in $(seq 1 $quantity); do
@@ -110,7 +147,7 @@ for f in $(seq 0 $((hostnum - 1))); do
             groups[$g]="${groups[$g]}\n$name"
         done;
         # add host to inventory
-        echo "$name ansible_ssh_host=$name.$domain ansible_ssh_port=22 ansible_ssh_user='$user' ansible_ssh_private_key_file='$privatekey' domain=$domain" >> ${hostfile}.tmp
+        echo "$name ansible_ssh_host=$name.$domain ansible_ssh_port=22 ansible_ssh_user='$admin_user' ansible_ssh_private_key_file='$privatekey' domain=$domain" >> ${ansible_hosts_file}.tmp
         # launch host with following configuration
         incus launch images:$image $name --vm <<-EOF
 config:
@@ -147,10 +184,10 @@ config:
     runcmd:
       - [ systemctl, enable, sshd, --now ]
     users:
-      - name: ${user}
+      - name: ${admin_user}
         sudo: ALL=(ALL) NOPASSWD:ALL
         ssh-authorized-keys:
-          - $(cat $privatekey.pub)
+          - $(cat ${private_key}.pub)
         shell: /bin/bash
 name: $name
 architecture: x86_64
@@ -161,11 +198,13 @@ EOF
     done;
 done;
 
-echo >> ${hostfile}.tmp
+echo >> ${ansible_hosts_file}.tmp
 for key in "${!groups[@]}"; do
     # generate ini section of current group (key)
-    echo -e "[$key]${groups[$key]}\n" >> ${hostfile}.tmp
+    echo -e "[$key]${groups[$key]}\n" >> ${ansible_hosts_file}.tmp
 done
 
 #when finished, replace host file by tmp version
-mv ${hostfile}.tmp ${hostfile}
+echo 'Write finalized ansible hosts.ini file'
+mv ${ansible_hosts_file}.tmp ${host_file}
+echo 'Finished successfully!'
diff --git a/source.sh b/source.sh
new file mode 100755
index 0000000..164276b
--- /dev/null
+++ b/source.sh
@@ -0,0 +1,78 @@
+#!/bin/sh
+
+error(){
+    echo "$@" >&2;
+    exit 1;
+}
+
+requires(){
+    error "This script requires $1 to run, please install"
+}
+
+if ! command -v incus  &>/dev/null; then
+    requires 'incus'
+fi
+if command -v yq &>/dev/null; then
+    echo 'yq is available'
+    _yq=yq
+elif [ -x './yq' ]; then
+    _yq='./yq'
+else
+    read -p "yq not installed. Do you want to install yq? Press Enter to continue (or type 'n' to skip): " answer
+    if [[ "${answer}" != "n" ]]; then
+        os_type=$(uname -s)
+        if [[ "$os_type" == "Darwin" ]]; then
+            echo "MacOS detected. Install with brew"
+            brew install yq
+        elif [[ "$os_type" == "Linux" ]]; then
+            # Check for Arch Linux
+            if [[ -f /etc/arch-release ]]; then
+                sudo pacman -S yq
+            else
+                echo "This is a (non-Arch) Linux distribution."
+                read -p "Download script and install in /usr/local/bin? (Requires sudo) Press n for local install" global
+                version=`curl https://api.github.com/repos/mikefarah/yq/releases/latest | grep '"tag_name":' | cut -d':' -f2 | sed 's/^.*\"\(.*\)\".*$/\1/g'`
+                if [[ "${global}" != "n" ]]; then
+                    sudo wget https://github.com/mikefarah/yq/releases/download/${version}/yq_linux_amd64 -O /usr/local/bin/yq && \
+                    sudo chmod +x /usr/local/bin/yq
+                else
+                    sudo wget https://github.com/mikefarah/yq/releases/download/${version}/yq_linux_amd64 -O ./yq && \
+                    sudo chmod +x ./yq
+                    _yq='./yq'
+                fi
+            fi
+        else
+            echo "yq not available and OS cannot be detected. Exiting"
+            exit 1;
+        fi
+    else
+        "Do not install. Quitting..."
+    fi
+fi
+
+parseconf(){
+    ${_yq} "${1}" incus.yml | sed 's/"\(.*\)"/\1/g'
+}
+
+#variables extracted from incus config file
+project=$(parseconf '.project  // "tdp"')
+image=$(parseconf '.image // "rockylinux/8/cloud"')
+hosts_num=$(parseconf '.hosts | length')
+admin_user=$(parseconf '.admin_user  // "tdp"')
+
+# Network
+network=$(parseconf '.network.name // "tdp"')
+network_manage=$(parseconf '.network.manage // true')
+
+# storage
+storage=$(parseconf '.storage.name // "tdp"')
+storage_manage=$(parseconf '.storage.manage // true')
+root_size=$(parseconf '.storage.root_size // "40GB"')
+
+# profiles
+profiles_num=$(parseconf '.profiles | length')
+
+# output dir
+output_dir=`realpath $(parseconf '.output_dir // "../tdp-getting-started"')`
+ansible_hosts_file="${output_dir}/inventory/hosts.ini"
+private_key="$output_dir/incus_key";