(docs) Clean up and reorganise build:

- docs build is now pure and doesn't require breaking the sandbox, yay!
- passthru attributes are now all in their own files
- add docs.figures which builds svgs from draw.io diagrams
- Added graphviz dependency for diagram generation
- Added docs.fetch-inventories script for updating Intersphinx inventory files
- Intersphinx inventories use ROS distro specified in flake.nix if provided

.gitignore

@@ -12,8 +12,6 @@ build/
 public/
 # draw.io stuff
 .$*
-# generated figures
-figures-output/
 
 # Prerequisites
 *.d

docs/default.nix

@@ -10,12 +10,9 @@
   uv2nix,
   # non-python packages to build docs
   doxygen,
-  # shell env
-  mkShell,
-  uv,
-  # compressed docs
-  p7zip,
-  writeShellScriptBin,
+  graphviz,
+  # ros version to link to with intersphinx - defaults to humble
+  rosDistro ? "humble",
 }:
 let
   # mostly taken from uv2nix docs https://adisbladis.github.io/uv2nix/usage/hello-world.html
@@ -31,18 +28,18 @@ let
     (callPackage pyproject-nix.build.packages { python = python312; }).overrideScope
       overlay;
   pyEnv = pythonPkgSet.mkVirtualEnv "roar-docs-py-env" workspace.deps.default;
-  # add Doxygen to the final environment so we can generate code docs
+  # add generation tools to the final environment so we can generate code docs
   env = symlinkJoin {
     name = "roar-docs-env";
     paths = [
       pyEnv
       doxygen
+      graphviz
     ];
   };
 
   # create derivation which builds the docs
   docs = stdenv.mkDerivation {
-    inherit SOURCE_DATE_EPOCH;
     name = "roar-docs";
     buildInputs = [ env ];
     src = lib.cleanSourceWith {
@@ -61,20 +58,40 @@ let
         );
     };
 
-    # with sandbox=relaxed, disables sandboxing for this derivation
-    __noChroot = true;
+    passthru = {
+      inherit
+        env
+        shell
+        compressed
+        decompress
+        figures
+        fetch-inventories
+        setup
+        ;
+    };
+
+    ROS_DISTRO = rosDistro; # for intersphinx config
+
     # make needs to be run from the docs directory, but we need the whole tree for Doxygen generation
     # (hence source being the whole project)
-    # note that the _sources directory is removed as we don't want to accidentally leak things
+    # note that (although they should not be present anyway) the program_listing files are removed to
+    # doubly ensure that there are no source code leaks
     buildPhase = ''
       cd docs
+
+      # copy prebuilt figures into place
+      mkdir -p source/generated
+      cp -a ${figures}/. source/generated
+
       # needed to prevent /homeless-shelter from being created
       export HOME=$PWD
 
       # this needed because Sphinx does a dumb thing and overrides the "current year" in copyright with SOURCE_DATE_EPOCH
       # see https://github.com/sphinx-doc/sphinx/issues/3451#issuecomment-877801728
       unset SOURCE_DATE_EPOCH
-      # build the docs
+      # make the directory writable - allows us to modify the files during the Sphinx build
+      chmod -R +w .
+      # finally build the docs
       make html
 
       # failsafe to *ensure* that program listings are removed - we REALLY don't want to leak source code
@@ -85,36 +102,16 @@ let
       mkdir -p $out/html
       cp -a ./build/html/. $out/html
     '';
-    # provide dev shell in passthru
-    passthru = {
-      inherit env compressed decompress;
-      shell = mkShell {
-        buildInputs = [
-          env
-          uv # uv added to manage python dependencies
-        ];
-        shellHook = ''
-          # Undo dependency propagation by nixpkgs.
-          unset PYTHONPATH
-        '';
-      };
-    };
   };
 
-  compressed = stdenv.mkDerivation {
-    name = "roar-docs-compressed";
-    buildInputs = [
-      docs
-      p7zip
-    ];
-    src = docs;
-    installPhase = ''
-      mkdir -p $out
-      7z a -t7z -m0=lzma -mx=9 -mfb=64 -md=32m -ms=on $out/docs.7z .
-    '';
+  shell = callPackage (import ./nix/shell.nix) { inherit rosDistro env; };
+  compressed = callPackage (import ./nix/compressed.nix) { inherit docs; };
+  decompress = callPackage (import ./nix/decompress.nix) { };
+  cd-docs-source = callPackage (import ./nix/cd-docs-source.nix) { };
+  figures = callPackage (import ./nix/figures.nix) { };
+  fetch-inventories = callPackage (import ./nix/fetch-inventories.nix) {
+    inherit rosDistro cd-docs-source;
   };
-  decompress = writeShellScriptBin "decompress" ''
-    ${lib.getExe p7zip} x $1 -o$2
-  '';
+  setup = callPackage (import ./nix/setup.nix) { inherit cd-docs-source figures; };
 in
 docs

docs/nix/cd-docs-source.nix

@@ -0,0 +1,10 @@
+{ writeShellScript }:
+writeShellScript "roar-docs-inventory" ''
+  # cd to docs/source assuming they exist (we're not already there)
+  if [ -d docs ]; then
+    cd docs
+  fi
+  if [ -d source ]; then
+    cd source
+  fi
+''

docs/nix/compressed.nix

@@ -0,0 +1,17 @@
+{
+  stdenv,
+  p7zip,
+  docs,
+}:
+stdenv.mkDerivation {
+  name = "roar-docs-compressed";
+  buildInputs = [
+    docs
+    p7zip
+  ];
+  src = docs;
+  installPhase = ''
+    mkdir -p $out
+    7z a -t7z -m0=lzma -mx=9 -mfb=64 -md=32m -ms=on $out/docs.7z .
+  '';
+}

docs/nix/decompress.nix

@@ -0,0 +1,8 @@
+{
+  lib,
+  p7zip,
+  writeShellScriptBin,
+}:
+writeShellScriptBin "decompress" ''
+  ${lib.getExe p7zip} x $1 -o$2
+''

docs/nix/fetch-inventories.nix

@@ -0,0 +1,49 @@
+# Fetches intersphinx inventory files
+{
+  rosDistro,
+  writeShellScriptBin,
+  cd-docs-source,
+}:
+writeShellScriptBin "roar-docs-fetch-inventories" ''
+  set -e
+
+  INVENTORY_LOCATION="intersphinx"
+
+  source ${cd-docs-source}
+
+  # get intersphinx inventories
+  # clear existing
+  rm -rf "$INVENTORY_LOCATION"
+
+  # get the files
+  echo "Fetching intersphinx inventories"
+  curl --parallel \
+    --create-dirs --output-dir "$INVENTORY_LOCATION" \
+    --output python.inv https://docs.python.org/3/objects.inv \
+    --variable rosDistro="${rosDistro}" \
+    --output ros.inv --expand-url https://docs.ros.org/en/{{rosDistro:trim:url}}/objects.inv
+
+  if [[ "$*" == *--no-commit* ]]; then
+      echo "Will not commit changes"
+      exit 0
+  fi
+
+  if ! git diff --cached --quiet >/dev/null; then
+      HAS_GIT_STAGING=1
+      echo "Stashing staged changes"
+      git stash push -S >/dev/null
+  fi
+
+  cleanup() {
+    if [[ -v HAS_GIT_STAGING ]]; then
+        echo "Restoring git stash"
+        git stash pop >/dev/null
+    fi
+  }
+  trap cleanup EXIT
+
+  echo "Staging changes"
+  git add "$INVENTORY_LOCATION"
+  echo "Committing changes"
+  git commit -m "$(date +%Y-%m-%dT%H:%M:%S) Intersphinx inventory fetch" >/dev/null
+''

docs/nix/figures.nix

@@ -0,0 +1,68 @@
+{
+  stdenv,
+  lib,
+  writeShellScript,
+  drawio,
+  parallel,
+  xvfb-run,
+}:
+# Partly a ripoff of the drawio-headless package at:
+# https://github.com/NixOS/nixpkgs/blob/nixpkgs-unstable/pkgs/applications/graphics/drawio/headless.nix
+# Except highly specialised for our use case, and with no --auto-display because that breaks everything somehow.
+# With that flag, xvfb-run doesn't work - it doesn't actually run any commands.
+# Perhaps because it's in a Wayland environment not X11, and thus there's no pre-existing X server to talk to?
+# -a works, but supposedly it's deprecated. Very strange. Running it with no flags works too, so that's what we do.
+let
+  output-dir = "generated";
+
+  figures = stdenv.mkDerivation {
+    name = "roar-docs-figures";
+    src = lib.cleanSource ../figures-source;
+    # after making the temp output dir, this finds all .drawio files in the source directory,
+    # and parallel processes them using parallel with draw-io-convert, all run through xvfb-run so electron is happy
+    buildPhase = ''
+      mkdir ${output-dir}
+      find ${../figures-source} -type f -name "*.drawio" -print0 | ${lib.getExe xvfb-run} ${lib.getExe parallel} -0 -j $(nproc) ${draw-io-convert}
+    '';
+    installPhase = ''
+      mkdir -p $out
+      cp -a ${output-dir}/. $out
+    '';
+  };
+
+  # script to convert .drawio files to .svg in both light and dark themes
+  # note that there's a whole bunch of cache/dbus errors: since they don't stop draw.io from running,
+  # we can just ignore them!
+  draw-io-convert = writeShellScript "draw-io-convert" ''
+    set -e
+
+    BASENAME=$(basename "$1")
+    OUTPUT_BASE="${output-dir}"/"$BASENAME"
+
+    convert() {
+      INPUT_FILE="$1"
+      OUTPUT_FILE="$2"
+      shift 2
+      ${lib.getExe drawio} -x "$INPUT_FILE" -o "$OUTPUT_FILE" --disable-gpu "$@" 
+    }
+
+    # Electron really wants a configuration directory to not die with:
+    # "Error: Failed to get 'appData' path"
+    # so we give it some temp dir as XDG_CONFIG_HOME
+    tmpdir=$(mktemp -d)
+
+    function cleanup {
+      rm -rf "$tmpdir"
+    }
+    trap cleanup EXIT
+
+    # Drawio needs to run in a virtual X session, because Electron
+    # refuses to work and dies with an unhelpful error message otherwise:
+    # "The futex facility returned an unexpected error code."
+    export XDG_CONFIG_HOME="$tmpdir"
+
+    convert "$1" "$OUTPUT_BASE"".svg" --svg-theme light
+    convert "$1" "$OUTPUT_BASE""-dark.svg" --svg-theme dark
+  '';
+in
+figures

docs/nix/setup.nix

@@ -0,0 +1,17 @@
+{
+  writeShellScriptBin,
+  cd-docs-source,
+  figures,
+}:
+let
+  output-dir = "generated";
+in
+writeShellScriptBin "roar-docs-setup" ''
+  set -e
+
+  source ${cd-docs-source}
+  mkdir -p "${output-dir}"
+  cp -a ${figures}/. "${output-dir}"
+  # nix store is read-only by default, so fix that after copying
+  chmod -R +w "${output-dir}"
+''

docs/nix/shell.nix

@@ -0,0 +1,22 @@
+{
+  rosDistro,
+  mkShell,
+  env,
+  uv,
+  drawio,
+}:
+let
+  shell = mkShell {
+    ROS_DISTRO = rosDistro;
+    buildInputs = [
+      env
+      uv # uv added to manage python dependencies
+      drawio
+    ];
+    shellHook = ''
+      # Undo dependency propagation by nixpkgs.
+      unset PYTHONPATH
+    '';
+  };
+in
+shell

flake.nix

@@ -71,6 +71,7 @@
             # add uv2nix + pyproject-nix to the package set
             (final: prev: { inherit pyproject-nix uv2nix; })
           ];
+          config.allowUnfree = true; # needed for draw.io for the docs
         };
 
         # --- INPUT PACKAGE SETS ---
@@ -142,7 +143,8 @@
 
         # --- PYTHON (UV) WORKSPACES ---
         # note: called with pkgs-unstable since we need the uv tool to be up-to-date due to rapid development
-        docs = pkgs-unstable.callPackage (import ./docs) { };
+        # note 2: called with rosDistro to link correct intersphinx inventory
+        docs = pkgs-unstable.callPackage (import ./docs) { inherit rosDistro; };
 
         # --- LAUNCH SCRIPTS ---
         perseus = pkgs.writeShellScriptBin "perseus" ''
@@ -194,11 +196,17 @@
       "qutrc-roar.cachix.org-1:lARPhJL+PLuGd021HeN8CQOGGiYVEVGws5za+39M1Z0="
       "ros.cachix.org-1:dSyZxI8geDCJrwgvCOHDoAfOm5sV1wCPjBkKL+38Rvo="
     ];
-    # note that this is normally a VERY BAD IDEA but it's needed so the docs can have internet access
-    # This is needed for:
-    # - sphinx-immaterial to build (it pulls fonts and CSS from Google CDN)
-    # - intersphinx to pull inventory files so it can link to other Sphinx projects
-    #   (like Python or ROS2 docs)
-    sandbox = "relaxed";
+    # note that this is normally a VERY BAD IDEA but it may be needed so the docs can have internet access,
+    # with certain configurations. Currently, everything is configured to work offline with cached files in the git repo.
+
+    # Unless otherwise configured, sphinx-immaterial will pull fonts from Google's CDN, which obviously requires internet access.
+    # The Roboto (and RobotoMono) fonts have been downloaded locally and are used instead.
+    # These should never change, so it really doesn't matter.
+
+    # intersphinx also normally expects to be able to download inventory (.inv) files from the target projects,
+    # but it has also been configured to use local copies. Updating these files is done with `nix run .#docs.fetch-inventories`,
+    # and is run automatically from the CI pipeline (it makes a commit with the update).
+
+    # sandbox = "relaxed";
   };
 }