From a2625656503c7b23c9f94f5e6d55ea4269c440cf Mon Sep 17 00:00:00 2001
From: Ben Potter <me@bpmct.net>
Date: Fri, 24 Apr 2026 13:50:34 +0000
Subject: [PATCH] refactor(incus-vm): split NixOS provisioning into nixos.tf

Move local.is_nixos and null_resource.provision_nixos out of main.tf
into a dedicated nixos.tf to keep main.tf focused on core infrastructure.
---
 registry/bpmct/templates/incus-vm/main.tf  | 113 +-------------------
 registry/bpmct/templates/incus-vm/nixos.tf | 117 +++++++++++++++++++++
 2 files changed, 118 insertions(+), 112 deletions(-)
 create mode 100644 registry/bpmct/templates/incus-vm/nixos.tf

diff --git a/registry/bpmct/templates/incus-vm/main.tf b/registry/bpmct/templates/incus-vm/main.tf
index 19982ade..881d4a20 100644
--- a/registry/bpmct/templates/incus-vm/main.tf
+++ b/registry/bpmct/templates/incus-vm/main.tf
@@ -208,9 +208,7 @@ resource "incus_image" "image" {
   remote = local.incus_remote
   source_image = {
     remote = "images"
-    # NixOS images on images.linuxcontainers.org use just "nixos/25.11" (no arch suffix in alias).
-    # Other distros like ubuntu append the arch: "ubuntu/jammy/cloud/amd64".
-    name         = local.is_nixos ? data.coder_parameter.image.value : "${data.coder_parameter.image.value}/${data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"}"
+    name   = local.is_nixos ? data.coder_parameter.image.value : "${data.coder_parameter.image.value}/${data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"}"
     type         = "virtual-machine"
     architecture = data.coder_parameter.host.value == "ThinkStation" ? "x86_64" : "aarch64"
   }
@@ -357,112 +355,6 @@ resource "null_resource" "token_refresh" {
   }
 }
 
-# Provisioner for NixOS VMs.
-# NixOS does not support cloud-init in the traditional sense.
-# We use incus file push + nixos-rebuild to declare the user and coder-agent service.
-resource "null_resource" "provision_nixos" {
-  count = data.coder_workspace.me.start_count == 1 && local.is_nixos ? 1 : 0
-
-  triggers = {
-    agent_token = local.agent_token
-    instance    = incus_instance.dev.name
-  }
-
-  depends_on = [incus_instance.dev]
-
-  provisioner "local-exec" {
-    # Write the nix module and coder agent files into the VM, then run nixos-rebuild.
-    # We use incus file push for files containing sensitive values or complex content,
-    # and incus exec for commands. This avoids shell quoting issues with heredocs.
-    command = <<-EOT
-      set -e
-      REMOTE="${local.incus_remote}"
-      INSTANCE="${incus_instance.dev.name}"
-      WUSER="${local.workspace_user}"
-      ARCH="${data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"}"
-
-      echo "Waiting for NixOS VM incus-agent to be ready..."
-      for i in $(seq 1 60); do
-        if incus exec "$REMOTE:$INSTANCE" -- true 2>/dev/null; then
-          echo "incus-agent ready after $i attempts"
-          break
-        fi
-        echo "Attempt $i: incus-agent not ready yet, waiting..."
-        sleep 5
-      done
-
-      # Write init script into the VM
-      incus exec "$REMOTE:$INSTANCE" -- mkdir -p /opt/coder
-      echo "${base64encode(local.agent_init_script)}" | base64 -d | incus file push - "$REMOTE:$INSTANCE/opt/coder/init"
-      incus exec "$REMOTE:$INSTANCE" -- chmod 755 /opt/coder/init
-
-      # Write env file into the VM
-      printf 'CODER_AGENT_TOKEN=${local.agent_token}\nCODER_AGENT_URL=${data.coder_workspace.me.access_url}\n' \
-        | incus file push - "$REMOTE:$INSTANCE/opt/coder/init.env" --mode 0600
-
-      # Write the NixOS coder module, substituting the username
-      NIXMOD=$(cat <<NIXMOD_EOF
-{ config, pkgs, lib, ... }:
-{
-  users.users."$WUSER" = {
-    isNormalUser = true;
-    uid = 1000;
-    home = "/home/$WUSER";
-    shell = pkgs.bash;
-    extraGroups = [ "wheel" ];
-  };
-
-  security.sudo.wheelNeedsPassword = false;
-
-  systemd.services.coder-agent = {
-    description = "Coder Agent";
-    after = [ "network-online.target" ];
-    wants = [ "network-online.target" ];
-    wantedBy = [ "multi-user.target" ];
-    serviceConfig = {
-      User = "$WUSER";
-      EnvironmentFile = "/opt/coder/init.env";
-      ExecStart = "/opt/coder/init";
-      Environment = "PATH=/run/current-system/sw/bin:/run/wrappers/bin:/usr/local/bin:/usr/bin:/bin";
-      Restart = "always";
-      RestartSec = 10;
-      TimeoutStopSec = 90;
-      KillMode = "process";
-      OOMScoreAdjust = -900;
-      SyslogIdentifier = "coder-agent";
-    };
-  };
-}
-NIXMOD_EOF
-)
-      echo "$NIXMOD" | incus file push - "$REMOTE:$INSTANCE/etc/nixos/coder.nix"
-
-      # Patch configuration.nix to import coder.nix if not already imported
-      incus exec "$REMOTE:$INSTANCE" -- \
-        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
-        "grep -q coder.nix /etc/nixos/configuration.nix || \
-         sed -i 's|imports = \[|imports = [\n    ./coder.nix|' /etc/nixos/configuration.nix"
-
-      echo "Running nixos-rebuild switch (this may take a few minutes)..."
-      incus exec "$REMOTE:$INSTANCE" -- \
-        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -l -c \
-        "nixos-rebuild switch; EC=\$?; [ \$EC -eq 0 ] || [ \$EC -eq 4 ] || exit \$EC"
-
-      echo "Restarting coder-agent service..."
-      incus exec "$REMOTE:$INSTANCE" -- \
-        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
-        "systemctl daemon-reload; systemctl restart coder-agent.service; sleep 3; systemctl status coder-agent.service || true"
-
-      # Ensure home dir ownership (nixos-rebuild will have created the user home)
-      incus exec "$REMOTE:$INSTANCE" -- \
-        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
-        "mkdir -p /home/$WUSER && chown 1000:1000 /home/$WUSER && chmod 755 /home/$WUSER"
-
-      echo "NixOS provisioning complete."
-    EOT
-  }
-}
-
 resource "incus_instance_snapshot" "on_stop" {
   count    = data.coder_parameter.snapshot_on_stop.value == "true" ? 1 : 0
   remote   = local.incus_remote
@@ -483,9 +375,6 @@ locals {
   agent_token       = data.coder_workspace.me.start_count == 1 ? coder_agent.main[0].token : ""
   agent_init_script = data.coder_workspace.me.start_count == 1 ? coder_agent.main[0].init_script : ""
 
-  # Detect NixOS image
-  is_nixos = startswith(data.coder_parameter.image.value, "nixos/")
-
   # USB device map — add more entries here as needed
   usb_devices = {
     kindle      = { name = "kindle", vendorid = "1949", productid = "0004" }
diff --git a/registry/bpmct/templates/incus-vm/nixos.tf b/registry/bpmct/templates/incus-vm/nixos.tf
new file mode 100644
index 00000000..77fd0cf7
--- /dev/null
+++ b/registry/bpmct/templates/incus-vm/nixos.tf
@@ -0,0 +1,117 @@
+# NixOS-specific provisioning for incus-vm workspaces.
+#
+# NixOS doesn't support cloud-init, so instead we:
+#   1. Push the coder agent init script and env file via incus file push
+#   2. Generate /etc/nixos/coder.nix declaring the user + coder-agent service
+#   3. Patch configuration.nix to import coder.nix
+#   4. Run nixos-rebuild switch
+#   5. Restart coder-agent.service with the fresh token
+#
+# This provisioner runs on every workspace start (null_resource is recreated
+# each cycle), which also handles token rotation.
+
+locals {
+  # NixOS images on images.linuxcontainers.org use just "nixos/25.11" with no
+  # arch suffix in the alias — unlike Ubuntu which appends e.g. "/amd64".
+  is_nixos = startswith(data.coder_parameter.image.value, "nixos/")
+}
+
+resource "null_resource" "provision_nixos" {
+  count = data.coder_workspace.me.start_count == 1 && local.is_nixos ? 1 : 0
+
+  triggers = {
+    agent_token = local.agent_token
+    instance    = incus_instance.dev.name
+  }
+
+  depends_on = [incus_instance.dev]
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      set -e
+      REMOTE="${local.incus_remote}"
+      INSTANCE="${incus_instance.dev.name}"
+      WUSER="${local.workspace_user}"
+      ARCH="${data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"}"
+
+      echo "Waiting for NixOS VM incus-agent to be ready..."
+      for i in $(seq 1 60); do
+        if incus exec "$REMOTE:$INSTANCE" -- true 2>/dev/null; then
+          echo "incus-agent ready after $i attempts"
+          break
+        fi
+        echo "Attempt $i: incus-agent not ready yet, waiting..."
+        sleep 5
+      done
+
+      # Write init script into the VM
+      incus exec "$REMOTE:$INSTANCE" -- mkdir -p /opt/coder
+      echo "${base64encode(local.agent_init_script)}" | base64 -d | incus file push - "$REMOTE:$INSTANCE/opt/coder/init"
+      incus exec "$REMOTE:$INSTANCE" -- chmod 755 /opt/coder/init
+
+      # Write env file into the VM
+      printf 'CODER_AGENT_TOKEN=${local.agent_token}\nCODER_AGENT_URL=${data.coder_workspace.me.access_url}\n' \
+        | incus file push - "$REMOTE:$INSTANCE/opt/coder/init.env" --mode 0600
+
+      # Write the NixOS coder module, substituting the username
+      NIXMOD=$(cat <<NIXMOD_EOF
+{ config, pkgs, lib, ... }:
+{
+  users.users."$WUSER" = {
+    isNormalUser = true;
+    uid = 1000;
+    home = "/home/$WUSER";
+    shell = pkgs.bash;
+    extraGroups = [ "wheel" ];
+  };
+
+  security.sudo.wheelNeedsPassword = false;
+
+  systemd.services.coder-agent = {
+    description = "Coder Agent";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      User = "$WUSER";
+      EnvironmentFile = "/opt/coder/init.env";
+      ExecStart = "/opt/coder/init";
+      Environment = "PATH=/run/current-system/sw/bin:/run/wrappers/bin:/usr/local/bin:/usr/bin:/bin";
+      Restart = "always";
+      RestartSec = 10;
+      TimeoutStopSec = 90;
+      KillMode = "process";
+      OOMScoreAdjust = -900;
+      SyslogIdentifier = "coder-agent";
+    };
+  };
+}
+NIXMOD_EOF
+)
+      echo "$NIXMOD" | incus file push - "$REMOTE:$INSTANCE/etc/nixos/coder.nix"
+
+      # Patch configuration.nix to import coder.nix if not already imported
+      incus exec "$REMOTE:$INSTANCE" -- \
+        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
+        "grep -q coder.nix /etc/nixos/configuration.nix || \
+         sed -i 's|imports = \[|imports = [\n    ./coder.nix|' /etc/nixos/configuration.nix"
+
+      echo "Running nixos-rebuild switch (this may take a few minutes)..."
+      incus exec "$REMOTE:$INSTANCE" -- \
+        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -l -c \
+        "nixos-rebuild switch; EC=\$?; [ \$EC -eq 0 ] || [ \$EC -eq 4 ] || exit \$EC"
+
+      echo "Restarting coder-agent service..."
+      incus exec "$REMOTE:$INSTANCE" -- \
+        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
+        "systemctl daemon-reload; systemctl restart coder-agent.service; sleep 3; systemctl status coder-agent.service || true"
+
+      # Ensure home dir ownership (nixos-rebuild will have created the user home)
+      incus exec "$REMOTE:$INSTANCE" -- \
+        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
+        "mkdir -p /home/$WUSER && chown 1000:1000 /home/$WUSER && chmod 755 /home/$WUSER"
+
+      echo "NixOS provisioning complete."
+    EOT
+  }
+}