From c86793f43c0d44c0992dd046e2c974c8b130c202 Mon Sep 17 00:00:00 2001
From: Ben Potter <me@bpmct.net>
Date: Thu, 23 Apr 2026 23:50:59 +0000
Subject: [PATCH] feat(incus-vm): add NixOS VM support

- Add nixos/25.11 and nixos/unstable image options
- Skip cloud-init config for NixOS images (not supported)
- Add null_resource.provision_nixos that:
  - Waits for incus-agent readiness
  - Pushes coder agent init script and env file via incus file push
  - Generates /etc/nixos/coder.nix declaring the workspace user,
    sudoers, and coder-agent systemd service
  - Patches /etc/nixos/configuration.nix to import coder.nix
  - Runs nixos-rebuild switch (tolerates exit 4 = service warning)
  - Explicitly restarts coder-agent.service post-rebuild
- Service unit sets Environment PATH so curl/tools are findable
- Disable token_refresh resource for NixOS (not needed)
- NixOS image alias does not append arch suffix (unlike Ubuntu)
---
 registry/bpmct/templates/incus-vm/main.tf | 523 ++++++++++++++++++++++
 1 file changed, 523 insertions(+)
 create mode 100644 registry/bpmct/templates/incus-vm/main.tf

diff --git a/registry/bpmct/templates/incus-vm/main.tf b/registry/bpmct/templates/incus-vm/main.tf
new file mode 100644
index 00000000..f3628cfd
--- /dev/null
+++ b/registry/bpmct/templates/incus-vm/main.tf
@@ -0,0 +1,523 @@
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = ">= 2.4.0"
+    }
+    incus = {
+      source  = "lxc/incus"
+      version = "~> 1.0"
+    }
+    null = {
+      source  = "hashicorp/null"
+      version = "~> 3.0"
+    }
+  }
+}
+
+module "portabledesktop" {
+  count    = data.coder_workspace.me.start_count
+  source   = "registry.coder.com/coder/portabledesktop/coder"
+  version  = "~> 0.1"
+  agent_id = coder_agent.main[0].id
+}
+
+provider "incus" {}
+
+data "coder_workspace" "me" {}
+data "coder_workspace_owner" "me" {}
+
+data "coder_parameter" "host" {
+  name         = "host"
+  display_name = "Host"
+  description  = "Select the host to run this workspace on. **ThinkStation** is an amd64 desktop machine. **CoderPi** is an arm64 Raspberry Pi."
+  type         = "string"
+  default      = "ThinkStation"
+  mutable      = false
+  order        = 1
+
+  option {
+    name  = "ThinkStation"
+    value = "ThinkStation"
+    icon  = "/icon/desktop.svg"
+  }
+
+  option {
+    name  = "CoderPi"
+    value = "CoderPi"
+    icon  = "/icon/memory.svg"
+  }
+}
+
+data "coder_parameter" "image" {
+  name         = "image"
+  display_name = "Image"
+  description  = "The image to use. Ubuntu images use cloud-init. NixOS images are provisioned via incus exec + nixos-rebuild."
+  default      = "ubuntu/jammy/cloud"
+  icon         = "/icon/image.svg"
+  mutable      = true
+
+  option {
+    name  = "Ubuntu 22.04 LTS (Jammy)"
+    value = "ubuntu/jammy/cloud"
+    icon  = "/icon/ubuntu.svg"
+  }
+
+  option {
+    name  = "Ubuntu 24.04 LTS (Noble)"
+    value = "ubuntu/noble/cloud"
+    icon  = "/icon/ubuntu.svg"
+  }
+
+  option {
+    name  = "NixOS 25.11"
+    value = "nixos/25.11"
+    icon  = "/icon/nix.svg"
+  }
+
+  option {
+    name  = "NixOS Unstable"
+    value = "nixos/unstable"
+    icon  = "/icon/nix.svg"
+  }
+}
+
+data "coder_parameter" "cpu" {
+  name         = "cpu"
+  display_name = "CPU"
+  description  = "Number of CPUs to allocate."
+  type         = "number"
+  form_type    = "dropdown"
+  default      = 2
+  icon         = "https://raw.githubusercontent.com/matifali/logos/main/cpu-3.svg"
+  mutable      = true
+  order        = 2
+
+  dynamic "option" {
+    for_each = data.coder_parameter.host.value == "ThinkStation" ? [1, 2, 4, 6, 8, 12] : [0.5, 1, 1.5, 2]
+    content {
+      name  = tostring(option.value)
+      value = option.value
+    }
+  }
+}
+
+data "coder_parameter" "memory" {
+  name         = "memory"
+  display_name = "Memory (GB)"
+  description  = "Amount of memory in GB."
+  type         = "number"
+  form_type    = "slider"
+  default      = 4
+  icon         = "/icon/memory.svg"
+  mutable      = true
+  order        = 3
+  validation {
+    min = 1
+    max = data.coder_parameter.host.value == "ThinkStation" ? 24 : 12
+  }
+}
+
+data "coder_parameter" "usb_passthrough" {
+  name         = "usb_passthrough"
+  display_name = "USB Passthrough"
+  description  = "Pass a USB device through to the VM. Only applicable when host is ThinkStation."
+  type         = "string"
+  form_type    = "dropdown"
+  default      = "none"
+  mutable      = true
+  order        = 4
+
+  option {
+    name  = "None"
+    value = "none"
+  }
+
+  option {
+    name  = "Kindle Paperwhite (1949:0004)"
+    value = "kindle"
+  }
+
+  option {
+    name  = "Nook Simple Touch (2080:0003)"
+    value = "nook"
+  }
+
+  option {
+    name  = "Kindle Fire 1st Gen (1949:0006)"
+    value = "kindle_fire"
+  }
+}
+
+data "coder_parameter" "snapshot_on_stop" {
+  name         = "snapshot_on_stop"
+  display_name = "Snapshot on stop"
+  description  = "Take a snapshot of the VM when the workspace stops."
+  type         = "bool"
+  form_type    = "checkbox"
+  default      = false
+  mutable      = true
+  ephemeral    = true
+  order        = 5
+}
+
+data "coder_parameter" "snapshot_name" {
+  count        = data.coder_parameter.snapshot_on_stop.value == "true" ? 1 : 0
+  name         = "snapshot_name"
+  display_name = "Snapshot name"
+  description  = "Name for the snapshot."
+  type         = "string"
+  default      = "snap-${formatdate("YYYYMMDD-hhmmss", timestamp())}"
+  mutable      = true
+  ephemeral    = true
+  order        = 6
+}
+
+resource "coder_agent" "main" {
+  count = data.coder_workspace.me.start_count
+  arch  = data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"
+  os    = "linux"
+  dir   = "/home/${local.workspace_user}"
+
+  metadata {
+    display_name = "CPU Usage"
+    key          = "0_cpu_usage"
+    script       = "coder stat cpu"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "RAM Usage"
+    key          = "1_ram_usage"
+    script       = "coder stat mem"
+    interval     = 10
+    timeout      = 1
+  }
+
+  metadata {
+    display_name = "Home Disk"
+    key          = "3_home_disk"
+    script       = "coder stat disk --path /home/${local.workspace_user}"
+    interval     = 60
+    timeout      = 1
+  }
+}
+
+resource "incus_image" "image" {
+  remote = local.incus_remote
+  source_image = {
+    remote = "images"
+    # NixOS images on images.linuxcontainers.org use just "nixos/25.11" (no arch suffix in alias).
+    # Other distros like ubuntu append the arch: "ubuntu/jammy/cloud/amd64".
+    name         = local.is_nixos ? data.coder_parameter.image.value : "${data.coder_parameter.image.value}/${data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"}"
+    type         = "virtual-machine"
+    architecture = data.coder_parameter.host.value == "ThinkStation" ? "x86_64" : "aarch64"
+  }
+}
+
+resource "incus_instance" "dev" {
+  remote  = local.incus_remote
+  running = data.coder_workspace.me.start_count == 1
+  name    = "coder-${lower(data.coder_workspace_owner.me.name)}-${lower(data.coder_workspace.me.name)}"
+  image   = incus_image.image.fingerprint
+  type    = "virtual-machine"
+
+  dynamic "device" {
+    for_each = local.usb_device != null ? [local.usb_device] : []
+    content {
+      name = device.value.name
+      type = "usb"
+      properties = {
+        vendorid  = device.value.vendorid
+        productid = device.value.productid
+      }
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      config["cloud-init.user-data"],
+      config["user.coder-agent-token"],
+      config["raw.qemu.conf"],
+      image,
+    ]
+  }
+
+  config = merge(
+    {
+      "limits.cpu"    = tostring(local.cpu)
+      "limits.memory" = "${local.memory}GiB"
+      "raw.qemu.conf" = <<-QEMUCONF
+        [device "qemu_balloon"]
+        driver = "virtio-balloon-pci"
+        bus = "qemu_pcie0"
+        addr = "00.0"
+        multifunction = "on"
+        free-page-reporting = "on"
+      QEMUCONF
+      "security.secureboot"    = false
+      "boot.autostart"         = true
+      "user.coder-agent-token" = local.agent_token
+    },
+    local.is_nixos ? {} : {
+      "cloud-init.user-data" = <<-EOF
+#cloud-config
+hostname: ${lower(data.coder_workspace.me.name)}
+users:
+  - name: ${local.workspace_user}
+    uid: 1000
+    groups: sudo
+    shell: /bin/bash
+    sudo: ALL=(ALL) NOPASSWD:ALL
+write_files:
+  - path: /opt/coder/init
+    permissions: "0755"
+    encoding: b64
+    content: ${base64encode(local.agent_init_script)}
+  - path: /opt/coder/init.env
+    permissions: "0600"
+    content: |
+      CODER_AGENT_TOKEN=${local.agent_token}
+      CODER_AGENT_URL=${data.coder_workspace.me.access_url}
+  - path: /etc/systemd/system/coder-agent.service
+    permissions: "0644"
+    content: |
+      [Unit]
+      Description=Coder Agent
+      After=network-online.target
+      Wants=network-online.target
+
+      [Service]
+      User=${local.workspace_user}
+      EnvironmentFile=/opt/coder/init.env
+      ExecStart=/opt/coder/init
+      Restart=always
+      RestartSec=10
+      TimeoutStopSec=90
+      KillMode=process
+      OOMScoreAdjust=-900
+      SyslogIdentifier=coder-agent
+
+runcmd:
+  - apt-get update -qq && apt-get install -y curl adb
+  - chown -R ${local.workspace_user}:${local.workspace_user} /home/${local.workspace_user}
+  - |
+    if [ ! -s /opt/coder/init ]; then
+      curl -fsSL '${data.coder_workspace.me.access_url}/bin/coder-linux-amd64' -o /opt/coder/coder-agent-bin
+      chmod +x /opt/coder/coder-agent-bin
+      printf '#!/bin/bash\nsource /opt/coder/init.env\nexec /opt/coder/coder-agent-bin agent\n' > /opt/coder/init
+      chmod +x /opt/coder/init
+    fi
+  - systemctl enable --now coder-agent.service
+EOF
+    }
+  )
+}
+
+# Token refresh for Ubuntu/cloud-init VMs
+resource "null_resource" "token_refresh" {
+  count = data.coder_workspace.me.start_count == 1 && !local.is_nixos ? 1 : 0
+
+  triggers = {
+    agent_token = local.agent_token
+    instance    = incus_instance.dev.name
+  }
+
+  depends_on = [incus_instance.dev]
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      echo "Waiting for VM agent to be ready..."
+      for i in $(seq 1 30); do
+        if incus exec ${local.incus_remote}:${incus_instance.dev.name} -- true 2>/dev/null; then
+          break
+        fi
+        echo "Attempt $i: VM agent not ready yet, waiting..."
+        sleep 5
+      done
+      echo "Waiting for cloud-init to complete..."
+      incus exec ${local.incus_remote}:${incus_instance.dev.name} -- bash -c '
+        for i in $(seq 1 60); do
+          if [ -f /var/lib/cloud/instance/boot-finished ]; then
+            break
+          fi
+          sleep 5
+        done
+      '
+      echo "Updating Coder agent token..."
+      incus config set ${local.incus_remote}:${incus_instance.dev.name} user.coder-agent-token ${local.agent_token}
+      incus exec ${local.incus_remote}:${incus_instance.dev.name} -- bash -c '
+        printf "CODER_AGENT_TOKEN=${local.agent_token}\nCODER_AGENT_URL=${data.coder_workspace.me.access_url}\n" > /opt/coder/init.env
+        chown root:root /opt/coder/init.env
+        chmod 600 /opt/coder/init.env
+        systemctl restart coder-agent
+      '
+    EOT
+  }
+}
+
+# Provisioner for NixOS VMs.
+# NixOS does not support cloud-init in the traditional sense.
+# We use incus file push + nixos-rebuild to declare the user and coder-agent service.
+resource "null_resource" "provision_nixos" {
+  count = data.coder_workspace.me.start_count == 1 && local.is_nixos ? 1 : 0
+
+  triggers = {
+    agent_token = local.agent_token
+    instance    = incus_instance.dev.name
+  }
+
+  depends_on = [incus_instance.dev]
+
+  provisioner "local-exec" {
+    # Write the nix module and coder agent files into the VM, then run nixos-rebuild.
+    # We use incus file push for files containing sensitive values or complex content,
+    # and incus exec for commands. This avoids shell quoting issues with heredocs.
+    command = <<-EOT
+      set -e
+      REMOTE="${local.incus_remote}"
+      INSTANCE="${incus_instance.dev.name}"
+      WUSER="${local.workspace_user}"
+      ARCH="${data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"}"
+
+      echo "Waiting for NixOS VM incus-agent to be ready..."
+      for i in $(seq 1 60); do
+        if incus exec "$REMOTE:$INSTANCE" -- true 2>/dev/null; then
+          echo "incus-agent ready after $i attempts"
+          break
+        fi
+        echo "Attempt $i: incus-agent not ready yet, waiting..."
+        sleep 5
+      done
+
+      # Write init script into the VM
+      incus exec "$REMOTE:$INSTANCE" -- mkdir -p /opt/coder
+      echo "${base64encode(local.agent_init_script)}" | base64 -d | incus file push - "$REMOTE:$INSTANCE/opt/coder/init"
+      incus exec "$REMOTE:$INSTANCE" -- chmod 755 /opt/coder/init
+
+      # Write env file into the VM
+      printf 'CODER_AGENT_TOKEN=${local.agent_token}\nCODER_AGENT_URL=${data.coder_workspace.me.access_url}\n' \
+        | incus file push - "$REMOTE:$INSTANCE/opt/coder/init.env" --mode 0600
+
+      # Write the NixOS coder module, substituting the username
+      NIXMOD=$(cat <<NIXMOD_EOF
+{ config, pkgs, lib, ... }:
+{
+  users.users."$WUSER" = {
+    isNormalUser = true;
+    uid = 1000;
+    home = "/home/$WUSER";
+    shell = pkgs.bash;
+    extraGroups = [ "wheel" ];
+  };
+
+  security.sudo.wheelNeedsPassword = false;
+
+  systemd.services.coder-agent = {
+    description = "Coder Agent";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      User = "$WUSER";
+      EnvironmentFile = "/opt/coder/init.env";
+      ExecStart = "/opt/coder/init";
+      Environment = "PATH=/run/current-system/sw/bin:/run/wrappers/bin:/usr/local/bin:/usr/bin:/bin";
+      Restart = "always";
+      RestartSec = 10;
+      TimeoutStopSec = 90;
+      KillMode = "process";
+      OOMScoreAdjust = -900;
+      SyslogIdentifier = "coder-agent";
+    };
+  };
+}
+NIXMOD_EOF
+)
+      echo "$NIXMOD" | incus file push - "$REMOTE:$INSTANCE/etc/nixos/coder.nix"
+
+      # Patch configuration.nix to import coder.nix if not already imported
+      incus exec "$REMOTE:$INSTANCE" -- \
+        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
+        "grep -q coder.nix /etc/nixos/configuration.nix || \
+         sed -i 's|imports = \[|imports = [\n    ./coder.nix|' /etc/nixos/configuration.nix"
+
+      echo "Running nixos-rebuild switch (this may take a few minutes)..."
+      incus exec "$REMOTE:$INSTANCE" -- \
+        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -l -c \
+        "nixos-rebuild switch; EC=\$?; [ \$EC -eq 0 ] || [ \$EC -eq 4 ] || exit \$EC"
+
+      echo "Restarting coder-agent service..."
+      incus exec "$REMOTE:$INSTANCE" -- \
+        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
+        "systemctl daemon-reload; systemctl restart coder-agent.service; sleep 3; systemctl status coder-agent.service || true"
+
+      # Ensure home dir ownership (nixos-rebuild will have created the user home)
+      incus exec "$REMOTE:$INSTANCE" -- \
+        env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
+        "mkdir -p /home/$WUSER && chown 1000:1000 /home/$WUSER && chmod 755 /home/$WUSER"
+
+      echo "NixOS provisioning complete."
+    EOT
+  }
+}
+
+resource "incus_instance_snapshot" "on_stop" {
+  count    = data.coder_parameter.snapshot_on_stop.value == "true" ? 1 : 0
+  remote   = local.incus_remote
+  instance = incus_instance.dev.name
+  name     = try(data.coder_parameter.snapshot_name[0].value, "snap")
+  stateful = false
+  lifecycle {
+    ignore_changes = all
+  }
+}
+
+locals {
+  incus_remote      = data.coder_parameter.host.value == "ThinkStation" ? "thinkstation" : "local"
+  workspace_user    = lower(data.coder_workspace_owner.me.name)
+  cpu               = data.coder_parameter.cpu.value
+  memory            = data.coder_parameter.memory.value
+  agent_id          = data.coder_workspace.me.start_count == 1 ? coder_agent.main[0].id : ""
+  agent_token       = data.coder_workspace.me.start_count == 1 ? coder_agent.main[0].token : ""
+  agent_init_script = data.coder_workspace.me.start_count == 1 ? coder_agent.main[0].init_script : ""
+
+  # Detect NixOS image
+  is_nixos = startswith(data.coder_parameter.image.value, "nixos/")
+
+  # USB device map — add more entries here as needed
+  usb_devices = {
+    kindle      = { name = "kindle", vendorid = "1949", productid = "0004" }
+    nook        = { name = "nook", vendorid = "2080", productid = "0003" }
+    kindle_fire = { name = "kindle_fire", vendorid = "1949", productid = "0006" }
+  }
+
+  usb_passthrough_value = data.coder_parameter.usb_passthrough.value
+  usb_device = (
+    local.usb_passthrough_value != "none" &&
+    data.coder_parameter.host.value == "ThinkStation" &&
+    contains(keys(local.usb_devices), local.usb_passthrough_value)
+  ) ? local.usb_devices[local.usb_passthrough_value] : null
+}
+
+resource "coder_metadata" "info" {
+  count       = data.coder_workspace.me.start_count
+  resource_id = incus_instance.dev.name
+  item {
+    key   = "instance"
+    value = incus_instance.dev.name
+  }
+  item {
+    key   = "image"
+    value = "images:${data.coder_parameter.image.value}"
+  }
+  item {
+    key   = "cpus"
+    value = tostring(local.cpu)
+  }
+  item {
+    key   = "memory"
+    value = tostring(local.memory)
+  }
+}