feat(incus-vm): add NixOS VM support

- Add nixos/25.11 and nixos/unstable image options
- Skip cloud-init config for NixOS images (not supported)
- Add null_resource.provision_nixos that:
  - Waits for incus-agent readiness
  - Pushes coder agent init script and env file via incus file push
  - Generates /etc/nixos/coder.nix declaring the workspace user,
    sudoers, and coder-agent systemd service
  - Patches /etc/nixos/configuration.nix to import coder.nix
  - Runs nixos-rebuild switch (tolerates exit 4 = service warning)
  - Explicitly restarts coder-agent.service post-rebuild
- Service unit sets Environment PATH so curl/tools are findable
- Disable token_refresh resource for NixOS (not needed)
- NixOS image alias does not append arch suffix (unlike Ubuntu)
This commit is contained in:
Ben Potter 2026-04-23 23:50:59 +00:00
parent 22e574926e
commit c86793f43c

View File

@ -0,0 +1,523 @@
terraform {
required_providers {
coder = {
source = "coder/coder"
version = ">= 2.4.0"
}
incus = {
source = "lxc/incus"
version = "~> 1.0"
}
null = {
source = "hashicorp/null"
version = "~> 3.0"
}
}
}
module "portabledesktop" {
count = data.coder_workspace.me.start_count
source = "registry.coder.com/coder/portabledesktop/coder"
version = "~> 0.1"
agent_id = coder_agent.main[0].id
}
provider "incus" {}
data "coder_workspace" "me" {}
data "coder_workspace_owner" "me" {}
data "coder_parameter" "host" {
name = "host"
display_name = "Host"
description = "Select the host to run this workspace on. **ThinkStation** is an amd64 desktop machine. **CoderPi** is an arm64 Raspberry Pi."
type = "string"
default = "ThinkStation"
mutable = false
order = 1
option {
name = "ThinkStation"
value = "ThinkStation"
icon = "/icon/desktop.svg"
}
option {
name = "CoderPi"
value = "CoderPi"
icon = "/icon/memory.svg"
}
}
data "coder_parameter" "image" {
name = "image"
display_name = "Image"
description = "The image to use. Ubuntu images use cloud-init. NixOS images are provisioned via incus exec + nixos-rebuild."
default = "ubuntu/jammy/cloud"
icon = "/icon/image.svg"
mutable = true
option {
name = "Ubuntu 22.04 LTS (Jammy)"
value = "ubuntu/jammy/cloud"
icon = "/icon/ubuntu.svg"
}
option {
name = "Ubuntu 24.04 LTS (Noble)"
value = "ubuntu/noble/cloud"
icon = "/icon/ubuntu.svg"
}
option {
name = "NixOS 25.11"
value = "nixos/25.11"
icon = "/icon/nix.svg"
}
option {
name = "NixOS Unstable"
value = "nixos/unstable"
icon = "/icon/nix.svg"
}
}
data "coder_parameter" "cpu" {
name = "cpu"
display_name = "CPU"
description = "Number of CPUs to allocate."
type = "number"
form_type = "dropdown"
default = 2
icon = "https://raw.githubusercontent.com/matifali/logos/main/cpu-3.svg"
mutable = true
order = 2
dynamic "option" {
for_each = data.coder_parameter.host.value == "ThinkStation" ? [1, 2, 4, 6, 8, 12] : [0.5, 1, 1.5, 2]
content {
name = tostring(option.value)
value = option.value
}
}
}
data "coder_parameter" "memory" {
name = "memory"
display_name = "Memory (GB)"
description = "Amount of memory in GB."
type = "number"
form_type = "slider"
default = 4
icon = "/icon/memory.svg"
mutable = true
order = 3
validation {
min = 1
max = data.coder_parameter.host.value == "ThinkStation" ? 24 : 12
}
}
data "coder_parameter" "usb_passthrough" {
name = "usb_passthrough"
display_name = "USB Passthrough"
description = "Pass a USB device through to the VM. Only applicable when host is ThinkStation."
type = "string"
form_type = "dropdown"
default = "none"
mutable = true
order = 4
option {
name = "None"
value = "none"
}
option {
name = "Kindle Paperwhite (1949:0004)"
value = "kindle"
}
option {
name = "Nook Simple Touch (2080:0003)"
value = "nook"
}
option {
name = "Kindle Fire 1st Gen (1949:0006)"
value = "kindle_fire"
}
}
data "coder_parameter" "snapshot_on_stop" {
name = "snapshot_on_stop"
display_name = "Snapshot on stop"
description = "Take a snapshot of the VM when the workspace stops."
type = "bool"
form_type = "checkbox"
default = false
mutable = true
ephemeral = true
order = 5
}
data "coder_parameter" "snapshot_name" {
count = data.coder_parameter.snapshot_on_stop.value == "true" ? 1 : 0
name = "snapshot_name"
display_name = "Snapshot name"
description = "Name for the snapshot."
type = "string"
default = "snap-${formatdate("YYYYMMDD-hhmmss", timestamp())}"
mutable = true
ephemeral = true
order = 6
}
resource "coder_agent" "main" {
count = data.coder_workspace.me.start_count
arch = data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"
os = "linux"
dir = "/home/${local.workspace_user}"
metadata {
display_name = "CPU Usage"
key = "0_cpu_usage"
script = "coder stat cpu"
interval = 10
timeout = 1
}
metadata {
display_name = "RAM Usage"
key = "1_ram_usage"
script = "coder stat mem"
interval = 10
timeout = 1
}
metadata {
display_name = "Home Disk"
key = "3_home_disk"
script = "coder stat disk --path /home/${local.workspace_user}"
interval = 60
timeout = 1
}
}
resource "incus_image" "image" {
remote = local.incus_remote
source_image = {
remote = "images"
# NixOS images on images.linuxcontainers.org use just "nixos/25.11" (no arch suffix in alias).
# Other distros like ubuntu append the arch: "ubuntu/jammy/cloud/amd64".
name = local.is_nixos ? data.coder_parameter.image.value : "${data.coder_parameter.image.value}/${data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"}"
type = "virtual-machine"
architecture = data.coder_parameter.host.value == "ThinkStation" ? "x86_64" : "aarch64"
}
}
resource "incus_instance" "dev" {
remote = local.incus_remote
running = data.coder_workspace.me.start_count == 1
name = "coder-${lower(data.coder_workspace_owner.me.name)}-${lower(data.coder_workspace.me.name)}"
image = incus_image.image.fingerprint
type = "virtual-machine"
dynamic "device" {
for_each = local.usb_device != null ? [local.usb_device] : []
content {
name = device.value.name
type = "usb"
properties = {
vendorid = device.value.vendorid
productid = device.value.productid
}
}
}
lifecycle {
ignore_changes = [
config["cloud-init.user-data"],
config["user.coder-agent-token"],
config["raw.qemu.conf"],
image,
]
}
config = merge(
{
"limits.cpu" = tostring(local.cpu)
"limits.memory" = "${local.memory}GiB"
"raw.qemu.conf" = <<-QEMUCONF
[device "qemu_balloon"]
driver = "virtio-balloon-pci"
bus = "qemu_pcie0"
addr = "00.0"
multifunction = "on"
free-page-reporting = "on"
QEMUCONF
"security.secureboot" = false
"boot.autostart" = true
"user.coder-agent-token" = local.agent_token
},
local.is_nixos ? {} : {
"cloud-init.user-data" = <<-EOF
#cloud-config
hostname: ${lower(data.coder_workspace.me.name)}
users:
- name: ${local.workspace_user}
uid: 1000
groups: sudo
shell: /bin/bash
sudo: ALL=(ALL) NOPASSWD:ALL
write_files:
- path: /opt/coder/init
permissions: "0755"
encoding: b64
content: ${base64encode(local.agent_init_script)}
- path: /opt/coder/init.env
permissions: "0600"
content: |
CODER_AGENT_TOKEN=${local.agent_token}
CODER_AGENT_URL=${data.coder_workspace.me.access_url}
- path: /etc/systemd/system/coder-agent.service
permissions: "0644"
content: |
[Unit]
Description=Coder Agent
After=network-online.target
Wants=network-online.target
[Service]
User=${local.workspace_user}
EnvironmentFile=/opt/coder/init.env
ExecStart=/opt/coder/init
Restart=always
RestartSec=10
TimeoutStopSec=90
KillMode=process
OOMScoreAdjust=-900
SyslogIdentifier=coder-agent
runcmd:
- apt-get update -qq && apt-get install -y curl adb
- chown -R ${local.workspace_user}:${local.workspace_user} /home/${local.workspace_user}
- |
if [ ! -s /opt/coder/init ]; then
curl -fsSL '${data.coder_workspace.me.access_url}/bin/coder-linux-amd64' -o /opt/coder/coder-agent-bin
chmod +x /opt/coder/coder-agent-bin
printf '#!/bin/bash\nsource /opt/coder/init.env\nexec /opt/coder/coder-agent-bin agent\n' > /opt/coder/init
chmod +x /opt/coder/init
fi
- systemctl enable --now coder-agent.service
EOF
}
)
}
# Token refresh for Ubuntu/cloud-init VMs
resource "null_resource" "token_refresh" {
count = data.coder_workspace.me.start_count == 1 && !local.is_nixos ? 1 : 0
triggers = {
agent_token = local.agent_token
instance = incus_instance.dev.name
}
depends_on = [incus_instance.dev]
provisioner "local-exec" {
command = <<-EOT
echo "Waiting for VM agent to be ready..."
for i in $(seq 1 30); do
if incus exec ${local.incus_remote}:${incus_instance.dev.name} -- true 2>/dev/null; then
break
fi
echo "Attempt $i: VM agent not ready yet, waiting..."
sleep 5
done
echo "Waiting for cloud-init to complete..."
incus exec ${local.incus_remote}:${incus_instance.dev.name} -- bash -c '
for i in $(seq 1 60); do
if [ -f /var/lib/cloud/instance/boot-finished ]; then
break
fi
sleep 5
done
'
echo "Updating Coder agent token..."
incus config set ${local.incus_remote}:${incus_instance.dev.name} user.coder-agent-token ${local.agent_token}
incus exec ${local.incus_remote}:${incus_instance.dev.name} -- bash -c '
printf "CODER_AGENT_TOKEN=${local.agent_token}\nCODER_AGENT_URL=${data.coder_workspace.me.access_url}\n" > /opt/coder/init.env
chown root:root /opt/coder/init.env
chmod 600 /opt/coder/init.env
systemctl restart coder-agent
'
EOT
}
}
# Provisioner for NixOS VMs.
# NixOS does not support cloud-init in the traditional sense.
# We use incus file push + nixos-rebuild to declare the user and coder-agent service.
resource "null_resource" "provision_nixos" {
count = data.coder_workspace.me.start_count == 1 && local.is_nixos ? 1 : 0
triggers = {
agent_token = local.agent_token
instance = incus_instance.dev.name
}
depends_on = [incus_instance.dev]
provisioner "local-exec" {
# Write the nix module and coder agent files into the VM, then run nixos-rebuild.
# We use incus file push for files containing sensitive values or complex content,
# and incus exec for commands. This avoids shell quoting issues with heredocs.
command = <<-EOT
set -e
REMOTE="${local.incus_remote}"
INSTANCE="${incus_instance.dev.name}"
WUSER="${local.workspace_user}"
ARCH="${data.coder_parameter.host.value == "ThinkStation" ? "amd64" : "arm64"}"
echo "Waiting for NixOS VM incus-agent to be ready..."
for i in $(seq 1 60); do
if incus exec "$REMOTE:$INSTANCE" -- true 2>/dev/null; then
echo "incus-agent ready after $i attempts"
break
fi
echo "Attempt $i: incus-agent not ready yet, waiting..."
sleep 5
done
# Write init script into the VM
incus exec "$REMOTE:$INSTANCE" -- mkdir -p /opt/coder
echo "${base64encode(local.agent_init_script)}" | base64 -d | incus file push - "$REMOTE:$INSTANCE/opt/coder/init"
incus exec "$REMOTE:$INSTANCE" -- chmod 755 /opt/coder/init
# Write env file into the VM
printf 'CODER_AGENT_TOKEN=${local.agent_token}\nCODER_AGENT_URL=${data.coder_workspace.me.access_url}\n' \
| incus file push - "$REMOTE:$INSTANCE/opt/coder/init.env" --mode 0600
# Write the NixOS coder module, substituting the username
NIXMOD=$(cat <<NIXMOD_EOF
{ config, pkgs, lib, ... }:
{
users.users."$WUSER" = {
isNormalUser = true;
uid = 1000;
home = "/home/$WUSER";
shell = pkgs.bash;
extraGroups = [ "wheel" ];
};
security.sudo.wheelNeedsPassword = false;
systemd.services.coder-agent = {
description = "Coder Agent";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
User = "$WUSER";
EnvironmentFile = "/opt/coder/init.env";
ExecStart = "/opt/coder/init";
Environment = "PATH=/run/current-system/sw/bin:/run/wrappers/bin:/usr/local/bin:/usr/bin:/bin";
Restart = "always";
RestartSec = 10;
TimeoutStopSec = 90;
KillMode = "process";
OOMScoreAdjust = -900;
SyslogIdentifier = "coder-agent";
};
};
}
NIXMOD_EOF
)
echo "$NIXMOD" | incus file push - "$REMOTE:$INSTANCE/etc/nixos/coder.nix"
# Patch configuration.nix to import coder.nix if not already imported
incus exec "$REMOTE:$INSTANCE" -- \
env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
"grep -q coder.nix /etc/nixos/configuration.nix || \
sed -i 's|imports = \[|imports = [\n ./coder.nix|' /etc/nixos/configuration.nix"
echo "Running nixos-rebuild switch (this may take a few minutes)..."
incus exec "$REMOTE:$INSTANCE" -- \
env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -l -c \
"nixos-rebuild switch; EC=\$?; [ \$EC -eq 0 ] || [ \$EC -eq 4 ] || exit \$EC"
echo "Restarting coder-agent service..."
incus exec "$REMOTE:$INSTANCE" -- \
env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
"systemctl daemon-reload; systemctl restart coder-agent.service; sleep 3; systemctl status coder-agent.service || true"
# Ensure home dir ownership (nixos-rebuild will have created the user home)
incus exec "$REMOTE:$INSTANCE" -- \
env PATH=/run/current-system/sw/bin /run/current-system/sw/bin/bash -c \
"mkdir -p /home/$WUSER && chown 1000:1000 /home/$WUSER && chmod 755 /home/$WUSER"
echo "NixOS provisioning complete."
EOT
}
}
resource "incus_instance_snapshot" "on_stop" {
count = data.coder_parameter.snapshot_on_stop.value == "true" ? 1 : 0
remote = local.incus_remote
instance = incus_instance.dev.name
name = try(data.coder_parameter.snapshot_name[0].value, "snap")
stateful = false
lifecycle {
ignore_changes = all
}
}
locals {
incus_remote = data.coder_parameter.host.value == "ThinkStation" ? "thinkstation" : "local"
workspace_user = lower(data.coder_workspace_owner.me.name)
cpu = data.coder_parameter.cpu.value
memory = data.coder_parameter.memory.value
agent_id = data.coder_workspace.me.start_count == 1 ? coder_agent.main[0].id : ""
agent_token = data.coder_workspace.me.start_count == 1 ? coder_agent.main[0].token : ""
agent_init_script = data.coder_workspace.me.start_count == 1 ? coder_agent.main[0].init_script : ""
# Detect NixOS image
is_nixos = startswith(data.coder_parameter.image.value, "nixos/")
# USB device map add more entries here as needed
usb_devices = {
kindle = { name = "kindle", vendorid = "1949", productid = "0004" }
nook = { name = "nook", vendorid = "2080", productid = "0003" }
kindle_fire = { name = "kindle_fire", vendorid = "1949", productid = "0006" }
}
usb_passthrough_value = data.coder_parameter.usb_passthrough.value
usb_device = (
local.usb_passthrough_value != "none" &&
data.coder_parameter.host.value == "ThinkStation" &&
contains(keys(local.usb_devices), local.usb_passthrough_value)
) ? local.usb_devices[local.usb_passthrough_value] : null
}
resource "coder_metadata" "info" {
count = data.coder_workspace.me.start_count
resource_id = incus_instance.dev.name
item {
key = "instance"
value = incus_instance.dev.name
}
item {
key = "image"
value = "images:${data.coder_parameter.image.value}"
}
item {
key = "cpus"
value = tostring(local.cpu)
}
item {
key = "memory"
value = tostring(local.memory)
}
}