feat(coder/modules/git-clone): add support for recursive submodule cloning and parallel jobs

registry/coder/modules/git-clone/README.md

@@ -14,7 +14,7 @@ This module allows you to automatically clone a repository by URL and skip if it
 module "git-clone" {
   count    = data.coder_workspace.me.start_count
   source   = "registry.coder.com/coder/git-clone/coder"
-  version  = "1.3.1"
+  version  = "1.4.0"
   agent_id = coder_agent.example.id
   url      = "https://github.com/coder/coder"
 }
@@ -28,7 +28,7 @@ module "git-clone" {
 module "git-clone" {
   count    = data.coder_workspace.me.start_count
   source   = "registry.coder.com/coder/git-clone/coder"
-  version  = "1.3.1"
+  version  = "1.4.0"
   agent_id = coder_agent.example.id
   url      = "https://github.com/coder/coder"
   base_dir = "~/projects/coder"
@@ -43,7 +43,7 @@ To use with [Git Authentication](https://coder.com/docs/v2/latest/admin/git-prov
 module "git-clone" {
   count    = data.coder_workspace.me.start_count
   source   = "registry.coder.com/coder/git-clone/coder"
-  version  = "1.3.1"
+  version  = "1.4.0"
   agent_id = coder_agent.example.id
   url      = "https://github.com/coder/coder"
 }
@@ -70,7 +70,7 @@ data "coder_parameter" "git_repo" {
 module "git_clone" {
   count    = data.coder_workspace.me.start_count
   source   = "registry.coder.com/coder/git-clone/coder"
-  version  = "1.3.1"
+  version  = "1.4.0"
   agent_id = coder_agent.example.id
   url      = data.coder_parameter.git_repo.value
 }
@@ -105,7 +105,7 @@ Configuring `git-clone` for a self-hosted GitHub Enterprise Server running at `g
 module "git-clone" {
   count    = data.coder_workspace.me.start_count
   source   = "registry.coder.com/coder/git-clone/coder"
-  version  = "1.3.1"
+  version  = "1.4.0"
   agent_id = coder_agent.example.id
   url      = "https://github.example.com/coder/coder/tree/feat/example"
   git_providers = {
@@ -125,7 +125,7 @@ To GitLab clone with a specific branch like `feat/example`
 module "git-clone" {
   count    = data.coder_workspace.me.start_count
   source   = "registry.coder.com/coder/git-clone/coder"
-  version  = "1.3.1"
+  version  = "1.4.0"
   agent_id = coder_agent.example.id
   url      = "https://gitlab.com/coder/coder/-/tree/feat/example"
 }
@@ -137,7 +137,7 @@ Configuring `git-clone` for a self-hosted GitLab running at `gitlab.example.com`
 module "git-clone" {
   count    = data.coder_workspace.me.start_count
   source   = "registry.coder.com/coder/git-clone/coder"
-  version  = "1.3.1"
+  version  = "1.4.0"
   agent_id = coder_agent.example.id
   url      = "https://gitlab.example.com/coder/coder/-/tree/feat/example"
   git_providers = {
@@ -159,7 +159,7 @@ For example, to clone the `feat/example` branch:
 module "git-clone" {
   count       = data.coder_workspace.me.start_count
   source      = "registry.coder.com/coder/git-clone/coder"
-  version     = "1.3.1"
+  version     = "1.4.0"
   agent_id    = coder_agent.example.id
   url         = "https://github.com/coder/coder"
   branch_name = "feat/example"
@@ -177,7 +177,7 @@ For example, this will clone into the `~/projects/coder/coder-dev` folder:
 module "git-clone" {
   count       = data.coder_workspace.me.start_count
   source      = "registry.coder.com/coder/git-clone/coder"
-  version     = "1.3.1"
+  version     = "1.4.0"
   agent_id    = coder_agent.example.id
   url         = "https://github.com/coder/coder"
   folder_name = "coder-dev"
@@ -196,13 +196,33 @@ If not defined, the default, `0`, performs a full clone.
 module "git-clone" {
   count    = data.coder_workspace.me.start_count
   source   = "registry.coder.com/coder/git-clone/coder"
-  version  = "1.3.1"
+  version  = "1.4.0"
   agent_id = coder_agent.example.id
   url      = "https://github.com/coder/coder"
   depth    = 1
 }
 ```
 
+## Recurse submodules
+
+Set `recurse_submodules = true` to initialize and clone submodules during the
+clone (equivalent to `git clone --recurse-submodules`).
+
+Pair it with `clone_jobs` to fetch submodules in parallel (equivalent to
+`git clone --jobs <n>`) and speed up workspace start.
+
+```tf
+module "git-clone" {
+  count              = data.coder_workspace.me.start_count
+  source             = "registry.coder.com/coder/git-clone/coder"
+  version            = "1.4.0"
+  agent_id           = coder_agent.example.id
+  url                = "https://github.com/coder/coder"
+  recurse_submodules = true
+  clone_jobs         = 8
+}
+```
+
 ## Pre-clone script
 
 Run a custom script before cloning the repository by setting the `pre_clone_script` variable.
@@ -212,7 +232,7 @@ This is useful for preparing the environment or validating prerequisites before
 module "git-clone" {
   count            = data.coder_workspace.me.start_count
   source           = "registry.coder.com/coder/git-clone/coder"
-  version          = "1.3.1"
+  version          = "1.4.0"
   agent_id         = coder_agent.example.id
   url              = "https://github.com/coder/coder"
   pre_clone_script = <<-EOT
@@ -235,7 +255,7 @@ This is useful for running initialization tasks like installing dependencies or
 module "git-clone" {
   count             = data.coder_workspace.me.start_count
   source            = "registry.coder.com/coder/git-clone/coder"
-  version           = "1.3.1"
+  version           = "1.4.0"
   agent_id          = coder_agent.example.id
   url               = "https://github.com/coder/coder"
   post_clone_script = <<-EOT

registry/coder/modules/git-clone/main.test.ts

@@ -1,11 +1,37 @@
 import { describe, expect, it } from "bun:test";
 import {
-  executeScriptInContainer,
+  execContainer,
+  findResourceInstance,
+  runContainer,
   runTerraformApply,
   runTerraformInit,
   testRequiredVariables,
+  type scriptOutput,
+  type TerraformState,
 } from "~test";
 
+// The clone script uses bash arrays, which busybox `sh` (alpine's default
+// shell) cannot parse. Install bash in the container, then run the script
+// with bash. The optional `before` setup step still runs with `sh`.
+const executeScriptInContainer = async (
+  state: TerraformState,
+  image: string,
+  before?: string,
+): Promise<scriptOutput> => {
+  const instance = findResourceInstance(state, "coder_script");
+  const id = await runContainer(image);
+  await execContainer(id, ["sh", "-c", "apk add --no-cache bash >/dev/null"]);
+  if (before) {
+    await execContainer(id, ["sh", "-c", before]);
+  }
+  const resp = await execContainer(id, ["bash", "-c", instance.script]);
+  return {
+    exitCode: resp.exitCode,
+    stdout: resp.stdout.trim().split("\n"),
+    stderr: resp.stderr.trim().split("\n"),
+  };
+};
+
 describe("git-clone", async () => {
   await runTerraformInit(import.meta.dir);
 
@@ -31,8 +57,8 @@ describe("git-clone", async () => {
     });
     const output = await executeScriptInContainer(state, "alpine/git");
     expect(output.stdout).toEqual([
-      "Creating directory ~/fake-url...",
+      "Creating directory /root/fake-url...",
-      "Cloning fake-url to ~/fake-url...",
+      "Cloning fake-url to /root/fake-url...",
     ]);
     expect(output.stderr.join(" ")).toContain("fatal");
     expect(output.stderr.join(" ")).toContain("fake-url");
@@ -207,8 +233,8 @@ describe("git-clone", async () => {
     const output = await executeScriptInContainer(state, "alpine/git");
     expect(output.exitCode).toBe(0);
     expect(output.stdout).toEqual([
-      "Creating directory ~/repo-tests.log...",
+      "Creating directory /root/repo-tests.log...",
-      "Cloning https://github.com/michaelbrewer/repo-tests.log to ~/repo-tests.log on branch feat/branch...",
+      "Cloning https://github.com/michaelbrewer/repo-tests.log to /root/repo-tests.log on branch feat/branch...",
     ]);
   });
 
@@ -220,8 +246,8 @@ describe("git-clone", async () => {
     const output = await executeScriptInContainer(state, "alpine/git");
     expect(output.exitCode).toBe(0);
     expect(output.stdout).toEqual([
-      "Creating directory ~/repo-tests.log...",
+      "Creating directory /root/repo-tests.log...",
-      "Cloning https://gitlab.com/mike.brew/repo-tests.log to ~/repo-tests.log on branch feat/branch...",
+      "Cloning https://gitlab.com/mike.brew/repo-tests.log to /root/repo-tests.log on branch feat/branch...",
     ]);
   });
 
@@ -241,8 +267,8 @@ describe("git-clone", async () => {
     const output = await executeScriptInContainer(state, "alpine/git");
     expect(output.exitCode).toBe(0);
     expect(output.stdout).toEqual([
-      "Creating directory ~/repo-tests.log...",
+      "Creating directory /root/repo-tests.log...",
-      "Cloning https://github.com/michaelbrewer/repo-tests.log to ~/repo-tests.log on branch feat/branch...",
+      "Cloning https://github.com/michaelbrewer/repo-tests.log to /root/repo-tests.log on branch feat/branch...",
     ]);
   });
 
@@ -256,7 +282,6 @@ describe("git-clone", async () => {
     const output = await executeScriptInContainer(
       state,
       "alpine/git",
-      "sh",
       "mkdir -p /tmp/fake-url && echo 'existing' > /tmp/fake-url/file.txt",
     );
     expect(output.stdout).toContain("Running post-clone script...");
@@ -272,7 +297,7 @@ describe("git-clone", async () => {
     const output = await executeScriptInContainer(state, "alpine/git");
     expect(output.stdout).toContain("Running pre-clone script...");
     expect(output.stdout).toContain("Pre-clone script executed");
-    expect(output.stdout).toContain("Cloning fake-url to ~/fake-url...");
+    expect(output.stdout).toContain("Cloning fake-url to /root/fake-url...");
   });
 
   it("fails when pre-clone script fails", async () => {
@@ -285,7 +310,64 @@ describe("git-clone", async () => {
     expect(output.exitCode).toBe(42);
     expect(output.stdout).toContain("Running pre-clone script...");
     expect(output.stdout).toContain("Pre-clone script failed");
-    expect(output.stdout).not.toContain("Cloning fake-url to ~/fake-url...");
+    expect(output.stdout).not.toContain(
+      "Cloning fake-url to /root/fake-url...",
+    );
+  });
+
+  it("defaults recurse_submodules to false and clone_jobs to 0", async () => {
+    const state = await runTerraformApply(import.meta.dir, {
+      agent_id: "foo",
+      url: "fake-url",
+    });
+    const script = findResourceInstance(state, "coder_script").script;
+    expect(script).toContain('RECURSE_SUBMODULES="false"');
+    expect(script).toContain('CLONE_JOBS="0"');
+  });
+
+  it("sets RECURSE_SUBMODULES=true when recurse_submodules is enabled", async () => {
+    const state = await runTerraformApply(import.meta.dir, {
+      agent_id: "foo",
+      url: "fake-url",
+      recurse_submodules: "true",
+    });
+    const script = findResourceInstance(state, "coder_script").script;
+    expect(script).toContain('RECURSE_SUBMODULES="true"');
+  });
+
+  it("sets CLONE_JOBS when clone_jobs > 0", async () => {
+    const state = await runTerraformApply(import.meta.dir, {
+      agent_id: "foo",
+      url: "fake-url",
+      recurse_submodules: "true",
+      clone_jobs: "8",
+    });
+    const script = findResourceInstance(state, "coder_script").script;
+    expect(script).toContain('CLONE_JOBS="8"');
+  });
+
+  it("rejects non-positive clone_jobs", async () => {
+    const t = async () => {
+      await runTerraformApply(import.meta.dir, {
+        agent_id: "foo",
+        url: "fake-url",
+        clone_jobs: "-1",
+      });
+    };
+    expect(t).toThrow("clone_jobs must be a positive integer when set.");
+  });
+
+  it("rejects clone_jobs without recurse_submodules", async () => {
+    const t = async () => {
+      await runTerraformApply(import.meta.dir, {
+        agent_id: "foo",
+        url: "fake-url",
+        clone_jobs: "4",
+      });
+    };
+    expect(t).toThrow(
+      "clone_jobs only affects submodule fetching, so it requires recurse_submodules",
+    );
   });
 
   it("fails when post-clone script fails", async () => {
@@ -298,7 +380,6 @@ describe("git-clone", async () => {
     const output = await executeScriptInContainer(
       state,
       "alpine/git",
-      "sh",
       "mkdir -p /tmp/fake-url && echo 'existing' > /tmp/fake-url/file.txt",
     );
     expect(output.exitCode).toBe(43);

registry/coder/modules/git-clone/main.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">= 1.0"
+  required_version = ">= 1.9"
 
   required_providers {
     coder = {
@@ -62,6 +62,26 @@ variable "depth" {
   default     = 0
 }
 
+variable "recurse_submodules" {
+  description = "If true, clone submodules recursively (equivalent to `git clone --recurse-submodules`)."
+  type        = bool
+  default     = false
+}
+
+variable "clone_jobs" {
+  description = "If set, fetch submodules in parallel using this many jobs (equivalent to `git clone --jobs <n>`). Only takes effect when `recurse_submodules = true`."
+  type        = number
+  default     = null
+  validation {
+    condition     = var.clone_jobs == null || var.clone_jobs > 0
+    error_message = "clone_jobs must be a positive integer when set."
+  }
+  validation {
+    condition     = var.clone_jobs == null || var.recurse_submodules
+    error_message = "clone_jobs only affects submodule fetching, so it requires recurse_submodules = true."
+  }
+}
+
 variable "post_clone_script" {
   description = "Custom script to run after cloning the repository. Runs always after git clone, even if the repository already exists."
   type        = string
@@ -136,6 +156,8 @@ resource "coder_script" "git_clone" {
     REPO_URL : local.clone_url,
     BRANCH_NAME : local.branch_name,
     DEPTH              = var.depth,
+    RECURSE_SUBMODULES = tostring(var.recurse_submodules),
+    CLONE_JOBS         = coalesce(var.clone_jobs, 0),
     POST_CLONE_SCRIPT : local.encoded_post_clone_script,
     PRE_CLONE_SCRIPT : local.encoded_pre_clone_script,
   })

registry/coder/modules/git-clone/run.sh

@@ -8,6 +8,8 @@ BRANCH_NAME="${BRANCH_NAME}"
 # Expand home if it's specified!
 CLONE_PATH="$${CLONE_PATH/#\~/$${HOME}}"
 DEPTH="${DEPTH}"
+RECURSE_SUBMODULES="${RECURSE_SUBMODULES}"
+CLONE_JOBS="${CLONE_JOBS}"
 POST_CLONE_SCRIPT="${POST_CLONE_SCRIPT}"
 PRE_CLONE_SCRIPT="${PRE_CLONE_SCRIPT}"
 
@@ -46,23 +48,27 @@ if [ -n "$PRE_CLONE_SCRIPT" ]; then
   rm "$PRE_CLONE_TMP"
 fi
 
+# Build optional git clone flags
+CLONE_FLAGS=()
+if [ "$DEPTH" -gt 0 ]; then
+  CLONE_FLAGS+=(--depth "$DEPTH")
+fi
+if [ "$RECURSE_SUBMODULES" = "true" ]; then
+  CLONE_FLAGS+=(--recurse-submodules)
+fi
+if [ "$CLONE_JOBS" -gt 0 ]; then
+  CLONE_FLAGS+=(--jobs "$CLONE_JOBS")
+fi
+
 # Check if the directory is empty
 # and if it is, clone the repo, otherwise skip cloning
 if [ -z "$(ls -A "$CLONE_PATH")" ]; then
   if [ -z "$BRANCH_NAME" ]; then
     echo "Cloning $REPO_URL to $CLONE_PATH..."
-    if [ "$DEPTH" -gt 0 ]; then
+    git clone "$${CLONE_FLAGS[@]}" "$REPO_URL" "$CLONE_PATH"
-      git clone --depth "$DEPTH" "$REPO_URL" "$CLONE_PATH"
-    else
-      git clone "$REPO_URL" "$CLONE_PATH"
-    fi
   else
     echo "Cloning $REPO_URL to $CLONE_PATH on branch $BRANCH_NAME..."
-    if [ "$DEPTH" -gt 0 ]; then
+    git clone "$${CLONE_FLAGS[@]}" -b "$BRANCH_NAME" "$REPO_URL" "$CLONE_PATH"
-      git clone --depth "$DEPTH" -b "$BRANCH_NAME" "$REPO_URL" "$CLONE_PATH"
-    else
-      git clone "$REPO_URL" -b "$BRANCH_NAME" "$CLONE_PATH"
-    fi
   fi
 else
   echo "$CLONE_PATH already exists and isn't empty, skipping clone!"