Add partial-clone mirror caches and config upgrade

README.md

@@ -45,6 +45,7 @@ aspects of the mirroring process such as;
 * whether the mirror is locked to a particular version of the external library.
 * whether the mirror is tracking a tag or a branch.
 * whether the mirror includes the entire external library or just a subdirectory.
+* whether a subdirectory mirror uses Git partial clone to avoid unrelated blobs.
 
 ## Install
 
@@ -73,7 +74,7 @@ unrelated staged, unstaged, and untracked work untouched and out of Braid's
 commits. `status`, `diff`, and `push` are the usual commands for deciding
 whether to pull, prepare a patch, or send local mirror changes upstream.
 
-Use `--no-commit` with `add`, `pull`, or `remove` to stage Braid's changes
+Use `--no-commit` with `add`, `pull`, `remove`, or `upgrade-config` to stage Braid's changes
 without creating the automatic commit. Braid stages only `.braids.json` and the
 selected mirror paths; unrelated staged files stay staged and will be included
 in your next `git commit` unless you unstage them first.
@@ -110,6 +111,16 @@ braid add help
 braid add --help
 ```
 
+For an upstream with large blobs outside the mirrored subdirectory, opt into
+Git partial clone with `braid add <url> <local_path> --path <remote_path>
+--partial-clone`. This stores `"partial_clone": true` in config version 2 and
+uses Git's `blob:none` filter for repository-local cache hydration and fetches.
+The upstream server must support Git object filtering. The setting is ignored
+when caching is disabled or a global cache is selected.
+
+Repositories with config version 1 must run `braid upgrade-config`. The command
+commits the version 2 config by default; pass `--no-commit` to stage it instead.
+
 ### Shell Completion
 
 Braid can print a Bash completion script to `stdout`:

docs/migration-from-ruby-braid.md

@@ -150,7 +150,7 @@ Migration impact:
 
 | Area | Ruby Braid | Current Go Braid | Migration impact |
 | --- | --- | --- | --- |
-| Commands | `add`, `update`, `remove`, `diff`, `push`, `setup`, `version`, `status`, `upgrade-config` | `pull` is the documented mirror-update command; `update` and `up` are aliases; same other core commands, plus `sync`; no `upgrade-config` | Prefer `pull` in new docs and scripts. Scripts using `upgrade-config` must run Ruby Braid before migration or be removed. |
+| Commands | `add`, `update`, `remove`, `diff`, `push`, `setup`, `version`, `status`, `upgrade-config` | `pull` is the documented mirror-update command; `update` and `up` are aliases; same other core commands, plus `sync`; `upgrade-config` migrates versioned JSON config | Prefer `pull` in new docs and scripts. Use Go Braid's `upgrade-config` only for `.braids.json` version 1 to 2 migration. |
 | Help form | `braid help`, `braid add help`, `braid add --help`; the old README also advertised `braid help add`, but the `v1.1.10` gem does not provide command-specific help through that form | `braid help`, `braid add help`, `braid add --help` | Prefer `braid <command> help` or `braid <command> --help`. |
 | Verbose flag | Per-command `--verbose`/`-v` | Global `--verbose`/`-v` before the command | Use `braid -v pull ...`, not `braid pull -v ...`. |
 | Quiet flag | No global quiet flag | Global `--quiet` before the command; incompatible with `--verbose` | Use `braid --quiet <command> ...` in automation that wants data, warnings, errors, and recovery output without progress or informational chatter. |
@@ -165,7 +165,8 @@ Migration impact:
 ## Configuration Compatibility
 
 The Go tool supports only the modern `.braids.json` shape with
-`config_version: 1` and a `mirrors` object keyed by local mirror path.
+`config_version: 2` and a `mirrors` object keyed by local mirror path. Version 1
+JSON config can be migrated with `braid upgrade-config`.
 
 Supported mirror attributes remain:
 
@@ -174,12 +175,13 @@ Supported mirror attributes remain:
 - `tag`
 - `path`
 - `revision`
+- `partial_clone` (optional; requires `path`)
 
 Important differences:
 
 - Legacy `.braids` YAML/PStore config is not read or upgraded.
 - Older `.braids.json` layouts without `config_version` are not upgraded.
-- `upgrade-config` is not implemented.
+- `upgrade-config` does not migrate legacy `.braids` YAML/PStore data.
 - Unknown root fields and unknown mirror fields fail validation.
 - Missing `config_version`, missing `mirrors`, missing `url`, missing
   `revision`, or a mirror with both `branch` and `tag` fails validation.

integration/lifecycle_test.go

@@ -1,6 +1,7 @@
 package integration
 
 import (
+	"os"
 	"path/filepath"
 	"testing"
 )
@@ -135,3 +136,115 @@ func TestExecutablePrimaryLifecycle(t *testing.T) {
 	assertNoRemote(t, env, downstream, remote)
 	assertClean(t, env, downstream)
 }
+
+func TestUpgradeConfigCommitAndNoCommit(t *testing.T) {
+	for _, noCommit := range []bool{false, true} {
+		t.Run(map[bool]string{false: "commit", true: "no-commit"}[noCommit], func(t *testing.T) {
+			root := t.TempDir()
+			env := newProcessEnv(t, root)
+			braid := braidBinary(t)
+			repo := filepath.Join(root, "repo")
+			initRepo(t, env, repo)
+			if err := os.WriteFile(filepath.Join(repo, ".braids.json"), []byte("{\"config_version\":1,\"mirrors\":{}}\n"), 0o644); err != nil {
+				t.Fatal(err)
+			}
+			gitOK(t, env, repo, "add", ".braids.json")
+			gitOK(t, env, repo, "commit", "-m", "version one")
+			before := gitOutput(t, env, repo, "rev-parse", "HEAD")
+			args := []string{"upgrade-config"}
+			if noCommit {
+				args = append(args, "--no-commit")
+			}
+			result := runBraid(t, env, repo, braid, args...)
+			assertExit(t, result, 0)
+			if noCommit {
+				if got := gitOutput(t, env, repo, "rev-parse", "HEAD"); got != before {
+					t.Fatalf("HEAD moved: %s", got)
+				}
+				assertContains(t, gitOK(t, env, repo, "diff", "--cached").stdout, `"config_version": 2`)
+			} else {
+				assertLatestCommit(t, env, repo, defaultName+" <"+defaultEmail+">", "Upgrade Braid config to version 2")
+				assertClean(t, env, repo)
+			}
+		})
+	}
+}
+
+func TestPartialCloneSubdirectoryLifecycle(t *testing.T) {
+	root := t.TempDir()
+	env := newProcessEnv(t, root)
+	braid := braidBinary(t)
+	upstream := filepath.Join(root, "upstream")
+	initRepo(t, env, upstream)
+	gitOK(t, env, upstream, "config", "uploadpack.allowFilter", "true")
+	gitOK(t, env, upstream, "config", "receive.denyCurrentBranch", "updateInstead")
+	writeFile(t, upstream, "wanted/file.txt", "one\n")
+	writeFile(t, upstream, "other/large.txt", string(make([]byte, 1024*1024)))
+	first := commitAll(t, env, upstream, "initial")
+	unrelatedBlob := gitOutput(t, env, upstream, "rev-parse", "HEAD:other/large.txt")
+
+	downstream := filepath.Join(root, "downstream")
+	initRepo(t, env, downstream)
+	writeFile(t, downstream, "README.md", "downstream\n")
+	commitAll(t, env, downstream, "initial")
+
+	result := runBraid(t, env, downstream, braid, "add", upstream, "vendor/wanted", "--path", "wanted", "--partial-clone")
+	assertExit(t, result, 0)
+	assertFile(t, downstream, "vendor/wanted/file.txt", "one\n")
+	assertConfigRaw(t, downstream, map[string]configMirror{
+		"vendor/wanted": {URL: upstream, Branch: "main", Path: "wanted", Revision: first, PartialClone: true},
+	})
+	cachePath := repositoryCachePath(t, downstream, "vendor/wanted", configMirror{URL: upstream, Branch: "main", Path: "wanted"})
+	missing := runProcess(t, env.with("GIT_NO_LAZY_FETCH", "1"), cachePath, "git", "cat-file", "-e", unrelatedBlob)
+	if missing.exitCode == 0 {
+		t.Fatalf("unrelated blob %s unexpectedly present in partial cache", unrelatedBlob)
+	}
+	missing = runProcess(t, env.with("GIT_NO_LAZY_FETCH", "1"), downstream, "git", "cat-file", "-e", unrelatedBlob)
+	if missing.exitCode == 0 {
+		t.Fatalf("unrelated blob %s unexpectedly present downstream", unrelatedBlob)
+	}
+
+	writeFile(t, downstream, "vendor/wanted/file.txt", "pushed\n")
+	commitAll(t, env, downstream, "local partial mirror change")
+	result = runBraid(t, env, downstream, braid, "push", "vendor/wanted", "--message", "Push partial mirror")
+	assertExit(t, result, 0)
+	assertFile(t, upstream, "wanted/file.txt", "pushed\n")
+	assertFile(t, upstream, "other/large.txt", string(make([]byte, 1024*1024)))
+	assertLatestCommit(t, env, upstream, defaultName+" <"+defaultEmail+">", "Push partial mirror")
+	missing = runProcess(t, env.with("GIT_NO_LAZY_FETCH", "1"), cachePath, "git", "cat-file", "-e", unrelatedBlob)
+	if missing.exitCode == 0 {
+		t.Fatalf("unrelated blob %s unexpectedly present in partial cache after push", unrelatedBlob)
+	}
+	result = runBraid(t, env, downstream, braid, "pull", "vendor/wanted")
+	assertExit(t, result, 0)
+	assertFile(t, downstream, "vendor/wanted/file.txt", "pushed\n")
+
+	writeFile(t, upstream, "wanted/file.txt", "two\n")
+	second := commitAll(t, env, upstream, "update")
+	result = runBraid(t, env, downstream, braid, "pull", "vendor/wanted")
+	assertExit(t, result, 0)
+	assertFile(t, downstream, "vendor/wanted/file.txt", "two\n")
+	assertConfigRaw(t, downstream, map[string]configMirror{
+		"vendor/wanted": {URL: upstream, Branch: "main", Path: "wanted", Revision: second, PartialClone: true},
+	})
+	assertClean(t, env, downstream)
+}
+
+func TestPartialCloneRejectsUpstreamWithoutFilterSupport(t *testing.T) {
+	root := t.TempDir()
+	env := newProcessEnv(t, root)
+	braid := braidBinary(t)
+	upstream := filepath.Join(root, "upstream")
+	initRepo(t, env, upstream)
+	writeFile(t, upstream, "wanted/file.txt", "content\n")
+	commitAll(t, env, upstream, "initial")
+	downstream := filepath.Join(root, "downstream")
+	initRepo(t, env, downstream)
+	writeFile(t, downstream, "README.md", "downstream\n")
+	commitAll(t, env, downstream, "initial")
+
+	result := runBraid(t, env, downstream, braid, "add", upstream, "vendor/wanted", "--path", "wanted", "--partial-clone")
+	assertExit(t, result, 1)
+	assertContains(t, result.stderr, "does not support partial clone filtering")
+	assertClean(t, env, downstream)
+}

integration/scoped_state_test.go

@@ -113,7 +113,7 @@ func TestExecutableScopedAddRemovePrecheckBlockers(t *testing.T) {
 			},
 			dirty: func(t *testing.T, downstream string) {
 				t.Helper()
-				writeFile(t, downstream, ".braids.json", "{\"config_version\":1,\"mirrors\":{}}\n")
+				writeFile(t, downstream, ".braids.json", "{\"config_version\":2,\"mirrors\":{}}\n")
 			},
 			args:    []string{"add", "$upstream", "vendor/basic"},
 			wantErr: "local changes are present in .braids.json",

integration/support.go

@@ -280,16 +280,17 @@ type configFile struct {
 }
 
 type configMirror struct {
-	URL      string `json:"url"`
-	Branch   string `json:"branch,omitempty"`
-	Path     string `json:"path,omitempty"`
-	Tag      string `json:"tag,omitempty"`
-	Revision string `json:"revision"`
+	URL          string `json:"url"`
+	Branch       string `json:"branch,omitempty"`
+	Path         string `json:"path,omitempty"`
+	Tag          string `json:"tag,omitempty"`
+	Revision     string `json:"revision"`
+	PartialClone bool   `json:"partial_clone,omitempty"`
 }
 
 func expectedConfigRaw(t *testing.T, mirrors map[string]configMirror) string {
 	t.Helper()
-	data, err := json.MarshalIndent(configFile{ConfigVersion: 1, Mirrors: mirrors}, "", "  ")
+	data, err := json.MarshalIndent(configFile{ConfigVersion: 2, Mirrors: mirrors}, "", "  ")
 	if err != nil {
 		t.Fatalf("marshal expected config: %v", err)
 	}
@@ -311,8 +312,8 @@ func assertConfigRaw(t *testing.T, repo string, mirrors map[string]configMirror)
 	if err := json.Unmarshal(data, &parsed); err != nil {
 		t.Fatalf("parse .braids.json: %v", err)
 	}
-	if parsed.ConfigVersion != 1 || len(parsed.Mirrors) != len(mirrors) {
-		t.Fatalf(".braids.json semantic parse = %#v, want version 1 and %d mirrors", parsed, len(mirrors))
+	if parsed.ConfigVersion != 2 || len(parsed.Mirrors) != len(mirrors) {
+		t.Fatalf(".braids.json semantic parse = %#v, want version 2 and %d mirrors", parsed, len(mirrors))
 	}
 }