loomctl external: list, add, check — and its first run found two changed documents

Go, standard library only, with git shelled out for list alone. list enumerates a publisher's .loom/published by partial clone and ls-tree; add fetches one document, writes it under .loom/externals and records the resolved origin and the publisher's ETag in .loom/externals/.locks; check asks conditionally and reports. The first real run did what the tool exists for. All eight documents adopted by hand before it existed reported unlocked — the tool refuses to invent a lock by adopting whatever the remote currently serves, since that would assert the local copy is the one being served, which is the thing it was about to check. Locking them fetched two that had moved: bedrock/starting.md, which now says the worked example is private and will not link to something you cannot fetch, and cart.md, which went to v1. cart v1 changes a role we cast: a cart is not committed, because a committed cart grows a third file by itself — version control does not require anybody to ask, so the two-file rule is never invoked — and because ignored, gone means gone. Adds .loom/cart/ to .gitignore and supersedes the isolation entry rather than editing it. osprey and marmalade are already in history and are left there: rewriting to honour a rule adopted afterwards costs more than it buys. Records the conflict this creates rather than settling it: the annotation protocol here says commit before dissolving because git is the only archive, and an ignored cart has no archive, so dissolving would destroy the annotations outright. Credentials are read-only, per host, and passed to git through the environment rather than argv, because argv is visible to every process on the machine. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018UTxuSizozEA8yDitPuris
2026-09-07 14:48:29 -04:00
parent f0b3269610
commit 1543df0a0c
12 changed files with 897 additions and 7 deletions
@@ -0,0 +1,283 @@
+// Package external implements the operations on documents somebody else
+// published that we depend on.
+//
+// Every act is a fetch or a comparison. Nothing here repairs anything: a changed
+// external is a candidate, not a replacement, and somebody decides.
+package external
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"git.hypertheory-labs.dev/loom/loom-cli/internal/config"
+	"git.hypertheory-labs.dev/loom/loom-cli/internal/lock"
+)
+
+var client = &http.Client{Timeout: 30 * time.Second}
+
+// FindRoot walks up from dir looking for the .loom directory that marks a
+// repository using loom.
+func FindRoot(dir string) (string, error) {
+	d, err := filepath.Abs(dir)
+	if err != nil {
+		return "", err
+	}
+	for {
+		if fi, err := os.Stat(filepath.Join(d, ".loom")); err == nil && fi.IsDir() {
+			return d, nil
+		}
+		parent := filepath.Dir(d)
+		if parent == d {
+			return "", errors.New("no .loom directory found in this directory or any parent")
+		}
+		d = parent
+	}
+}
+
+// localPath derives where an adopted document is kept from the URL it came from.
+//
+// The path is for a person: <host>/<owner>/<repo>/<basename>. It deliberately
+// does not encode the route, the branch, or .loom/published/ — which is why it
+// cannot be turned back into a URL, and why the lock records the origin.
+func localPath(raw string) (string, error) {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return "", err
+	}
+	if u.Host == "" || u.Scheme == "" {
+		return "", fmt.Errorf("not an absolute URL: %s", raw)
+	}
+	segs := strings.Split(strings.Trim(u.Path, "/"), "/")
+	if len(segs) < 3 {
+		return "", fmt.Errorf("cannot tell owner and repository from %s — pass --path", raw)
+	}
+	base := segs[len(segs)-1]
+	if base == "" {
+		return "", fmt.Errorf("no file name in %s", raw)
+	}
+	return path.Join(u.Host, segs[0], segs[1], base), nil
+}
+
+// request builds a GET carrying the host's token, if we have one.
+func request(cfg *config.Config, method, raw string, ifNoneMatch string) (*http.Request, error) {
+	req, err := http.NewRequest(method, raw, nil)
+	if err != nil {
+		return nil, err
+	}
+	if t := cfg.TokenFor(req.URL.Host); t != "" {
+		req.Header.Set("Authorization", "token "+t)
+	}
+	if ifNoneMatch != "" {
+		req.Header.Set("If-None-Match", ifNoneMatch)
+	}
+	return req, nil
+}
+
+// Add fetches a document, writes it into .loom/externals/, and records its lock.
+//
+// It does not create a .usages.md: an empty facet asserts that we have something
+// to say and we do not.
+func Add(root, raw, override string, out io.Writer) error {
+	cfg, err := config.Load()
+	if err != nil {
+		return err
+	}
+	rel := override
+	if rel == "" {
+		if rel, err = localPath(raw); err != nil {
+			return err
+		}
+	}
+
+	req, err := request(cfg, http.MethodGet, raw, "")
+	if err != nil {
+		return err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+	case http.StatusNotFound:
+		return fmt.Errorf("404 unresolved: %s was withdrawn, or this credential cannot see it — "+
+			"over HTTP these are the same response", raw)
+	default:
+		return fmt.Errorf("%s: %s", resp.Status, raw)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	etag := resp.Header.Get("ETag")
+
+	dest := filepath.Join(root, lock.Dir, filepath.FromSlash(rel))
+	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
+		return err
+	}
+	if err := os.WriteFile(dest, body, 0o644); err != nil {
+		return err
+	}
+
+	locks, err := lock.Load(root)
+	if err != nil {
+		return err
+	}
+	locks.Put(lock.Record{Path: rel, URL: resp.Request.URL.String(), ETag: etag})
+	if err := locks.Save(); err != nil {
+		return err
+	}
+
+	fmt.Fprintf(out, "adopted  %s\n", rel)
+	fmt.Fprintf(out, "  from   %s\n", resp.Request.URL)
+	if etag == "" {
+		fmt.Fprintf(out, "  etag   (none served — check cannot ask conditionally)\n")
+	} else {
+		fmt.Fprintf(out, "  etag   %s\n", etag)
+	}
+	return nil
+}
+
+// Status is what check found for one document.
+type Status struct {
+	Path   string
+	Result string
+	Detail string
+}
+
+// Check asks every publisher whether their copy has moved.
+//
+// It reports and does nothing else. A changed document is a candidate, not a
+// replacement.
+func Check(root string, out io.Writer) error {
+	cfg, err := config.Load()
+	if err != nil {
+		return err
+	}
+	locks, err := lock.Load(root)
+	if err != nil {
+		return err
+	}
+
+	seen := map[string]bool{}
+	var results []Status
+
+	for _, rec := range locks.All() {
+		seen[rec.Path] = true
+		results = append(results, checkOne(cfg, root, rec))
+	}
+
+	// Documents in the tree with no lock. Never adopt whatever the remote is
+	// currently serving as the lock: that asserts the local copy is the one
+	// being served, which is the thing we were about to check.
+	unlocked, err := unlockedDocs(root, seen)
+	if err != nil {
+		return err
+	}
+	for _, p := range unlocked {
+		results = append(results, Status{Path: p, Result: "unlocked", Detail: "fetched by hand; run `loomctl external add` to lock it"})
+	}
+
+	if len(results) == 0 {
+		fmt.Fprintln(out, "no adopted documents")
+		return nil
+	}
+	w := 0
+	for _, r := range results {
+		if len(r.Path) > w {
+			w = len(r.Path)
+		}
+	}
+	for _, r := range results {
+		fmt.Fprintf(out, "%-*s  %-10s %s\n", w, r.Path, r.Result, r.Detail)
+	}
+	return nil
+}
+
+func checkOne(cfg *config.Config, root string, rec lock.Record) Status {
+	if _, err := os.Stat(filepath.Join(root, lock.Dir, filepath.FromSlash(rec.Path))); errors.Is(err, fs.ErrNotExist) {
+		return Status{rec.Path, "missing", "locked, but the local copy is gone"}
+	}
+	if rec.ETag == "" {
+		return Status{rec.Path, "no-etag", "publisher served none; freshness cannot be asked"}
+	}
+	req, err := request(cfg, http.MethodGet, rec.URL, rec.ETag)
+	if err != nil {
+		return Status{rec.Path, "error", err.Error()}
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return Status{rec.Path, "error", err.Error()}
+	}
+	defer resp.Body.Close()
+	io.Copy(io.Discard, resp.Body)
+
+	switch resp.StatusCode {
+	case http.StatusNotModified:
+		return Status{rec.Path, "same", ""}
+	case http.StatusOK:
+		return Status{rec.Path, "CHANGED", "upstream moved — the new copy is a candidate, not a replacement"}
+	case http.StatusGone:
+		return Status{rec.Path, "GONE", "410 — follow whatever the response points at"}
+	case http.StatusNotFound:
+		return Status{rec.Path, "404", "withdrawn, or access lost — over HTTP these are the same response"}
+	default:
+		return Status{rec.Path, resp.Status, ""}
+	}
+}
+
+// unlockedDocs finds adopted documents that no lock covers. Facets we wrote
+// ourselves are not adopted documents and are skipped.
+func unlockedDocs(root string, locked map[string]bool) ([]string, error) {
+	base := filepath.Join(root, lock.Dir)
+	var out []string
+	err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				return nil
+			}
+			return err
+		}
+		if d.IsDir() || !strings.HasSuffix(d.Name(), ".md") {
+			return nil
+		}
+		rel, err := filepath.Rel(base, p)
+		if err != nil {
+			return err
+		}
+		rel = filepath.ToSlash(rel)
+		if locked[rel] || isFacet(rel) {
+			return nil
+		}
+		out = append(out, rel)
+		return nil
+	})
+	return out, err
+}
+
+// isFacet reports whether a path is something we wrote beside an adopted
+// document rather than the document itself: x.usages.md, x.gaps.md, x.notes.md.
+func isFacet(rel string) bool {
+	base := strings.TrimSuffix(path.Base(rel), ".md")
+	i := strings.LastIndex(base, ".")
+	if i < 0 {
+		return false
+	}
+	switch base[i+1:] {
+	case "usages", "gaps", "notes":
+		return true
+	}
+	return false
+}
@@ -0,0 +1,72 @@
+package external
+
+import "testing"
+
+func TestLocalPath(t *testing.T) {
+	for _, tc := range []struct {
+		name, url, want string
+		wantErr         bool
+	}{
+		{
+			name: "gitea raw, published document",
+			url:  "https://git.hypertheory-labs.dev/loom/bedrock/raw/branch/main/.loom/published/starting.md",
+			want: "git.hypertheory-labs.dev/loom/bedrock/starting.md",
+		},
+		{
+			name: "github raw host",
+			url:  "https://raw.githubusercontent.com/octocat/Hello-World/main/README.md",
+			want: "raw.githubusercontent.com/octocat/Hello-World/README.md",
+		},
+		{
+			name:    "not enough path to name an owner and repository",
+			url:     "https://example.com/thing.md",
+			wantErr: true,
+		},
+		{
+			name:    "not absolute",
+			url:     "/loom/bedrock/starting.md",
+			wantErr: true,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := localPath(tc.url)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("localPath(%q) = %q, want an error", tc.url, got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("localPath(%q): %v", tc.url, err)
+			}
+			if got != tc.want {
+				t.Errorf("localPath(%q) = %q, want %q", tc.url, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestIsFacet(t *testing.T) {
+	// A facet is something we wrote beside an adopted document. check must not
+	// report our own writing as an unlocked external.
+	facets := []string{
+		"host/loom/cart/cart.usages.md",
+		"host/loom/externals/externals.gaps.md",
+		"host/o/r/plan.notes.md",
+	}
+	documents := []string{
+		"host/loom/cart/cart.md",
+		"host/loom/bedrock/recording-decisions.md", // a hyphen is not a facet
+		"host/o/r/starting.md",
+	}
+	for _, p := range facets {
+		if !isFacet(p) {
+			t.Errorf("isFacet(%q) = false, want true", p)
+		}
+	}
+	for _, p := range documents {
+		if isFacet(p) {
+			t.Errorf("isFacet(%q) = true, want false", p)
+		}
+	}
+}
@@ -0,0 +1,107 @@
+package external
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	"git.hypertheory-labs.dev/loom/loom-cli/internal/config"
+)
+
+// PublishedDir is the only directory in somebody else's repository that list
+// looks at. What is not exported is not hidden — it is simply not what you
+// depend on.
+const PublishedDir = ".loom/published"
+
+// List enumerates a publisher's published surface.
+//
+// It shells out to git rather than using a host's REST API, because git is the
+// one interface gitea, GitHub and GitLab all speak the same way: their contents
+// APIs have three different URL shapes, three JSON shapes and three auth
+// schemes, and a private repository refuses the anonymous ones. The cost is that
+// git must be on PATH.
+func List(repoURL string, out io.Writer) error {
+	cfg, err := config.Load()
+	if err != nil {
+		return err
+	}
+
+	dir, err := os.MkdirTemp("", "loomctl-list-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(dir)
+
+	clone := exec.Command("git", "clone",
+		"--filter=blob:none", // trees only: we want names, not contents
+		"--depth=1",          // and bound the damage if the server ignores the filter
+		"--no-checkout",
+		"--quiet",
+		repoURL, dir,
+	)
+	clone.Env = gitEnv(cfg, repoURL)
+	var stderr bytes.Buffer
+	clone.Stderr = &stderr
+	if err := clone.Run(); err != nil {
+		return fmt.Errorf("git clone: %w\n%s", err, strings.TrimSpace(stderr.String()))
+	}
+
+	// git's fallback when a server refuses the filter is silent apart from this
+	// warning, and the fallback is to download everything.
+	if strings.Contains(stderr.String(), "filtering not recognized by server") {
+		fmt.Fprintf(out, "warning: %s ignored --filter, so this fetched every blob at HEAD\n\n", repoURL)
+	}
+
+	ls := exec.Command("git", "-C", dir, "ls-tree", "--name-only", "HEAD:"+PublishedDir)
+	var names, lsErr bytes.Buffer
+	ls.Stdout, ls.Stderr = &names, &lsErr
+	if err := ls.Run(); err != nil {
+		fmt.Fprintf(out, "%s publishes nothing — no %s\n", repoURL, PublishedDir)
+		return nil
+	}
+
+	for _, n := range strings.Split(strings.TrimSpace(names.String()), "\n") {
+		if n != "" {
+			fmt.Fprintln(out, n)
+		}
+	}
+	return nil
+}
+
+// gitEnv passes credentials to git through the environment rather than through
+// -c on the command line, because argv is visible to every process on the
+// machine and an environment is not.
+func gitEnv(cfg *config.Config, repoURL string) []string {
+	env := append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
+	host := hostOf(repoURL)
+	if host == "" {
+		return env
+	}
+	t := cfg.TokenFor(host)
+	if t == "" {
+		return env
+	}
+	return append(env,
+		"GIT_CONFIG_COUNT=1",
+		"GIT_CONFIG_KEY_0=http.extraHeader",
+		"GIT_CONFIG_VALUE_0=Authorization: token "+t,
+	)
+}
+
+func hostOf(raw string) string {
+	i := strings.Index(raw, "://")
+	if i < 0 {
+		return ""
+	}
+	rest := raw[i+3:]
+	if at := strings.Index(rest, "@"); at >= 0 {
+		rest = rest[at+1:]
+	}
+	if s := strings.IndexAny(rest, "/:"); s >= 0 {
+		rest = rest[:s]
+	}
+	return rest
+}