// Package external implements the operations on documents somebody else // published that we depend on. // // Every act is a fetch or a comparison. Nothing here repairs anything: a changed // external is a candidate, not a replacement, and somebody decides. package external import ( "bytes" "errors" "fmt" "io" "io/fs" "net/http" "net/url" "os" "path" "path/filepath" "strings" "time" "git.hypertheory-labs.dev/loom/loom-cli/internal/config" "git.hypertheory-labs.dev/loom/loom-cli/internal/lock" ) var client = &http.Client{Timeout: 30 * time.Second} // FindRoot walks up from dir looking for the .loom directory that marks a // repository using loom. func FindRoot(dir string) (string, error) { d, err := filepath.Abs(dir) if err != nil { return "", err } for { if fi, err := os.Stat(filepath.Join(d, ".loom")); err == nil && fi.IsDir() { return d, nil } parent := filepath.Dir(d) if parent == d { return "", errors.New("no .loom directory found in this directory or any parent") } d = parent } } // localPath derives where an adopted document is kept from the URL it came from. // // The path is for a person: ///. It deliberately // does not encode the route, the branch, or .loom/published/ — which is why it // cannot be turned back into a URL, and why the lock records the origin. func localPath(raw string) (string, error) { u, err := url.Parse(raw) if err != nil { return "", err } if u.Host == "" || u.Scheme == "" { return "", fmt.Errorf("not an absolute URL: %s", raw) } segs := strings.Split(strings.Trim(u.Path, "/"), "/") if len(segs) < 3 { return "", fmt.Errorf("cannot tell owner and repository from %s — pass --path", raw) } base := segs[len(segs)-1] if base == "" { return "", fmt.Errorf("no file name in %s", raw) } return path.Join(u.Host, segs[0], segs[1], base), nil } // request builds a GET carrying the host's token, if we have one. func request(cfg *config.Config, method, raw string, ifNoneMatch string) (*http.Request, error) { req, err := http.NewRequest(method, raw, nil) if err != nil { return nil, err } if t := cfg.TokenFor(req.URL.Host); t != "" { req.Header.Set("Authorization", "token "+t) } if ifNoneMatch != "" { req.Header.Set("If-None-Match", ifNoneMatch) } return req, nil } // Add adopts a document that is not here yet. // // It refuses a path that already exists. Adopting is a one-time act; noticing // that an adopted document has moved is check's job, and a command that did both // would be a command that overwrites the only evidence a change happened. func Add(root, raw, override string, out io.Writer) error { cfg, err := config.Load() if err != nil { return err } rel := override if rel == "" { if rel, err = localPath(raw); err != nil { return err } } locks, err := lock.Load(root) if err != nil { return err } dest := filepath.Join(root, lock.Dir, filepath.FromSlash(rel)) _, onDisk := os.Stat(dest) _, isLocked := locks.Get(rel) if isLocked { return fmt.Errorf("%s is already adopted — `loomctl external check` is what notices it moving", rel) } body, etag, err := fetch(cfg, raw, "") if err != nil { return err } if body == nil { return fmt.Errorf("%s: unexpected 304 for a document we do not have", raw) } // A document that is here but unlocked was fetched by hand before the tool // existed. Supplying its URL is the only way it can ever be locked, because // the path does not round-trip and nothing else records the origin. It is // still not an overwrite: the bytes decide. if onDisk == nil { old, err := os.ReadFile(dest) if err != nil { return err } if !bytes.Equal(old, body.data) { if !cartOpen(root) { return fmt.Errorf("%s differs from what %s serves, and no cart is open to stage it in — "+ "the local copy is the only evidence of that and will not be touched", rel, body.url) } if err := stage(root, rel, body.data, lock.Record{Path: rel, URL: body.url, ETag: etag}); err != nil { return err } fmt.Fprintf(out, "staged %s\n", rel) fmt.Fprintf(out, " from %s\n", body.url) fmt.Fprintf(out, " NOTE the local copy differs and was left alone; it is the only evidence\n") fmt.Fprintf(out, " that this moved while nothing was watching. Apply or discard.\n") return nil } // Identical, so the assertion a lock makes — this local copy is the one // being served — is verified rather than assumed. locks.Put(lock.Record{Path: rel, URL: body.url, ETag: etag}) if err := locks.Save(); err != nil { return err } fmt.Fprintf(out, "locked %s\n", rel) fmt.Fprintf(out, " from %s\n", body.url) fmt.Fprintf(out, " etag %s (bytes verified identical; nothing was rewritten)\n", etag) return nil } if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { return err } if err := os.WriteFile(dest, body.data, 0o644); err != nil { return err } locks.Put(lock.Record{Path: rel, URL: body.url, ETag: etag}) if err := locks.Save(); err != nil { return err } fmt.Fprintf(out, "adopted %s\n", rel) fmt.Fprintf(out, " from %s\n", body.url) warnIfNotPublic(cfg, body.url, out) if etag == "" { fmt.Fprintf(out, " etag (none served — check cannot ask conditionally)\n") } else { fmt.Fprintf(out, " etag %s\n", etag) } return nil } // warnIfNotPublic says so when a document could only be fetched with a // credential. // // Adopting is copying, so a document from a repository somebody may not read // ends up in a repository they may, and the publisher loses control of it at the // moment of adoption. The tool can see half of that — whether this fetch needed // a credential — and cannot see the other half, which is who can read the // repository the copy is landing in. It reports the half it knows. func warnIfNotPublic(cfg *config.Config, raw string, out io.Writer) { req, err := http.NewRequest(http.MethodHead, raw, nil) if err != nil { return } if cfg.TokenFor(req.URL.Host) == "" { return // no credential was used, so the fetch was already anonymous } resp, err := client.Do(req) if err != nil { return } defer resp.Body.Close() if resp.StatusCode == http.StatusOK { return } fmt.Fprintf(out, " WARN this needed a credential — anonymously it is %s.\n", resp.Status) fmt.Fprintf(out, " Confidentiality does not travel with the copy: this now lives in\n") fmt.Fprintf(out, " .loom/externals/ and is readable by anyone who can read THIS\n") fmt.Fprintf(out, " repository, which I cannot see. Do not adopt from a source less\n") fmt.Fprintf(out, " readable than the repository you are adopting into.\n") fmt.Fprintf(out, " Two ways out: ask them to publish it — usually the thing you\n") fmt.Fprintf(out, " needed was not the confidential part — or keep no copy and\n") fmt.Fprintf(out, " record only the dependency, which loomctl cannot do yet.\n") } type fetched struct { data []byte url string } // fetch performs one request. A nil body with no error means 304. func fetch(cfg *config.Config, raw, ifNoneMatch string) (*fetched, string, error) { req, err := request(cfg, http.MethodGet, raw, ifNoneMatch) if err != nil { return nil, "", err } resp, err := client.Do(req) if err != nil { return nil, "", err } defer resp.Body.Close() switch resp.StatusCode { case http.StatusNotModified: io.Copy(io.Discard, resp.Body) return nil, resp.Header.Get("ETag"), nil case http.StatusOK: b, err := io.ReadAll(resp.Body) if err != nil { return nil, "", err } return &fetched{data: b, url: resp.Request.URL.String()}, resp.Header.Get("ETag"), nil case http.StatusNotFound: return nil, "", fmt.Errorf("404 unresolved: %s was withdrawn, or this credential cannot see it — "+ "over HTTP these are the same response", raw) default: return nil, "", fmt.Errorf("%s: %s", resp.Status, raw) } } // Status is what check found for one document. type Status struct { Path string Result string Detail string } // Check asks every publisher whether their copy has moved, and stages what did. // // It never edits an adopted document. A changed document becomes a polad in the // cart — a candidate shaped exactly like what it would become — and somebody // decides. func Check(root string, out io.Writer) error { cfg, err := config.Load() if err != nil { return err } locks, err := lock.Load(root) if err != nil { return err } open := cartOpen(root) seen := map[string]bool{} var results []Status staged := 0 for _, rec := range locks.All() { seen[rec.Path] = true st, did := checkLocked(cfg, root, rec, open) staged += did results = append(results, st) } unlocked, err := unlockedDocs(root, seen) if err != nil { return err } for _, rel := range unlocked { st, did := checkUnlocked(cfg, root, rel, locks, open) staged += did results = append(results, st) } if err := locks.Save(); err != nil { return err } if len(results) == 0 { fmt.Fprintln(out, "no adopted documents") return nil } w := 0 for _, r := range results { if len(r.Path) > w { w = len(r.Path) } } for _, r := range results { fmt.Fprintf(out, "%-*s %-9s %s\n", w, r.Path, r.Result, r.Detail) } if staged > 0 { fmt.Fprintf(out, "\n%d staged in %s — apply or discard; nothing here drifts into being kept.\n", staged, PoladDir) } return nil } // checkLocked asks conditionally. The second return is 1 if a polad was staged. func checkLocked(cfg *config.Config, root string, rec lock.Record, open bool) (Status, int) { if _, err := os.Stat(filepath.Join(root, lock.Dir, filepath.FromSlash(rec.Path))); errors.Is(err, fs.ErrNotExist) { return Status{rec.Path, "missing", "locked, but the local copy is gone"}, 0 } if rec.ETag == "" { return Status{rec.Path, "no-etag", "publisher served none; freshness cannot be asked"}, 0 } body, etag, err := fetch(cfg, rec.URL, rec.ETag) if err != nil { return Status{rec.Path, "error", err.Error()}, 0 } if body == nil { return Status{rec.Path, "same", ""}, 0 } if !open { return Status{rec.Path, "CHANGED", "upstream moved — no cart open, so nothing was staged"}, 0 } if err := stage(root, rec.Path, body.data, lock.Record{Path: rec.Path, URL: body.url, ETag: etag}); err != nil { return Status{rec.Path, "error", err.Error()}, 0 } return Status{rec.Path, "CHANGED", "staged as a polad" + usagesNote(root, rec.Path)}, 1 } // checkUnlocked fetches a document nothing has locked and compares the bytes. // // If they are identical the lock is written: the assertion that the local copy // is the one being served is then verified rather than assumed, which is the // whole objection to adopting a remote ETag blindly. If they differ, the local // copy is evidence and is not touched. func checkUnlocked(cfg *config.Config, root, rel string, locks *lock.Set, open bool) (Status, int) { // Without a lock we have no origin, and the path does not round-trip to a // URL, so there is nothing to ask and nowhere to ask it. Supplying the URL // through add is the only way out. return Status{rel, "unlocked", "no origin recorded — `loomctl external add ` supplies it without rewriting this copy"}, 0 } func usagesNote(root, rel string) string { u, ok := usagesFor(root, rel) if ok { return "; " + u + " names what to fix" } return "; no .usages.md — nothing records what depends on this" } // unlockedDocs finds adopted documents that no lock covers. Facets we wrote // ourselves are not adopted documents and are skipped. func unlockedDocs(root string, locked map[string]bool) ([]string, error) { base := filepath.Join(root, lock.Dir) var out []string err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error { if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil } return err } if d.IsDir() || !strings.HasSuffix(d.Name(), ".md") { return nil } rel, err := filepath.Rel(base, p) if err != nil { return err } rel = filepath.ToSlash(rel) if locked[rel] || isFacet(rel) { return nil } out = append(out, rel) return nil }) return out, err } // isFacet reports whether a path is something we wrote beside an adopted // document rather than the document itself: x.usages.md, x.gaps.md, x.notes.md. func isFacet(rel string) bool { base := strings.TrimSuffix(path.Base(rel), ".md") i := strings.LastIndex(base, ".") if i < 0 { return false } switch base[i+1:] { case "usages", "gaps", "notes": return true } return false }