loom-cli/internal/external/external.go

// Package external implements the operations on documents somebody else
// published that we depend on.
//
// Every act is a fetch or a comparison. Nothing here repairs anything: a changed
// external is a candidate, not a replacement, and somebody decides.
package external

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"git.hypertheory-labs.dev/loom/loom-cli/internal/config"
	"git.hypertheory-labs.dev/loom/loom-cli/internal/lock"
)

var client = &http.Client{Timeout: 30 * time.Second}

// FindRoot walks up from dir looking for the .loom directory that marks a
// repository using loom.
func FindRoot(dir string) (string, error) {
	d, err := filepath.Abs(dir)
	if err != nil {
		return "", err
	}
	for {
		if fi, err := os.Stat(filepath.Join(d, ".loom")); err == nil && fi.IsDir() {
			return d, nil
		}
		parent := filepath.Dir(d)
		if parent == d {
			return "", errors.New("no .loom directory found in this directory or any parent")
		}
		d = parent
	}
}

// localPath derives where an adopted document is kept from the URL it came from.
//
// The path is for a person: <host>/<owner>/<repo>/<basename>. It deliberately
// does not encode the route, the branch, or .loom/published/ — which is why it
// cannot be turned back into a URL, and why the lock records the origin.
func localPath(raw string) (string, error) {
	u, err := url.Parse(raw)
	if err != nil {
		return "", err
	}
	if u.Host == "" || u.Scheme == "" {
		return "", fmt.Errorf("not an absolute URL: %s", raw)
	}
	segs := strings.Split(strings.Trim(u.Path, "/"), "/")
	if len(segs) < 3 {
		return "", fmt.Errorf("cannot tell owner and repository from %s — pass --path", raw)
	}
	base := segs[len(segs)-1]
	if base == "" {
		return "", fmt.Errorf("no file name in %s", raw)
	}
	return path.Join(u.Host, segs[0], segs[1], base), nil
}

// request builds a GET carrying the host's token, if we have one.
func request(cfg *config.Config, method, raw string, ifNoneMatch string) (*http.Request, error) {
	req, err := http.NewRequest(method, raw, nil)
	if err != nil {
		return nil, err
	}
	if t := cfg.TokenFor(req.URL.Host); t != "" {
		req.Header.Set("Authorization", "token "+t)
	}
	if ifNoneMatch != "" {
		req.Header.Set("If-None-Match", ifNoneMatch)
	}
	return req, nil
}

// Add adopts a document that is not here yet.
//
// It refuses a path that already exists. Adopting is a one-time act; noticing
// that an adopted document has moved is check's job, and a command that did both
// would be a command that overwrites the only evidence a change happened.
func Add(root, raw, override string, out io.Writer) error {
	cfg, err := config.Load()
	if err != nil {
		return err
	}
	rel := override
	if rel == "" {
		if rel, err = localPath(raw); err != nil {
			return err
		}
	}

	locks, err := lock.Load(root)
	if err != nil {
		return err
	}
	dest := filepath.Join(root, lock.Dir, filepath.FromSlash(rel))
	_, onDisk := os.Stat(dest)
	_, isLocked := locks.Get(rel)

	if isLocked {
		return fmt.Errorf("%s is already adopted — `loomctl external check` is what notices it moving", rel)
	}

	body, etag, err := fetch(cfg, raw, "")
	if err != nil {
		return err
	}
	if body == nil {
		return fmt.Errorf("%s: unexpected 304 for a document we do not have", raw)
	}
	vis := sourceVisibility(cfg, body.url)

	// A document that is here but unlocked was fetched by hand before the tool
	// existed. Supplying its URL is the only way it can ever be locked, because
	// the path does not round-trip and nothing else records the origin. It is
	// still not an overwrite: the bytes decide.
	if onDisk == nil {
		old, err := os.ReadFile(dest)
		if err != nil {
			return err
		}
		if !bytes.Equal(old, body.data) {
			if !cartOpen(root) {
				return fmt.Errorf("%s differs from what %s serves, and no cart is open to stage it in — "+
					"the local copy is the only evidence of that and will not be touched", rel, body.url)
			}
			if err := stage(root, rel, body.data, lock.Record{Path: rel, URL: body.url, ETag: etag, Visibility: vis}); err != nil {
				return err
			}
			fmt.Fprintf(out, "staged   %s\n", rel)
			fmt.Fprintf(out, "  from   %s\n", body.url)
			fmt.Fprintf(out, "  NOTE   the local copy differs and was left alone; it is the only evidence\n")
			fmt.Fprintf(out, "         that this moved while nothing was watching. Apply or discard.\n")
			return nil
		}
		// Identical, so the assertion a lock makes — this local copy is the one
		// being served — is verified rather than assumed.
		locks.Put(lock.Record{Path: rel, URL: body.url, ETag: etag, Visibility: vis})
		if err := locks.Save(); err != nil {
			return err
		}
		fmt.Fprintf(out, "locked   %s\n", rel)
		fmt.Fprintf(out, "  from   %s\n", body.url)
		fmt.Fprintf(out, "  etag   %s  (bytes verified identical; nothing was rewritten)\n", etag)
		return nil
	}

	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
		return err
	}
	if err := os.WriteFile(dest, body.data, 0o644); err != nil {
		return err
	}
	locks.Put(lock.Record{Path: rel, URL: body.url, ETag: etag, Visibility: vis})
	if err := locks.Save(); err != nil {
		return err
	}

	fmt.Fprintf(out, "adopted  %s\n", rel)
	fmt.Fprintf(out, "  from   %s\n", body.url)
	warnIfNotPublic(root, body.url, vis, out)
	notePublishedSurface(body.url, out)
	if etag == "" {
		fmt.Fprintf(out, "  etag   (none served — check cannot ask conditionally)\n")
	} else {
		fmt.Fprintf(out, "  etag   %s\n", etag)
	}
	return nil
}

// warnIfNotPublic says so when a document could only be fetched with a
// credential.
//
// Adopting is copying, so a document from a repository somebody may not read
// ends up in a repository they may, and the publisher loses control of it at the
// moment of adoption. The tool can see half of that — whether this fetch needed
// a credential — and cannot see the other half, which is who can read the
// repository the copy is landing in. It reports the half it knows.
// sourceVisibility reports what a document could be read as at the moment it was
// fetched, given that the fetch had just succeeded.
//
// It is free: when no credential was configured the fetch itself was anonymous,
// so the answer is already known; when one was, the extra request is the one the
// warning needed anyway.
//
// It must not be used to re-check a document we are not fetching. "No credential
// configured" says nothing about whether a probe would succeed, and treating it
// as public there would silently clear a real alarm.
func sourceVisibility(cfg *config.Config, raw string) string {
	if u, err := url.Parse(raw); err == nil && cfg.TokenFor(u.Host) == "" {
		return lock.Public // it came back without a credential
	}
	return probeAnonymous(raw)
}

// probeAnonymous asks, with no credential at all, whether a URL can be read.
func probeAnonymous(raw string) string {
	req, err := http.NewRequest(http.MethodHead, raw, nil)
	if err != nil {
		return ""
	}
	resp, err := client.Do(req)
	if err != nil {
		return ""
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusOK {
		return lock.Public
	}
	return lock.NotPublic
}

// warnIfNotPublic says something when adopting would widen who can read a
// document.
//
// Adopting is copying, so a document from a repository somebody may not read
// ends up in a repository they may, and the publisher loses control of it at the
// moment of adoption.
func warnIfNotPublic(root, raw, vis string, out io.Writer) {
	if vis != lock.NotPublic {
		return
	}
	public, known := selfVisibility(root)
	if known && !public {
		fmt.Fprintf(out, "  NOTE   private source, and this repository is not public either.\n")
		fmt.Fprintf(out, "         Access is not widened by that alone — but this check only\n")
		fmt.Fprintf(out, "         tells public from not-public, so it cannot see two repositories\n")
		fmt.Fprintf(out, "         private to different people. That case does widen it.\n")
		return
	}
	if !known {
		fmt.Fprintf(out, "  WARN   this needed a credential, and I cannot tell who may read this\n")
		fmt.Fprintf(out, "         repository — no usable origin. Check before you commit.\n")
		return
	}
	fmt.Fprintf(out, "  WARN   this needed a credential, and THIS repository is public.\n")
	fmt.Fprintf(out, "         Confidentiality does not travel with the copy: adopting this\n")
	fmt.Fprintf(out, "         publishes it to everyone. Do not adopt from a source less\n")
	fmt.Fprintf(out, "         readable than the repository you are adopting into.\n")
	fmt.Fprintf(out, "         Two ways out: ask them to publish it — usually the thing you\n")
	fmt.Fprintf(out, "         needed was not the confidential part — or keep no copy and\n")
	fmt.Fprintf(out, "         record only the dependency, which loomctl cannot do yet.\n")
}

// NotFoundError is a 404, which over HTTP carries more than one reading.
//
// The convention names two — withdrawn, or access lost — because those are the
// two a locked document can have. Adoption by name has a third: a document that
// was never there under that name. Callers that know the repository can tell
// them apart; this type is how they get the chance.
type NotFoundError struct{ URL string }

func (e *NotFoundError) Error() string {
	return fmt.Sprintf("404 unresolved: %s was withdrawn, or this credential cannot see it — "+
		"over HTTP these are the same response", e.URL)
}

type fetched struct {
	data []byte
	url  string
}

// fetch performs one request. A nil body with no error means 304.
func fetch(cfg *config.Config, raw, ifNoneMatch string) (*fetched, string, error) {
	req, err := request(cfg, http.MethodGet, raw, ifNoneMatch)
	if err != nil {
		return nil, "", err
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, "", err
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusNotModified:
		io.Copy(io.Discard, resp.Body)
		return nil, resp.Header.Get("ETag"), nil
	case http.StatusOK:
		b, err := io.ReadAll(resp.Body)
		if err != nil {
			return nil, "", err
		}
		return &fetched{data: b, url: resp.Request.URL.String()}, resp.Header.Get("ETag"), nil
	case http.StatusNotFound:
		return nil, "", &NotFoundError{URL: raw}
	default:
		return nil, "", fmt.Errorf("%s: %s", resp.Status, raw)
	}
}

// Status is what check found for one document.
type Status struct {
	Path   string
	Result string
	Detail string
}

// Check asks every publisher whether their copy has moved, and stages what did.
//
// It never edits an adopted document. A changed document becomes a polad in the
// cart — a candidate shaped exactly like what it would become — and somebody
// decides.
func Check(root string, out io.Writer) error {
	cfg, err := config.Load()
	if err != nil {
		return err
	}
	locks, err := lock.Load(root)
	if err != nil {
		return err
	}
	open := cartOpen(root)

	seen := map[string]bool{}
	var results []Status
	staged := 0

	for _, rec := range locks.All() {
		seen[rec.Path] = true
		st, did := checkLocked(cfg, root, rec, open)
		staged += did
		results = append(results, st)
	}

	unlocked, err := unlockedDocs(root, seen)
	if err != nil {
		return err
	}
	for _, rel := range unlocked {
		st, did := checkUnlocked(cfg, root, rel, locks, open)
		staged += did
		results = append(results, st)
	}
	auditPrinted := &strings.Builder{}
	auditExposure(root, locks, auditPrinted)
	if err := locks.Save(); err != nil {
		return err
	}

	if len(results) == 0 {
		fmt.Fprintln(out, "no adopted documents")
		return nil
	}
	w := 0
	for _, r := range results {
		if len(r.Path) > w {
			w = len(r.Path)
		}
	}
	for _, r := range results {
		fmt.Fprintf(out, "%-*s  %-9s %s\n", w, r.Path, r.Result, r.Detail)
	}
	if staged > 0 {
		fmt.Fprintf(out, "\n%d staged in %s — apply or discard; nothing here drifts into being kept.\n", staged, PoladDir)
	}
	io.WriteString(out, auditPrinted.String())
	return nil
}

// checkLocked asks conditionally. The second return is 1 if a polad was staged.
func checkLocked(cfg *config.Config, root string, rec lock.Record, open bool) (Status, int) {
	if _, err := os.Stat(filepath.Join(root, lock.Dir, filepath.FromSlash(rec.Path))); errors.Is(err, fs.ErrNotExist) {
		return Status{rec.Path, "missing", "locked, but the local copy is gone"}, 0
	}
	if rec.ETag == "" {
		return Status{rec.Path, "no-etag", "publisher served none; freshness cannot be asked"}, 0
	}
	body, etag, err := fetch(cfg, rec.URL, rec.ETag)
	if err != nil {
		return Status{rec.Path, "error", err.Error()}, 0
	}
	if body == nil {
		return Status{rec.Path, "same", ""}, 0
	}
	if !open {
		return Status{rec.Path, "CHANGED", "upstream moved — no cart open, so nothing was staged"}, 0
	}
	// A staged copy is new bytes, so its visibility is evaluated afresh rather
	// than inherited: a source that has gone private since adoption is a new
	// exposure, created now.
	staged := lock.Record{Path: rec.Path, URL: body.url, ETag: etag, Visibility: sourceVisibility(cfg, body.url)}
	if err := stage(root, rec.Path, body.data, staged); err != nil {
		return Status{rec.Path, "error", err.Error()}, 0
	}
	return Status{rec.Path, "CHANGED", "staged as a polad" + usagesNote(root, rec.Path)}, 1
}

// checkUnlocked fetches a document nothing has locked and compares the bytes.
//
// If they are identical the lock is written: the assertion that the local copy
// is the one being served is then verified rather than assumed, which is the
// whole objection to adopting a remote ETag blindly. If they differ, the local
// copy is evidence and is not touched.
func checkUnlocked(cfg *config.Config, root, rel string, locks *lock.Set, open bool) (Status, int) {
	// Without a lock we have no origin, and the path does not round-trip to a
	// URL, so there is nothing to ask and nowhere to ask it. Supplying the URL
	// through add is the only way out.
	return Status{rel, "unlocked", "no origin recorded — `loomctl external add <url>` supplies it without rewriting this copy"}, 0
}

func usagesNote(root, rel string) string {
	u, ok := usagesFor(root, rel)
	if ok {
		return "; " + u + " names what to fix"
	}
	return "; no .usages.md — nothing records what depends on this"
}

// unlockedDocs finds adopted documents that no lock covers. Facets we wrote
// ourselves are not adopted documents and are skipped.
func unlockedDocs(root string, locked map[string]bool) ([]string, error) {
	base := filepath.Join(root, lock.Dir)
	var out []string
	err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			if errors.Is(err, fs.ErrNotExist) {
				return nil
			}
			return err
		}
		// Every file here is an adopted document or something we wrote beside
		// one — except the lock itself, and anything else beginning with a dot.
		if d.IsDir() || strings.HasPrefix(d.Name(), ".") {
			return nil
		}
		rel, err := filepath.Rel(base, p)
		if err != nil {
			return err
		}
		rel = filepath.ToSlash(rel)
		if locked[rel] || isFacet(rel) {
			return nil
		}
		out = append(out, rel)
		return nil
	})
	return out, err
}

// isFacet reports whether a path is something we wrote beside an adopted
// document rather than the document itself: x.usages.md, x.gaps.md, x.notes.md.
func isFacet(rel string) bool {
	base := strings.TrimSuffix(path.Base(rel), ".md")
	i := strings.LastIndex(base, ".")
	if i < 0 {
		return false
	}
	switch base[i+1:] {
	case "usages", "gaps", "notes":
		return true
	}
	return false
}