docs/scripts/generate.mjs

#!/usr/bin/env node
// Walk .loom/externals/.locks and project every adopted document into a
// Starlight page.
//
// Two kinds of page, and the difference is the whole design:
//
//   generated  one per adopted document. A rendering of somebody else's
//              document, safe to overwrite because nobody typed it.
//   guide      one index.mdx per section. Hand-written, never overwritten,
//              and stamped with the etags it was written against.
//
// A guide goes stale when a source moves. The build cannot tell whether the
// guide is still true — only a person can — so it renders the question onto
// the page rather than answering it. It reports; a person raises.
//
// The one thing this writes into a hand-written file is the `loom:` block in
// its frontmatter. That block is machine-owned. Everything below the closing
// `---` is yours and is copied byte for byte.

import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'

const root = join(dirname(fileURLToPath(import.meta.url)), '..')
const LOCKS = join(root, '.loom/externals/.locks')
const EXTERNALS = join(root, '.loom/externals')
const OUT = join(root, 'src/content/docs')

const argv = new Set(process.argv.slice(2))
const ACK = argv.has('--ack')
const STRICT = argv.has('--strict')

// ---------------------------------------------------------------- locks

// path <TAB> url <TAB> etag [<TAB> visibility]. Comments start with #.
function readLocks() {
  if (!existsSync(LOCKS)) return []
  return readFileSync(LOCKS, 'utf8')
    .split('\n')
    .filter((l) => l.trim() && !l.startsWith('#'))
    .map((line) => {
      const [path, url, etag, visibility] = line.split('\t')
      // host/owner/repo/name.md
      const parts = path.split('/')
      return {
        path,
        url,
        etag,
        visibility: visibility ?? 'unknown',
        host: parts[0],
        owner: parts[1],
        repo: parts[2],
        name: parts.slice(3).join('/').replace(/\.md$/, ''),
      }
    })
    // Ordered by path so the output is byte-identical across runs. The diff is
    // most of the value; a generated file that churns is a file nobody reads.
    .sort((a, b) => a.path.localeCompare(b.path))
}

// ---------------------------------------------------------------- frontmatter

function splitFrontmatter(text) {
  if (!text.startsWith('---\n')) return { data: {}, body: text }
  const end = text.indexOf('\n---\n', 3)
  if (end === -1) return { data: {}, body: text }
  return {
    data: parseYaml(text.slice(4, end + 1)) ?? {},
    body: text.slice(end + 5),
  }
}

function joinFrontmatter(data, body) {
  return `---\n${stringifyYaml(data, { lineWidth: 0 })}---\n${body}`
}

// The published documents carry no frontmatter, so the title is the first H1.
// It is removed from the body, or Starlight renders the heading twice.
function takeTitle(md, fallback) {
  const lines = md.split('\n')
  const i = lines.findIndex((l) => l.startsWith('# '))
  if (i === -1) return { title: fallback, body: md.trimStart() }
  const title = lines[i].slice(2).trim()
  lines.splice(i, 1)
  return { title, body: lines.join('\n').trimStart() }
}

// ---------------------------------------------------------------- pages

function generatedPage(rec) {
  const src = readFileSync(join(EXTERNALS, rec.path), 'utf8')
  const { title, body } = takeTitle(src, rec.name)

  // What we hold beside the copy. Named, never summarised: extracting a claim
  // from prose eventually extracts it wrong.
  const facets = []
  for (const [suffix, label] of [
    ['.usages.md', 'What of ours depends on it'],
    ['.gaps.md', 'What we expected and did not find'],
  ]) {
    const p = `${rec.path.replace(/\.md$/, '')}${suffix}`
    if (existsSync(join(EXTERNALS, p))) facets.push(`${label}: \`.loom/externals/${p}\``)
  }

  const provenance = [
    '',
    '<div class="loom-provenance">',
    '',
    `This page is a copy of a document published by \`${rec.owner}/${rec.repo}\`, rendered here.`,
    `The source is [${rec.url}](${rec.url}) and is what the copy is checked against.`,
    facets.length ? '\n' + facets.map((f) => `${f}  `).join('\n') : '',
    '',
    '</div>',
    '',
  ].join('\n')

  return joinFrontmatter(
    {
      title,
      // Nobody typed this page, so offering to edit it is an invitation to
      // lose work on the next build.
      editUrl: false,
      loom: {
        generated: true,
        path: rec.path,
        source: rec.url,
        etag: rec.etag,
        visibility: rec.visibility,
      },
    },
    body + '\n' + provenance,
  )
}

const GUIDE_TEMPLATE = (repo) => `
This page is yours. Nothing overwrites it.

Put what a copy cannot carry here — a worked example, the order to read things
in, the thing that only makes sense once you have done it twice.

Prefer an example to an explanation. An explanation is a second saying of a rule
that is owned on one of the pages beside this one, and it goes stale silently. An
example goes stale visibly, because the artifacts in it are the wrong shape.

The pages in this section are copies of what \`loom/${repo}\` publishes, and are
regenerated. When one of them moves, this page gets a banner asking whether it is
still true — the build cannot answer that, so it asks.
`.trimStart()

function guidePage(repo, records, existing) {
  const writtenAgainst = records.map((r) => ({ path: r.path, etag: r.etag }))

  let data = { title: repo, loom: {} }
  let body = GUIDE_TEMPLATE(repo)
  let stale = []

  if (existing) {
    const split = splitFrontmatter(existing)
    data = { ...split.data }
    body = split.body
    const was = new Map((data.loom?.writtenAgainst ?? []).map((s) => [s.path, s.etag]))
    stale = records.filter((r) => was.has(r.path) && was.get(r.path) !== r.etag).map((r) => r.name)
  }

  data.title ??= repo
  data.loom = { writtenAgainst: existing && !ACK ? (data.loom?.writtenAgainst ?? writtenAgainst) : writtenAgainst }

  if (stale.length && !ACK) {
    // Starlight renders `banner` above the page. The reader sees the doubt even
    // if nobody has reconciled it yet — the site degrades honestly rather than
    // reading as true.
    // Deliberately says nothing about which side is newer. An etag is opaque —
    // that is the point of it — so two of them cannot be ordered, and this can
    // only ever know that they differ. The first time it fired in real use, the
    // guide was written against a version newer than the copy here, and a
    // message claiming "an older X" was confidently backwards.
    const which = stale.map((s) => `<code>${s}</code>`).join(', ')
    data.banner = {
      content: `Written against a different ${which} than the copy in this repository — so one of the two is behind. Run <code>loomctl external check</code>, or re-read this page, then <code>npm run ack</code>.`,
    }
  } else {
    delete data.banner
  }

  return { text: joinFrontmatter(data, body), stale }
}

// ---------------------------------------------------------------- run

const locks = readLocks()
if (locks.length === 0) {
  console.error('generate: no adopted documents in .loom/externals/.locks')
  process.exit(1)
}

const sections = new Map()
for (const rec of locks) {
  // A section is a repository, not a host and not a document. `bedrock` with
  // five pages is how a person holds it; nine one-page sections is not.
  if (!sections.has(rec.repo)) sections.set(rec.repo, [])
  sections.get(rec.repo).push(rec)
}

let staleTotal = 0
const written = []

for (const [repo, records] of [...sections].sort(([a], [b]) => a.localeCompare(b))) {
  const dir = join(OUT, repo)
  mkdirSync(dir, { recursive: true })

  const keep = new Set(['index.mdx'])
  for (const rec of records) {
    const file = join(dir, `${rec.name}.md`)
    keep.add(`${rec.name}.md`)
    writeFileSync(file, generatedPage(rec))
    written.push(`${repo}/${rec.name}.md`)
  }

  // A document that is no longer adopted leaves. A page for something this
  // repository no longer depends on is worse than a missing page: it reads as
  // current and nothing will ever correct it.
  for (const f of readdirSync(dir)) {
    if (!keep.has(f)) {
      rmSync(join(dir, f), { recursive: true })
      console.log(`  removed  ${repo}/${f}  (no longer adopted)`)
    }
  }

  const indexPath = join(dir, 'index.mdx')
  const existing = existsSync(indexPath) ? readFileSync(indexPath, 'utf8') : null
  const { text, stale } = guidePage(repo, records, existing)
  writeFileSync(indexPath, text)
  staleTotal += stale.length
  if (stale.length && ACK) console.log(`  acked    ${repo}/index.mdx  now current with ${stale.join(', ')}`)
  else if (stale.length) console.log(`  stale    ${repo}/index.mdx  written against older ${stale.join(', ')}`)
  else if (!existing) console.log(`  created  ${repo}/index.mdx  (blank guide, yours to fill)`)
}

// The sidebar named its sections by hand, which made adopting from a new
// repository a config edit — a second place to update, which is the thing the
// autogenerated directories were already avoiding one level down.
const sidebar = [
  { label: 'Start here', link: '/' },
  ...[...sections.keys()].sort().map((repo) => ({ label: repo, autogenerate: { directory: repo } })),
]
writeFileSync(join(root, 'src/sidebar.json'), JSON.stringify(sidebar, null, 2) + '\n')

console.log(
  `generate: ${written.length} pages from ${sections.size} sections` +
    (ACK ? ', stamps refreshed' : staleTotal ? `, ${staleTotal} stale` : ''),
)

if (STRICT && staleTotal && !ACK) process.exit(1)