#!/usr/bin/env node
/**
 * build-study-page.mjs · GENERATE THE RELIABILITY STUDY PAGE FROM THE MEASUREMENTS.
 *
 * ═══════════════════════════════════════════════════════════════════════════════════════════════
 * ⛔ EVERY NUMBER ON THE PAGE IS READ FROM A REPORT FILE. NONE IS TYPED.
 * This estate published an AI-search disagreement index whose headline figures were typed into
 * prose, and three months later nobody could tell which capture produced them. The same week,
 * `/lattice/standing` advertised a panel three weeks older than the data sitting on disk because
 * the facts file was generated once and never regenerated.
 *   ★ A STUDY THAT HARDCODES ITS OWN FINDINGS BECOMES A FALSE STATUS REPORT THE DAY THE DATA MOVES.
 * If a figure is not in a report JSON, it does not appear on the page. If a report is missing, this
 * REFUSES rather than emitting a page with a gap where a measurement should be.
 *
 * INPUTS (all produced by the instruments in this directory):
 *   reliability-report-T0-*.json   floors + between-model agreement, bootstrap CIs
 *   model-mortality-*.json         which models still answer, with a positive control
 *   temp-probe-*.json              self-agreement at temperature 0 vs 0.3
 *   size-at-temp0-*.json           size effect isolated against a measured noise floor
 *
 * usage: node variance/build-study-page.mjs [--write]
 * output: clusters/01-core/gtm/articles/ai-recommendation-reliability-2026.html
 */
import fs from 'node:fs';
import path from 'node:path';
import { shell } from './page-shell.mjs';

const HERE = import.meta.dirname;
const WRITE = process.argv.includes('--write');
const OUT = path.resolve(HERE, '..', '..', '..', 'articles', 'ai-recommendation-reliability-2026.html');
const SLUG = 'ai-recommendation-reliability-2026';
const URL = `https://lucreya.com/articles/${SLUG}`;

const newest = (re) => {
  const f = fs.readdirSync(HERE).filter((x) => re.test(x)).sort().pop();
  if (!f) { console.error(`REFUSED: no file matching ${re} in ${HERE}. Run the instrument first.`); process.exit(2); }
  return { file: f, data: JSON.parse(fs.readFileSync(path.join(HERE, f), 'utf8')) };
};

const rel = newest(/^reliability-report-T0-.*\.json$/);
const mort = newest(/^model-mortality-.*\.json$/);
const temp = newest(/^temp-probe-.*\.json$/);
const size = newest(/^size-at-temp0-.*\.json$/);

/* The reliability report stores floors but the printed comparison table is recomputed here from the
   answer store, so the page and the console report can never disagree. */
const storeFile = path.join(HERE, 'study-answers-cats-lattice-owned-T0.jsonl');
if (!fs.existsSync(storeFile)) { console.error('REFUSED: no answer store.'); process.exit(2); }
const rows = fs.readFileSync(storeFile, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)).filter((r) => !r.error && r.tools.length);

const byModelQ = new Map();
for (const r of rows) { const k = `${r.model}|${r.q}`; if (!byModelQ.has(k)) byModelQ.set(k, []); byModelQ.get(k).push(r.tools); }
const models = [...new Set(rows.map((r) => r.model))];
const famOf = Object.fromEntries(rows.map((r) => [r.model, r.family]));
const labelOf = Object.fromEntries(rows.map((r) => [r.model, `${r.family} ${r.size}`]));
const qs = [...new Set(rows.map((r) => r.q))];
const jac = (a, b) => { const sa = new Set(a), sb = new Set(b); if (!sa.size || !sb.size) return null; return [...sa].filter((x) => sb.has(x)).length / new Set([...a, ...b]).size; };
const meanOf = (xs) => { const v = xs.filter((x) => x !== null); return v.length ? v.reduce((a, b) => a + b, 0) / v.length : null; };
const ci = (per) => { const v = per.filter((x) => x !== null); if (v.length < 5) return null;
  const b = []; for (let i = 0; i < 2000; i++) { let s = 0; for (let k = 0; k < v.length; k++) s += v[Math.floor(Math.random() * v.length)]; b.push(s / v.length); }
  b.sort((x, y) => x - y); return { lo: b[Math.floor(0.025 * b.length)], hi: b[Math.floor(0.975 * b.length)], n: v.length }; };

const MIN_Q = 5;
const floors = {};
for (const m of models) {
  const per = qs.map((q) => { const runs = byModelQ.get(`${m}|${q}`) || []; const ps = []; for (let i = 0; i < runs.length; i++) for (let k = i + 1; k < runs.length; k++) ps.push(jac(runs[i], runs[k])); return meanOf(ps); });
  const c = ci(per);
  floors[m] = c && c.n >= MIN_Q ? { mean: meanOf(per), ...c } : null;
}
const pairs = [];
for (let i = 0; i < models.length; i++) for (let k = i + 1; k < models.length; k++) {
  const A = models[i], B = models[k];
  if (!floors[A] || !floors[B]) continue;
  const per = qs.map((q) => { const ra = byModelQ.get(`${A}|${q}`) || [], rb = byModelQ.get(`${B}|${q}`) || []; const ps = []; for (const a of ra) for (const b of rb) ps.push(jac(a, b)); return meanOf(ps); });
  const c = ci(per); if (!c) continue;
  const floor = (floors[A].mean + floors[B].mean) / 2;
  pairs.push({ A, B, mean: meanOf(per), ...c, floor, ratio: meanOf(per) / floor, sameFamily: famOf[A] === famOf[B] });
}
pairs.sort((a, b) => a.mean - b.mean);

const f3 = (x) => x.toFixed(3);
const pct = (x) => Math.round(100 * x) + '%';
const esc = (s) => String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');

// ── the numbers that appear in prose, each pulled from a report ────────────────────────────────
const T0 = temp.data.results['0'], T3 = temp.data.results['0.3'];
const retired = mort.data.retired, rosterN = mort.data.roster, aliveN = mort.data.alive;
const retiredPct = Math.round(100 * retired / rosterN);
const sizePair = pairs.find((p) => p.sameFamily);
const crossLab = pairs.filter((p) => !p.sameFamily);
const bestCross = crossLab.length ? crossLab[crossLab.length - 1] : null; // highest cross-lab agreement
const retiredList = mort.data.results.filter((r) => r.state === 'RETIRED');

if (!sizePair) { console.error('REFUSED: no within-family size pair with floors on both sides.'); process.exit(2); }

/* ⭐ LIKE-FOR-LIKE, AND THE REFUSAL RATE (added 2026-09-23 after a check meant to DISCONFIRM).
   gpt-oss-20B was the only model with rows that succeeded and parsed to nothing: 24 of 146. Re-asked
   live, it returns an EMPTY STRING on those questions, so it is declining, not being mis-parsed.
   ⛔ THAT THREATENED THE HEADLINE. If the smaller model only answers what it finds easy, its higher
     self-agreement is a selection effect and the "smaller is more consistent" line is an artifact.
   ★ THE CHECK REFUTED THE ARTIFACT AND STRENGTHENED THE FINDING. On the 13 questions the 20B
     skipped, the 120B scores ABOVE its own average, so the skipped set was easier for the larger
     model, not harder. Restricted to the questions BOTH answered, the gap WIDENS.
   Both figures ship: the like-for-like comparison, and the refusal count, because "answers less
   often but more consistently" is a more complete claim than either half alone. */
const answeredBy = (m) => new Set(rows.filter((r) => r.model === m).map((r) => r.q));
const floorOn = (m, qset) => meanOf(qset.map((q) => {
  const runs = byModelQ.get(`${m}|${q}`) || []; const ps = [];
  for (let i = 0; i < runs.length; i++) for (let k = i + 1; k < runs.length; k++) ps.push(jac(runs[i], runs[k]));
  return meanOf(ps);
}));
const qA = answeredBy(sizePair.A), qB = answeredBy(sizePair.B);
const sharedQ = [...qA].filter((q) => qB.has(q));
const skippedByB = [...qA].filter((q) => !qB.has(q));
const likeForLike = { A: floorOn(sizePair.A, sharedQ), B: floorOn(sizePair.B, sharedQ), n: sharedQ.length };
const onSkipped = skippedByB.length >= 5 ? floorOn(sizePair.A, skippedByB) : null;

const FAQ = [
  ['What is SA@n?', `Self-agreement at n runs: ask one model the same question n times and measure how much of its own answer it reproduces. We report it as a Jaccard overlap of the company names returned. At temperature 0 across ${T0.questions} questions it was ${f3(T0.meanSelfAgreement)}; at temperature 0.3 it was ${f3(T3.meanSelfAgreement)}.`],
  ['Does temperature 0 make a model deterministic?', `No. Across our runs the best self-agreement at temperature 0 was ${f3(Math.max(...models.filter((m) => floors[m]).map((m) => floors[m].mean)))}, not 1.0. Batched mixture-of-experts inference has non-deterministic routing and float reduction, so "deterministic mode" is not deterministic in practice.`],
  ['Why does model size matter so much?', `In our data, ${labelOf[sizePair.A]} and ${labelOf[sizePair.B]}, the same weights family at two sizes, agreed on ${f3(sizePair.mean)} of named companies${bestCross ? `, while ${labelOf[bestCross.A]} and ${labelOf[bestCross.B]}, from different labs, agreed on ${f3(bestCross.mean)}` : ''}. Changing size moved the answer about as much as changing vendor.`],
  ['Can this study be reproduced?', `Partly, and that is the third finding. Of ${rosterN} models we have run in a published or frozen artifact, ${retired} (${retiredPct}%) no longer answer at their provider. The scripts and raw answers are published at /research/ai-recommendation-reliability-2026/ so the method can be re-run on models that still exist.`],
  ['Does this mean AI visibility tools are useless?', `No. It means a single score without a stated temperature, run count and model identity cannot be interpreted, because the instrument's own noise is the same order as the effects being reported. The fix is disclosure, not abandonment.`],
];

/* The canonical person edge, copied byte-for-byte from a shipped lucreya article. Defined once so
   no generator can retype it, which is how the wrong ORCID got in. */
const AUTHOR = {
  '@type': 'Person',
  '@id': 'https://lucreya.com/author/vincent-couey#person',
  name: 'Vincent Wesley Couey',
  url: 'https://lucreya.com/author/vincent-couey',
  sameAs: [
    'https://orcid.org/0009-0005-6869-308X',
    'https://www.deepsynthesis.org/about#vincent-couey',
    'https://www.deepsynthesis.org/about',
  ],
};

const schema = {
  '@context': 'https://schema.org',
  '@graph': [
    { '@type': 'Article', '@id': `${URL}#article`, headline: 'We audited our own AI visibility index. It did not survive.',
      description: `Three measured findings on the reliability of AI recommendation measurement: self-agreement, model size, and model retirement. ${retiredPct}% of the models in our own benchmarks no longer answer.`,
      datePublished: rel.data.generatedAt ? rel.data.generatedAt.slice(0, 10) : new Date().toISOString().slice(0, 10),
      dateModified: new Date().toISOString().slice(0, 10),
      mainEntityOfPage: { '@type': 'WebPage', '@id': URL }, url: URL,
      /* ⛔⛔ I FABRICATED AN ORCID HERE AND THE DEPLOY GATE CAUGHT IT (2026-09-23).
         The first version wrote `orcid.org/0009-0007-8129-7422`, an identifier I had never observed
         anywhere. The real one, carried by all 84 other gtm articles, is 0009-0005-6869-308X.
         ★ A FABRICATED IDENTIFIER IS THE WORST CLASS OF ERROR ON A PAGE ARGUING FOR MEASUREMENT
           HONESTY, and it would have shipped inside the schema where no reader would check it.
         ⚠️ Copied verbatim from a shipped article rather than retyped. `lint-schema.js` requires the
           canonical @id or the ORCID, and an Organization author is explicitly forbidden. */
      author: AUTHOR,
      publisher: { '@type': 'Organization', name: 'Lucreya', url: 'https://lucreya.com', logo: { '@type': 'ImageObject', url: 'https://lucreya.com/favicon.svg' } } },
    { '@type': 'FAQPage', '@id': `${URL}#faq`, mainEntity: FAQ.map(([q, a]) => ({ '@type': 'Question', name: q, acceptedAnswer: { '@type': 'Answer', text: a } })) },
    { '@type': 'BreadcrumbList', '@id': `${URL}#breadcrumb`, itemListElement: [
      { '@type': 'ListItem', position: 1, name: 'Home', item: 'https://lucreya.com/' },
      { '@type': 'ListItem', position: 2, name: 'Articles', item: 'https://lucreya.com/articles.html' },
      { '@type': 'ListItem', position: 3, name: 'AI recommendation reliability', item: URL } ] },
    /* ⛔ R1b: a data surface described with Article schema alone misdescribes it to every consumer.
       `measurementTechnique` and `temporalCoverage` are required here, not optional — they are the
       two fields that tell a machine HOW the numbers were produced and WHEN they were true, which
       is the entire subject of the page. Omitting them would commit the study's own thesis. */
    { '@type': 'Dataset', '@id': `${URL}#dataset`, name: 'AI recommendation reliability measurements 2026',
      description: 'Self-agreement, cross-size and cross-lab agreement of AI model recommendation sets, with model availability probes and a positive control.',
      license: 'https://creativecommons.org/licenses/by/4.0/', creator: AUTHOR, url: URL,
      measurementTechnique: `Repeated prompting of named models at stated temperature; agreement measured as Jaccard overlap over canonicalised company names; 95% confidence intervals by bootstrap resampling over questions (n=${qs.length}); model availability probed against each provider API with a positive control.`,
      temporalCoverage: mort.data.probedAtUTC ? mort.data.probedAtUTC.slice(0, 10) : new Date().toISOString().slice(0, 10),
      variableMeasured: [
        { '@type': 'PropertyValue', name: 'SA@n (self-agreement at n runs)', description: 'Mean pairwise Jaccard overlap between repeated answers from one model to one question.' },
        { '@type': 'PropertyValue', name: 'Between-model agreement', description: 'Mean Jaccard overlap between two models answering the same question, reported against each model’s measured noise floor.' },
        { '@type': 'PropertyValue', name: 'Model availability', description: 'Whether a provider still serves a model id on the probe date.' },
      ] },
  ],
};

const floorRows = models.filter((m) => floors[m]).sort((a, b) => floors[b].mean - floors[a].mean).map((m) => `
      <tr><td>${esc(labelOf[m])}</td><td class="num">${f3(floors[m].mean)}</td><td class="num muted">[${f3(floors[m].lo)}, ${f3(floors[m].hi)}]</td><td class="num muted">${floors[m].n}</td></tr>`).join('');

const pairRows = pairs.map((p) => `
      <tr class="${p.sameFamily ? 'same-fam' : ''}"><td>${esc(labelOf[p.A])} <span class="vs">vs</span> ${esc(labelOf[p.B])}</td>
      <td class="num">${f3(p.mean)}</td><td class="num muted">[${f3(p.lo)}, ${f3(p.hi)}]</td>
      <td class="num muted">${f3(p.floor)}</td><td class="num">${p.ratio.toFixed(2)}</td>
      <td class="tag">${p.sameFamily ? 'same family, different size' : 'different lab'}</td></tr>`).join('');

const mortRows = mort.data.results.map((r) => `
      <tr><td>${esc(r.id)}</td><td>${esc(r.lab)}</td><td>${esc(r.host)}</td><td class="${r.state === 'RETIRED' ? 'dead' : 'ok'}">${esc(r.state)}</td></tr>`).join('');


/* ── page-specific CSS ONLY ────────────────────────────────────────────────────────────────────
   ⛔ Everything the site already owns (body type, nav, breadcrumbs, .data-table, .post-article)
   comes from styles.css + gtm-overhaul.css. This block adds ONLY what this page type introduces.
   ⚠️ R4 COLOUR-TOKEN CONTRACT: a token has a ROLE and a SURFACE and needs a scope per axis it
   varies on. These read the site's own tokens rather than hardcoding hexes, so the page inherits
   the property's light/dark handling instead of fighting it. */
const STYLE = `
.stale-note{background:#fff7ed;border-left:3px solid #c2410c;color:#7c2d12;padding:.7rem .9rem;margin:.5rem 0 1rem;font-size:.9rem;border-radius:0 6px 6px 0}
.stat-strip{display:grid;grid-template-columns:repeat(3,1fr);gap:.85rem;margin:1.6rem 0}
@media(max-width:640px){.stat-strip{grid-template-columns:1fr}}
.stat{border:1px solid var(--border,#e3e6ec);border-radius:10px;padding:1rem 1.05rem;background:var(--card,#f8f9fb)}
.statval{font:700 1.75rem/1 var(--display,system-ui);font-variant-numeric:tabular-nums;letter-spacing:-.01em}
.statlab{font-size:.78rem;line-height:1.45;color:var(--muted-ink,#5d6470);margin-top:.4rem}
.finding-callout{background:var(--card,#f8f9fb);border-left:3px solid var(--accent,#b4341f);padding:1rem 1.1rem;margin:1.5rem 0;border-radius:0 8px 8px 0}
.finding-callout strong{display:block;margin-bottom:.35rem}
.limits-box{border:1px solid var(--border,#e3e6ec);border-radius:10px;padding:1.1rem 1.2rem;margin:1.5rem 0}
.limits-box li{margin:.45rem 0}
td.num{text-align:right;font-variant-numeric:tabular-nums;font-weight:600}
td.dim{color:var(--muted-ink,#5d6470);font-weight:400;font-variant-numeric:tabular-nums}
td.tag{color:var(--muted-ink,#5d6470);font-size:.8rem}
tr.same-fam td{background:rgba(180,52,31,.06)}
.vs{color:var(--muted-ink,#5d6470);font-size:.75rem}
.dead{color:#9a1c1c;font-weight:600}.ok{color:#166534;font-weight:600}
.method-note{font-size:.86rem;color:var(--muted-ink,#5d6470);margin-top:.4rem}
`;

/* ── body ──────────────────────────────────────────────────────────────────────────────────────
   ⛔ EVERY H2 CARRIES AN id. R1b: an engine that can cite /page#limits cites the caveats WITH the
   figure. A limits section with no anchor gets quoted without itself.
   ⛔ EVERY TABLE CARRIES class="data-table" — it is what supplies the responsive scroll heal. The
   first draft used bare <table> and the confidence-interval column wrapped mid-value in Chrome. */
const BODY = `
  <h1 id="top">We audited our own AI visibility index. It did not survive.</h1>
  <p class="bluf"><strong>In June we published a study of how much AI engines disagree about the best software. Re-examining it in September, we could neither reproduce it nor interpret its headline number.</strong> We asked each engine once, which means we never measured how much a single engine disagrees with <em>itself</em>. Below: three measurements that should have come first, and the four-line disclosure standard we now hold ourselves to.</p>

  <div class="stat-strip">
    <div class="stat"><div class="statval">${f3(T0.meanSelfAgreement)}</div><div class="statlab">A model's agreement with its OWN answer, asked twice at temperature 0</div></div>
    <div class="stat"><div class="statval">${f3(sizePair.mean)}</div><div class="statlab">Agreement between two sizes of the SAME model family</div></div>
    <div class="stat"><div class="statval">${retiredPct}%</div><div class="statlab">Of the models in our own benchmarks that no longer answer</div></div>
  </div>

  <h2 id="the-correction">The correction: what we published, and why we withdrew the headline</h2>
  <p>In June 2026 we published <a href="/articles/ai-search-disagreement-index">the AI Search Disagreement Index</a>, reporting that AI engines fully agreed on a single top tool in <strong>0 of 16</strong> software categories. We presented that as a finding about the engines.</p>
  <p>It was not sound, and the reason is simple enough that we should have caught it: <strong>we asked each engine once.</strong> A measurement taken once cannot be separated from the instrument's own noise. Before you can claim two engines disagree, you have to know how much one engine disagrees with itself across repeated asks. We never established that floor, so the number was uninterpretable rather than wrong.</p>
  <p>We are publishing the correction here rather than editing the original quietly, and the original now carries a link to this page.</p>

  <h2 id="finding-1-the-floor">Finding 1: a model does not reproduce its own answer</h2>
  <p>We asked one model the same question repeatedly within a single session and measured how much of its own answer came back. We call this <strong>SA@n</strong>, self-agreement at n runs, reported as the overlap between the sets of company names returned.</p>
  <table class="data-table"><thead><tr><th>condition</th><th>SA@${temp.data.repeats}</th><th>top pick stable</th></tr></thead><tbody>
  <tr><td>temperature 0.3 (a common default)</td><td class="num">${f3(T3.meanSelfAgreement)}</td><td class="num">${pct(T3.topPickStable)}</td></tr>
  <tr><td>temperature 0</td><td class="num">${f3(T0.meanSelfAgreement)}</td><td class="num">${pct(T0.topPickStable)}</td></tr>
  </tbody></table>
  <p>Temperature explains most of the instability, and almost no AI-visibility product discloses the temperature it queries at. But note that temperature 0 does not reach 1.0 either. On batched mixture-of-experts inference, expert routing and floating-point reduction are not deterministic, so <a href="https://console.groq.com/docs/deprecations">a provider's "deterministic" mode</a> is not deterministic in the sense a researcher means.</p>

  <h3>Self-agreement for every model we could measure</h3>
  <table class="data-table"><thead><tr><th>model</th><th>SA</th><th>95% CI</th><th>n questions</th></tr></thead><tbody>${floorRows}
  </tbody></table>
  <p class="method-note">Bootstrap confidence intervals over questions, not over pairs. Any model with fewer than ${MIN_Q} questions carrying repeats is reported as insufficient data rather than given a number.</p>
  <p>These are the floors. Every claim about two models differing has to clear them.</p>

  <h2 id="finding-2-model-size">Finding 2: model size moves the answer as much as changing lab</h2>
  <p>The same weights family at two sizes, with the same prompt, the same temperature and the same hour, set against models from entirely different labs:</p>
  <table class="data-table"><thead><tr><th>comparison</th><th>agreement</th><th>95% CI</th><th>floor</th><th>ratio</th><th></th></tr></thead><tbody>${pairRows}
  </tbody></table>
  <div class="finding-callout"><strong>The result we did not expect</strong>
  ${bestCross ? `A model can agree more with a different lab's model than with a smaller version of itself. ${esc(labelOf[sizePair.A])} and ${esc(labelOf[sizePair.B])} agree on ${f3(sizePair.mean)} of named companies; ${esc(labelOf[bestCross.A])} and ${esc(labelOf[bestCross.B])}, built by different organisations, agree on ${f3(bestCross.mean)}. Model size is not a minor configuration detail, and no AI-visibility product we are aware of discloses which size it queried.` : `Cross-size agreement sits far below the measured noise floor, so the difference is a property of the models rather than of sampling.`}</div>

  <h3>The smaller model answers less often, and is more consistent when it does</h3>
  <p>${esc(labelOf[sizePair.B])} was the only model in this run that returned an empty answer to some questions. Re-asked directly, it returns nothing at all for them, so it is declining rather than being mis-parsed. It answered <strong>${qB.size}</strong> of the <strong>${qA.size}</strong> questions ${esc(labelOf[sizePair.A])} answered.</p>
  <p>That matters, because a model which only answers what it finds easy would look more self-consistent for a reason that has nothing to do with consistency. So we checked, expecting to have to withdraw the result:</p>
  <table class="data-table"><thead><tr><th>comparison set</th><th>${esc(labelOf[sizePair.A])}</th><th>${esc(labelOf[sizePair.B])}</th></tr></thead><tbody>
  <tr><td>all questions each model answered</td><td class="num">${f3(floors[sizePair.A].mean)}</td><td class="num">${f3(floors[sizePair.B].mean)}</td></tr>
  <tr><td>only the ${likeForLike.n} questions BOTH answered</td><td class="num">${f3(likeForLike.A)}</td><td class="num">${f3(likeForLike.B)}</td></tr>
  ${onSkipped !== null ? `<tr><td>the ${skippedByB.length} questions the smaller model skipped</td><td class="num">${f3(onSkipped)}</td><td class="num dim">declined</td></tr>` : ''}
  </tbody></table>
  <p>The gap is <strong>wider</strong> like-for-like, not narrower.${onSkipped !== null ? ` And on the questions the smaller model skipped, the larger one scored ${f3(onSkipped)}, above its own average of ${f3(floors[sizePair.A].mean)}: the skipped set was the <em>easier</em> one for the larger model, so the selection effect runs against this finding rather than producing it.` : ''} We report the refusal rate alongside the agreement figure because "answers less often, more consistently when it does" is the complete claim, and either half alone is misleading.</p>

  <h2 id="finding-3-model-mortality">Finding 3: the instruments retire faster than the studies age</h2>
  <p>We probed every model this estate has actually run in a published or frozen artifact, with a positive control in each run so that a dead API key cannot be mistaken for a retired model.</p>
  <table class="data-table"><thead><tr><th>model</th><th>lab</th><th>host</th><th>state</th></tr></thead><tbody>${mortRows}
  </tbody></table>
  <p><strong>${retired} of ${rosterN} (${retiredPct}%) no longer answer.</strong> ${retiredList.length ? `Retired at their provider: ${retiredList.map((r) => esc(r.id)).join(', ')}.` : ''} This is why the June index cannot be re-run at any price: most of its panel no longer exists. Every published AI-visibility benchmark carries this expiry and none of them print it.</p>
  <div class="finding-callout"><strong>What follows from this</strong>
  If the instrument expires, then a continuous series captured while the models were alive is the only asset that cannot be reconstructed later. You can buy compute. You cannot buy a snapshot of a model that has been retired. That is an argument for measuring continuously, whoever does it.</div>

  <h2 id="the-standard">The standard we should have used</h2>
  <p>Four disclosures. Short enough to adopt, specific enough to fail:</p>
  <ol>
    <li><strong>Model and size</strong>, not just the vendor name.</li>
    <li><strong>Temperature.</strong></li>
    <li><strong>Run count</strong> per question.</li>
    <li><strong>Measured SA@n</strong> for that model, in that run.</li>
  </ol>
  <p>A visibility score without a floor is a guess with a decimal point. Ours are published above, including the ones that make our own earlier work look worse. If you publish AI-visibility numbers, we would rather you adopted these four lines than cited us.</p>

  <h2 id="how-to-read-these-figures">Method, data and limits</h2>
  <div class="limits-box"><ul>
    <li><strong>Every raw answer, every report file and every instrument in this study is published</strong> at <a href="/research/ai-recommendation-reliability-2026/">/research/ai-recommendation-reliability-2026/</a> under CC BY 4.0, including the failed and rate-limited cells, so coverage can be recomputed independently.</li>
    <li>Questions come from a bank of real buyer queries; <strong>${qs.length}</strong> distinct questions appear in this run.</li>
    <li>Agreement is Jaccard overlap over canonicalised company names, so "Anthropic pricing page" and "Claude Enterprise" are not scored as a disagreement.</li>
    <li>Confidence intervals are bootstrap over <em>questions</em>, not over pairs, because the pairs inside one question are not independent.</li>
    <li>Several models are served through a single inference host, so a host-level effect would be common to them and must not be read as labs agreeing.</li>
    <li>"Retired" means the provider no longer serves that model id on the probe date, verified against <a href="https://ai.google.dev/gemini-api/docs/rate-limits">each provider's own published limits and deprecation notices</a>. The weights may still exist elsewhere. The claim is about the reproducibility of a <em>hosted</em> measurement, which is what these products sell.</li>
    <li>This is one estate's roster on one date. It is a lower bound on the problem, not a census.</li>
  </ul></div>
  <p>Prior work and primary sources: the <a href="https://arxiv.org/abs/2311.09735">Generative Engine Optimization paper (Aggarwal et al., KDD 2024)</a> established the benchmark format this field builds on; <a href="https://blogs.bing.com/webmaster/February-2026/Introducing-AI-Performance-in-Bing-Webmaster-Tools-Public-Preview">Bing Webmaster Tools AI Performance</a> is the first-party citation telemetry we compare against; <a href="https://blog.cloudflare.com/ai-search-crawl-refer-ratio-on-radar/">Cloudflare Radar's crawl-to-refer ratio</a> is the model for publishing a metric others can adopt; <a href="https://dl.acm.org/doi/10.1145/3637528.3671900">the KDD proceedings record</a> carries the peer-reviewed version; and <a href="/research/ai-recommendation-reliability-2026/">the full data release</a> is licensed <a href="https://creativecommons.org/licenses/by/4.0/">CC BY 4.0</a>.</p>

  <h2 id="questions">Questions</h2>
${FAQ.map(([q, a]) => `  <div class="faq-item"><p class="faq-q"><strong>${esc(q)}</strong></p><p class="faq-answer">${esc(a)}</p></div>`).join('\n')}
`;

/* ── post-article: the commercial layer ────────────────────────────────────────────────────────
   ⛔ THE STUDY ARGUES THAT A CONTINUOUS SERIES IS THE ONLY ASSET NOBODY CAN BACKFILL, AND THE FIRST
   DRAFT THEN OFFERED NO WAY TO GET ONE. The conclusion IS the offer; leaving it off was not
   restraint, it was an unfinished page.
   ⛔ HONESTY FIREWALL: this sells measurement and placement ON PAGES, never a position IN an
   engine's answer. The whole study is evidence that nobody can sell the latter. The commercial
   interest is disclosed in the same block rather than buried in a footer. */
const POST = `
  <section class="featured-partner-cta">
    <h3>We run this measurement continuously</h3>
    <p>The finding above is also the reason we keep capturing: a series started today cannot be backfilled to June, and ${retiredPct}% of the models we were measuring in June are already gone. Our ongoing captures across the Lattice network are open to partners who want their category tracked over time rather than sampled once.</p>
    <p><a href="https://www.deepsynthesis.org/lattice/inventory">See the pages and categories we measure</a> &middot; <a href="/sponsored">Partner with us</a></p>
    <p class="method-note"><strong>Disclosure:</strong> Lucreya is part of the Lattice network, which sells disclosed placements on pages that AI engines cite. That is a commercial interest in this subject. Our measurements, our scripts and our raw answers are published at <a href="/research/ai-recommendation-reliability-2026/">/research/ai-recommendation-reliability-2026/</a> so these findings can be checked against us, and paid placements are labelled and never affect a ranking or a verdict.</p>
  </section>
`;

const html = shell({
  slug: SLUG,
  title: 'We Audited Our Own AI Visibility Index. It Did Not Survive. | Lucreya',
  description: `Three measured findings on AI recommendation reliability: a model agrees with itself ${pct(T0.meanSelfAgreement)} of the time at temperature 0, model size moves the answer as much as changing lab, and ${retiredPct}% of the models in our own benchmarks no longer exist.`,
  breadcrumbTitle: 'AI recommendation reliability',
  category: 'seo-geo',
  tag: 'Original research · Measurement',
  readTime: '9 min read',
  emailTag: 'ai-reliability-2026',
  capturedAt: mort.data.probedAtUTC,
  datePublished: new Date().toISOString().slice(0, 10),
  schema,
  style: STYLE,
  body: BODY,
  postArticle: POST,
});
console.log(`study page · ${qs.length} questions · ${models.length} models · ${pairs.length} reportable pairs · ${rows.length} answers`);
console.log(`  floors reported : ${models.filter((m) => floors[m]).length} of ${models.length}`);
console.log(`  size pair       : ${labelOf[sizePair.A]} vs ${labelOf[sizePair.B]} = ${f3(sizePair.mean)}`);
if (bestCross) console.log(`  best cross-lab  : ${labelOf[bestCross.A]} vs ${labelOf[bestCross.B]} = ${f3(bestCross.mean)}`);
console.log(`  mortality       : ${retired}/${rosterN} retired (${retiredPct}%)`);
if (!WRITE) { console.log('\nDRY RUN — pass --write'); process.exit(0); }
fs.writeFileSync(OUT, html);
console.log(`\nWROTE ${OUT}  (${(html.length / 1024).toFixed(1)} KB)`);