#!/usr/bin/env node
/**
 * reliability-study.mjs · THE HARDENED INSTRUMENT FOR THE RECOMMENDATION RELIABILITY STUDY.
 *
 * ═══════════════════════════════════════════════════════════════════════════════════════════════
 * THE QUESTION. Every AI-visibility vendor reports a single score for "does the AI recommend you".
 * Almost none disclose model version, model SIZE, temperature, or how many times they asked.
 * This measures how much those undisclosed choices move the answer.
 *
 * WHAT THE PILOT ALREADY SHOWED (2026-09-23, gpt-oss on Groq, small n):
 *   · temperature 0.3 -> a model agrees with ITSELF on 0.293 of named companies; at 0 it is 0.752
 *   · at temperature 0, 120B vs 20B agree on 0.235, against a measured noise floor of 0.74-0.93
 *   · the SMALLER model was more self-consistent than the larger one, which nobody predicted
 * This file exists to turn those into numbers with confidence intervals, across more than one
 * model family, because a pilot with n=20 and two runs is an observation, not a result.
 *
 * ⛔ DESIGN RULES, EACH ONE PAID FOR EARLIER TODAY:
 *   1. RESUMABLE. Every answer is written the moment it arrives. A 600-call study that loses
 *      everything to a quota trip or an OOM kill is a study that never finishes. Re-running skips
 *      what exists.
 *   2. THE NOISE FLOOR IS MEASURED, NEVER ASSUMED. Cross-model numbers are meaningless without the
 *      within-model baseline from the SAME run, because comparing two models at a temperature where
 *      neither repeats itself cannot separate "they differ" from "neither is stable".
 *   3. CONFIDENCE INTERVALS BY BOOTSTRAP over QUESTIONS, not over pairs. Pairs within a question are
 *      not independent, and treating them as such would produce a confidently narrow interval.
 *   4. A FAMILY IS DECLARED, NOT INFERRED. Two sizes of gpt-oss are the same family and must never
 *      be counted as independent engines agreeing.
 *
 * usage:
 *   node variance/reliability-study.mjs --plan                      # what it would do, no calls
 *   node variance/reliability-study.mjs --run --questions 40 --repeats 5
 *   node variance/reliability-study.mjs --report                    # stats from what is on disk
 */
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { createRequire } from 'node:module';

const require = createRequire(import.meta.url);
const { canon, isJunk } = require('../score-audit.js');
const HERE = import.meta.dirname;
const arg = (k, d) => { const i = process.argv.indexOf(`--${k}`); return i > -1 ? process.argv[i + 1] : d; };
const has = (k) => process.argv.includes(`--${k}`);

const NQ = Number(arg('questions', 40));
const REPEATS = Number(arg('repeats', 5));
const TEMP = Number(arg('temp', 0));
/* ⭐ --bank lets the SAME instrument re-measure a DIFFERENT published study.
   Built 2026-09-23 to audit our own `ai-search-disagreement-index`, which reports 0% eight-way
   agreement and 163-of-354 single-engine tools while disclosing no temperature, no run count and no
   self-agreement floor. With a one-shot capture those figures cannot separate "the engines disagree"
   from "no engine reproduces itself", so they are uninterpretable rather than wrong.
   ⛔ THE STORE IS KEYED BY BANK AS WELL AS TEMPERATURE. Two studies sharing one answer file would
     silently pool questions from different banks into one "self-agreement" figure, which is the
     same class of error as pooling two engine panels into one series. */
const BANK_PATH = arg('bank', path.join(HERE, '..', 'cats-lattice-owned.json'));
const BANK_TAG = path.basename(BANK_PATH).replace(/\.json$/, '');
const STORE = path.join(HERE, `study-answers-${BANK_TAG}-T${TEMP}.jsonl`);

/* Prefers the plural pool file (one key per line) and falls back to the singular, mirroring
   collect.js. ⛔ THE POOL IS ONLY REAL IF THE KEYS SIT IN DIFFERENT PROJECTS: Gemini free quota is
   per-PROJECT-per-model, so two keys in one project share one bucket and rotating between them buys
   nothing. Measured the hard way today. */
const keysFor = (n, override) => {
  // ⚠️ Not every credential follows the `.<name>-api-key` convention: the HuggingFace one predates
  //    it and lives at `~/.hf-token`. A row may name its own file rather than force a rename.
  if (override) { const p = path.join(os.homedir(), `.${override}`); return fs.existsSync(p) ? [fs.readFileSync(p, 'utf8').trim()].filter(Boolean) : []; }
  const plural = path.join(os.homedir(), `.${n}-api-keys`);
  const single = path.join(os.homedir(), `.${n}-api-key`);
  const out = [];
  if (fs.existsSync(plural)) out.push(...fs.readFileSync(plural, 'utf8').split('\n').map((s) => s.trim()).filter(Boolean));
  if (fs.existsSync(single)) { const k = fs.readFileSync(single, 'utf8').trim(); if (k && !out.includes(k)) out.push(k); }
  return out;
};
const keyFile = (n, o) => keysFor(n, o)[0] || null;

/* ⭐ THE PANEL. `family` is what makes a comparison cross-LAB rather than cross-SIZE, and it is
   declared here by hand precisely so nobody can later count gpt-oss-120b and gpt-oss-20b as two
   independent engines agreeing with each other. Adding a provider is one row plus a key file. */
const MODELS = [
  { id: 'gptoss-120b', family: 'openai-gpt-oss', size: '120B', need: 'groq',
    url: 'https://api.groq.com/openai/v1/chat/completions', model: 'openai/gpt-oss-120b' },
  { id: 'gptoss-20b', family: 'openai-gpt-oss', size: '20B', need: 'groq',
    url: 'https://api.groq.com/openai/v1/chat/completions', model: 'openai/gpt-oss-20b' },
  /* ⭐⭐ THE SECOND FAMILY WAS ALREADY ON THIS MACHINE. Google, not OpenAI. It was written off
     earlier today because the free tier is GenerateRequestsPerDayPerProjectPerModel = 20, and a
     61-query bank cannot complete in a day.
     ★ THAT IS ONLY FATAL TO A RUN THAT MUST FINISH IN ONE SITTING. This harness is resumable by
       design, and the pool holds 3 keys across 2 projects, so ~40 calls/day/model accumulate a full
       arm over several days. The quota stops being a wall and becomes a schedule.
     ⚠️ flash and flash-lite are separate models with SEPARATE daily quotas, so running both is not
       double-spending one budget. They are also different sizes of one family, which gives a second
       independent size comparison in a different lab — exactly the axis the gpt-oss pair measures. */
  { id: 'gemini-flash', family: 'google-gemini', size: 'flash', need: 'gemini', shape: 'gemini',
    url: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent', model: 'gemini-2.5-flash' },
  { id: 'gemini-flash-lite', family: 'google-gemini', size: 'flash-lite', need: 'gemini', shape: 'gemini',
    url: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite:generateContent', model: 'gemini-2.5-flash-lite' },
  /* ⭐⭐⭐ FOUR MORE LABS, ON A TOKEN THAT WAS ALREADY ON THIS MACHINE (2026-09-23).
     OpenRouter's sign-in was down and Cerebras would have meant a new account. Neither was needed:
     `~/.hf-token` already existed, and the HuggingFace router serves an OpenAI-compatible endpoint
     across several vendors' weights. Probed live before being written down here — Mistral-Small and
     gemma-2-27b are NOT served by the router and are therefore absent rather than listed hopefully.
     ★ THE OBSTACLE WAS A MISSING CREDENTIAL. THE CREDENTIAL WAS ALREADY ON DISK. Before signing up
       for anything, inventory what is already authenticated: this estate had six model families
       available and was running the study on one.
     ⚠️ Host and lab are different things. These all run on HuggingFace infrastructure, so a
       host-level effect would be common to all four and must not be read as lab agreement. The
       gpt-oss pair (Groq) and the Gemini pair (Google) provide the off-host comparison. */
  { id: 'hf-llama-70b', family: 'meta-llama', size: '70B', need: 'hf', keyfile: 'hf-token',
    url: 'https://router.huggingface.co/v1/chat/completions', model: 'meta-llama/Llama-3.3-70B-Instruct' },
  { id: 'hf-qwen-72b', family: 'alibaba-qwen', size: '72B', need: 'hf', keyfile: 'hf-token',
    url: 'https://router.huggingface.co/v1/chat/completions', model: 'Qwen/Qwen2.5-72B-Instruct' },
  { id: 'hf-deepseek-v3', family: 'deepseek', size: 'V3', need: 'hf', keyfile: 'hf-token',
    url: 'https://router.huggingface.co/v1/chat/completions', model: 'deepseek-ai/DeepSeek-V3-0324' },
  { id: 'hf-phi-4', family: 'microsoft-phi', size: '14B', need: 'hf', keyfile: 'hf-token',
    url: 'https://router.huggingface.co/v1/chat/completions', model: 'microsoft/phi-4' },
];

const bank = JSON.parse(fs.readFileSync(BANK_PATH, 'utf8'));
const questions = bank.flatMap(([cat, qs]) => qs.map((q) => ({ q, cat }))).slice(0, NQ);

const PROMPT = (q) => `A user asks: "${q}". List the specific products, services or companies you would recommend, best first. Reply with ONLY a line starting "TOOLS:" followed by a comma-separated list of names. No commentary.`;
const parse = (t) => { const m = /TOOLS:\s*(.+)/i.exec(t || ''); return m ? [...new Set(m[1].split(',').map((s) => canon(s.trim())).filter((s) => s && !isJunk(s)))] : []; };

const live = MODELS.filter((m) => keyFile(m.need, m.keyfile));
const dormant = MODELS.filter((m) => !keyFile(m.need, m.keyfile));
const families = [...new Set(live.map((m) => m.family))];

if (has('plan') || (!has('run') && !has('report'))) {
  console.log(`PLAN · ${questions.length} questions x ${REPEATS} repeats at temperature ${TEMP}`);
  console.log(`live models   : ${live.map((m) => m.id + ' [' + m.family + '/' + m.size + ']').join(', ') || '(none)'}`);
  console.log(`dormant       : ${dormant.map((m) => m.id + ' (needs ~/.' + m.need + '-api-key)').join(', ') || '(none)'}`);
  console.log(`model families: ${families.length} -> ${families.join(', ')}`);
  console.log(`total calls   : ${questions.length * REPEATS * live.length}`);
  if (families.length < 2) console.log('\n⚠️ ONE FAMILY ONLY. Size effects are measurable; CROSS-LAB claims are not. Add a key to light a second family.');
  if (!has('plan')) console.log('\n(dry run — pass --run to execute, --report to analyse what is on disk)');
  if (!has('report')) process.exit(0);
}

// ── capture, resumable ─────────────────────────────────────────────────────────────────────────
/* ⛔⛔ A FAILED CELL IS NOT A DONE CELL. CAUGHT MID-RUN 2026-09-23.
   The first version keyed `done` on model|question|repeat for EVERY row, including rows written
   with `error: "retries exhausted"` while Groq was rate-limiting the 20B. Two compounding effects:
     · resume would skip those cells forever, so a transient rate limit became permanent data loss;
     · the analysis filters `!r.error`, so they vanish from n WITHOUT the report saying n shrank.
   ★ THE SILENT VERSION IS THE DANGEROUS ONE. A study that quietly drops its hardest cells reports
     a cleaner result than it earned, and nothing in the output admits it.
   Only successful, non-empty answers count as done. Error rows stay on disk as a record of what the
   provider did, and are retried on the next run. */
const done = new Set();
let priorErrors = 0;
if (fs.existsSync(STORE)) for (const line of fs.readFileSync(STORE, 'utf8').split('\n')) {
  if (!line.trim()) continue;
  try {
    const r = JSON.parse(line);
    if (r.error || !(r.tools || []).length) { priorErrors++; continue; }
    done.add(`${r.model}|${r.q}|${r.rep}`);
  } catch {}
}
if (priorErrors) console.log(`  ${priorErrors} previously-failed/empty cell(s) on disk — they will be retried, not skipped`);

/* ⭐ ROTATE THE POOL BEFORE PAYING ANY BACKOFF. Another project's daily quota is free and instant;
   a 20s sleep buys nothing when the exhausted thing is the project, not the minute. Once every key
   has been tried and refused, the day's budget for that model is genuinely gone: return
   `exhausted`, which the caller records and moves on rather than stalling the whole study. */
async function ask(m, q) {
  const pool = keysFor(m.need, m.keyfile);
  if (!pool.length) return { fatal: 'no key' };
  for (let ki = 0; ki < pool.length; ki++) {
    const key = pool[ki];
    for (let a = 0; a < 2; a++) {
      try {
        const isGemini = m.shape === 'gemini';
        const r = await fetch(isGemini ? `${m.url}?key=${key}` : m.url, {
          method: 'POST',
          headers: isGemini ? { 'Content-Type': 'application/json' } : { 'Content-Type': 'application/json', Authorization: 'Bearer ' + key },
          body: JSON.stringify(isGemini
            ? { contents: [{ parts: [{ text: PROMPT(q) }] }], generationConfig: { temperature: TEMP } }
            : { model: m.model, messages: [{ role: 'user', content: PROMPT(q) }], temperature: TEMP }),
        });
        const j = await r.json();
        if (j.error) {
          const s = JSON.stringify(j.error);
          /* ⛔ A MONTHLY CREDIT BUDGET IS NOT A RATE LIMIT AND MUST NOT BE RETRIED. Measured
             2026-09-23: HuggingFace returned "You have depleted your monthly included credits"
             **653 times** in one run, because the message matched no permanent pattern and every
             cell burned its full retry ladder against a wall that does not move until next month.
             ★ RETRY POLICY MUST MATCH THE RESET HORIZON: seconds for a per-minute limit, next day
               for a daily quota, next MONTH for a credit budget. Treating all three as "try again"
               is how a study spends its wall-clock proving something it already knew. */
          if (/depleted|monthly included credits|insufficient credit|payment required|402/i.test(s)) return { fatal: s.slice(0, 120) };
          if (/model_not_found|does not exist|invalid_api_key|API_KEY_INVALID|unauthorized|401|403|404/i.test(s)) return { fatal: s.slice(0, 120) };
          /* ⚠️ A PER-MINUTE RATE LIMIT AND AN EXHAUSTED DAILY QUOTA NEED OPPOSITE RESPONSES, AND THE
             FIRST VERSION TREATED THEM THE SAME. Groq throttles per minute: waiting ~8s clears it.
             Gemini's free tier is per DAY: waiting is useless and rotating keys is the only move.
             Conflating them cost ~60s per cell against a limit that would have cleared in eight. */
          const perMinute = /rate_limit|requests per minute|RPM|too many requests/i.test(s) || (pool.length === 1 && /429/.test(s));
          if (perMinute && a === 0) { await new Promise((z) => setTimeout(z, 9000)); continue; }
          if (/rate|quota|429|RESOURCE_EXHAUSTED/i.test(s)) break;  // this key is spent: next key, no backoff
          return { error: s.slice(0, 120) };
        }
        const text = isGemini
          ? (j.candidates?.[0]?.content?.parts || []).map((p) => p.text || '').join(' ')
          : j.choices?.[0]?.message?.content;
        return { tools: parse(text) };
      } catch (e) { if (a === 1) break; await new Promise((z) => setTimeout(z, 3000)); }
    }
  }
  return { exhausted: true };
}

if (has('run')) {
  if (!live.length) { console.error('REFUSED: no model has a key.'); process.exit(2); }
  const fh = fs.openSync(STORE, 'a');
  let calls = 0, skipped = 0;
  for (const m of live) {
    let fatal = null;
    for (const { q, cat } of questions) {
      for (let rep = 0; rep < REPEATS; rep++) {
        if (fatal) break;
        if (done.has(`${m.id}|${q}|${rep}`)) { skipped++; continue; }
        const res = await ask(m, q);
        if (res.fatal) { console.error(`\n  ! ${m.id} fatal: ${res.fatal} — dropping this model for the run`); fatal = res.fatal; break; }
        /* ⛔ AN EXHAUSTED DAILY QUOTA IS NOT A FAILED STUDY, IT IS AN UNFINISHED ONE, AND THE
           DIFFERENCE HAS TO SURVIVE INTO TOMORROW. Nothing is written for this cell, so the next
           run picks it up exactly here instead of recording a false empty answer that would then be
           analysed as "the model named nothing". */
        if (res.exhausted) { console.error(`\n  ! ${m.id}: every key's daily quota is spent. ${calls} captured; re-run tomorrow to continue.`); fatal = 'quota-exhausted'; break; }
        fs.writeSync(fh, JSON.stringify({ model: m.id, family: m.family, size: m.size, q, cat, rep, temp: TEMP, tools: res.tools || [], error: res.error || null, at: new Date().toISOString() }) + '\n');
        calls++; if (calls % 25 === 0) process.stdout.write(`\r  ${calls} calls`);
        await new Promise((z) => setTimeout(z, 700));
      }
      if (fatal) break;
    }
  }
  fs.closeSync(fh);
  console.log(`\ncaptured ${calls} new answers (${skipped} already on disk) -> ${path.basename(STORE)}`);
}

// ── analysis ───────────────────────────────────────────────────────────────────────────────────
if (has('report') || has('run')) {
  if (!fs.existsSync(STORE)) { console.error('no answers on disk yet'); process.exit(2); }
  const rows = fs.readFileSync(STORE, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)).filter((r) => !r.error && r.tools.length);
  const byModelQ = new Map();
  for (const r of rows) { const k = `${r.model}|${r.q}`; if (!byModelQ.has(k)) byModelQ.set(k, []); byModelQ.get(k).push(r.tools); }
  const models = [...new Set(rows.map((r) => r.model))];
  const famOf = Object.fromEntries(rows.map((r) => [r.model, r.family]));
  const qs = [...new Set(rows.map((r) => r.q))];
  const jac = (a, b) => { const sa = new Set(a), sb = new Set(b); if (!sa.size || !sb.size) return null; return [...sa].filter((x) => sb.has(x)).length / new Set([...a, ...b]).size; };
  const meanOf = (xs) => { const v = xs.filter((x) => x !== null && !Number.isNaN(x)); return v.length ? v.reduce((a, b) => a + b, 0) / v.length : null; };

  /* ⛔ BOOTSTRAP OVER QUESTIONS. Resampling PAIRS would treat the 10 pairs inside one question as 10
     independent observations and report an interval far narrower than the evidence supports. */
  const ci = (perQuestion) => {
    const v = perQuestion.filter((x) => x !== null);
    if (v.length < 3) return null;
    const boots = [];
    for (let b = 0; b < 2000; b++) {
      let s = 0; for (let i = 0; i < v.length; i++) s += v[Math.floor(Math.random() * v.length)];
      boots.push(s / v.length);
    }
    boots.sort((a, b) => a - b);
    return { lo: boots[Math.floor(0.025 * boots.length)], hi: boots[Math.floor(0.975 * boots.length)], n: v.length };
  };

  console.log(`\nRELIABILITY STUDY · temperature ${TEMP} · ${qs.length} questions · ${rows.length} answers\n`);

  /* ⛔ COVERAGE BEFORE CONCLUSIONS. The analysis drops error and empty rows, so without this a model
     that answered 30% of its cells is reported with the same confidence as one that answered all of
     them, and the only visible difference is a slightly wider interval. State what is MISSING. */
  const allRows = fs.readFileSync(STORE, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
  console.log('COVERAGE (usable answers vs attempted)');
  for (const m of [...new Set(allRows.map((r) => r.model))]) {
    const att = allRows.filter((r) => r.model === m).length;
    const ok = allRows.filter((r) => r.model === m && !r.error && r.tools.length).length;
    const planned = qs.length * REPEATS;
    const flag = ok < planned * 0.8 ? '  ⚠️ BELOW 80% OF PLAN' : '';
    console.log(`  ${m.padEnd(22)} ${String(ok).padStart(4)} usable / ${String(att).padStart(4)} attempted / ${planned} planned${flag}`);
  }
  console.log();
  console.log('WITHIN-MODEL SELF-AGREEMENT (the noise floor)');
  const floors = {};
  for (const m of models) {
    const per = qs.map((q) => { const runs = byModelQ.get(`${m}|${q}`) || []; const ps = []; for (let i = 0; i < runs.length; i++) for (let k = i + 1; k < runs.length; k++) ps.push(jac(runs[i], runs[k])); return meanOf(ps); });
    const mean = meanOf(per), c = ci(per);
    /* ⛔⛔ A FLOOR WITHOUT ENOUGH QUESTIONS IS NOT A FLOOR, AND PRINTING ONE IS THE EXACT SIN THIS
       STUDY AUDITS. Caught 2026-09-23: gemini-flash-lite had ONE question with repeats and the
       table rendered "1.000  95% CI [n/a]  n=0 questions" — a perfect reliability score, displayed
       more confidently than any genuinely measured number beside it, from almost no data.
       ★ THE MOST DANGEROUS NUMBER IN ANY TABLE IS THE ONE COMPUTED FROM TOO LITTLE DATA, BECAUSE IT
         LOOKS IDENTICAL TO THE OTHERS. Suppressed, not shown-with-an-asterisk: an asterisk is a
         footnote nobody reads and the figure still gets quoted. */
    const MIN_Q = 5;
    const usable = (c && c.n >= MIN_Q);
    floors[m] = usable ? mean : null;
    console.log(usable
      ? `  ${m.padEnd(22)} ${mean.toFixed(3)}  95% CI [${c.lo.toFixed(3)}, ${c.hi.toFixed(3)}]  n=${c.n} questions`
      : `  ${m.padEnd(22)} INSUFFICIENT DATA (n=${c ? c.n : 0} questions with repeats, need ${MIN_Q}) — no floor, so no comparison using it is reportable`);
  }
  console.log('\nBETWEEN-MODEL AGREEMENT');
  for (let i = 0; i < models.length; i++) for (let k = i + 1; k < models.length; k++) {
    const A = models[i], B = models[k];
    const per = qs.map((q) => { const ra = byModelQ.get(`${A}|${q}`) || [], rb = byModelQ.get(`${B}|${q}`) || []; const ps = []; for (const a of ra) for (const b of rb) ps.push(jac(a, b)); return meanOf(ps); });
    const mean = meanOf(per), c = ci(per);
    const kind = famOf[A] === famOf[B] ? 'same family, different size' : 'DIFFERENT FAMILY';
    // ⛔ No floor on either side ⇒ the comparison is uninterpretable and must not print a ratio.
    if (floors[A] === null || floors[B] === null || mean === null || !c || c.n < 5) {
      console.log(`  ${A} vs ${B}\n    NOT REPORTABLE — ${mean === null || !c || c.n < 5 ? 'too few shared questions' : 'one side has no measured noise floor'} · ${kind}`);
      continue;
    }
    const floor = (floors[A] + floors[B]) / 2;
    console.log(`  ${A} vs ${B}`);
    console.log(`    ${mean.toFixed(3)}  95% CI [${c.lo.toFixed(3)}, ${c.hi.toFixed(3)}]  n=${c.n}q · floor ${floor.toFixed(3)} · ratio ${(mean / floor).toFixed(2)} · ${kind}`);
  }
  const fams = [...new Set(models.map((m) => famOf[m]))];
  if (fams.length < 2) console.log('\n⚠️ ONE MODEL FAMILY ONLY — size claims are supported, cross-lab claims are NOT.');
  const out = path.join(HERE, `reliability-report-T${TEMP}-${new Date().toISOString().slice(0, 10)}.json`);
  fs.writeFileSync(out, JSON.stringify({ _contract: 'Self- and cross-agreement of AI recommendation sets. Jaccard over canonicalised names. Bootstrap CIs over questions. Not a claim about any company.', temperature: TEMP, questions: qs.length, answers: rows.length, models, families: fams, floors, generatedAt: new Date().toISOString() }, null, 1) + '\n');
  console.log(`\nWROTE ${path.basename(out)}`);
}