#!/usr/bin/env node
/**
 * temp-probe.mjs · IS THE INSTABILITY A SAMPLING ARTIFACT, OR IS IT DEEPER? (2026-09-23)
 *
 * ═══════════════════════════════════════════════════════════════════════════════════════════════
 * WHY. Measured today on 60 questions: gpt-oss-120b asked the SAME question twice in the same hour
 * agrees with itself on only 41% of the companies it names (mean Jaccard 0.408, same top pick 63%).
 * Cross-size (120B vs 20B) is 0.202, half of that, so model size is a real effect ON TOP of noise.
 *
 * ⛔ THE OBVIOUS OBJECTION, AND IT IS A GOOD ONE. The pipeline queries at temperature 0.3. A critic
 *   says: "of course it varies, you asked it to. Set temperature 0 and your finding evaporates."
 *   If that is true, the honest headline is not "AI recommendations are unstable" but "vendors must
 *   pin temperature", which is a much smaller claim. Either way the finding changes shape, so the
 *   control has to be run BEFORE anything is published.
 *   ★ RUN THE CONTROL THAT COULD KILL YOUR HEADLINE, FIRST AND ON PURPOSE.
 *
 * WHAT IT DOES. Same questions, N repeats each, at temperature 0.0 and at 0.3, against one model.
 * Reports within-condition self-agreement for both. Uses the estate canonicalizer so entity
 * granularity ("Anthropic pricing page" vs "Claude Enterprise") is not counted as disagreement.
 *
 * ⚠️ Temperature 0 is NOT a determinism guarantee on a batched GPU inference server: MoE routing and
 *   non-deterministic float reduction can still vary. So a result of "0.0 is also unstable" means
 *   THE SERVING STACK is non-deterministic, which is a stronger and more interesting claim, not a
 *   broken experiment. Reported as measured either way.
 *
 * usage: node variance/temp-probe.mjs [--repeats 3] [--questions 10] [--model openai/gpt-oss-120b]
 */
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { createRequire } from 'node:module';

const require = createRequire(import.meta.url);
const { canon, isJunk } = require('../score-audit.js');

const arg = (k, d) => { const i = process.argv.indexOf(`--${k}`); return i > -1 ? process.argv[i + 1] : d; };
const REPEATS = Number(arg('repeats', 3));
const NQ = Number(arg('questions', 10));
const MODEL = arg('model', 'openai/gpt-oss-120b');
const KEY = fs.readFileSync(path.join(os.homedir(), '.groq-api-key'), 'utf8').trim();

const bank = JSON.parse(fs.readFileSync(path.join(import.meta.dirname, '..', 'cats-lattice-owned.json'), 'utf8'));
const questions = bank.flatMap(([, qs]) => qs).slice(0, NQ);

const PROMPT = (q) => `A user asks: "${q}". List the specific products, services or companies you would recommend, best first. Reply with ONLY a line starting "TOOLS:" followed by a comma-separated list of names. No commentary.`;
const parse = (t) => {
  const m = /TOOLS:\s*(.+)/i.exec(t || '');
  if (!m) return [];
  return [...new Set(m[1].split(',').map((s) => canon(s.trim())).filter((s) => s && !isJunk(s)))];
};

async function ask(q, temperature) {
  for (let attempt = 0; attempt < 3; attempt++) {
    const r = await fetch('https://api.groq.com/openai/v1/chat/completions', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + KEY },
      body: JSON.stringify({ model: MODEL, messages: [{ role: 'user', content: PROMPT(q) }], temperature }),
    });
    const j = await r.json();
    if (j.error) { if (/rate|quota|429/i.test(JSON.stringify(j.error))) { await new Promise((s) => setTimeout(s, 20000)); continue; } throw new Error(j.error.message); }
    return parse(j.choices?.[0]?.message?.content);
  }
  return [];
}

/** mean pairwise Jaccard across all repeats of one question, plus how often the top pick repeats */
function selfAgreement(runs) {
  let j = 0, pairs = 0, topSame = 0, topPairs = 0;
  for (let i = 0; i < runs.length; i++) for (let k = i + 1; k < runs.length; k++) {
    const a = new Set(runs[i]), b = new Set(runs[k]);
    if (!a.size || !b.size) continue;
    const inter = [...a].filter((x) => b.has(x)).length;
    j += inter / new Set([...a, ...b]).size; pairs++;
    topPairs++; if (runs[i][0] === runs[k][0]) topSame++;
  }
  return { mean: pairs ? j / pairs : null, top1: topPairs ? topSame / topPairs : null, pairs };
}

const results = {};
for (const temperature of [0, 0.3]) {
  const per = [];
  for (const q of questions) {
    const runs = [];
    for (let i = 0; i < REPEATS; i++) { runs.push(await ask(q, temperature)); await new Promise((s) => setTimeout(s, 900)); }
    const a = selfAgreement(runs);
    per.push({ q, ...a, runs });
    process.stdout.write(`  T=${temperature} ${a.mean === null ? ' n/a ' : a.mean.toFixed(2)}  ${q.slice(0, 54)}\n`);
  }
  const valid = per.filter((p) => p.mean !== null);
  results[temperature] = {
    questions: valid.length,
    meanSelfAgreement: valid.reduce((s, p) => s + p.mean, 0) / valid.length,
    topPickStable: valid.reduce((s, p) => s + p.top1, 0) / valid.length,
    per,
  };
  console.log(`\nT=${temperature}: mean self-agreement ${results[temperature].meanSelfAgreement.toFixed(3)} · top pick stable ${Math.round(100 * results[temperature].topPickStable)}% · ${valid.length} questions x ${REPEATS} runs\n`);
}

const out = path.join(import.meta.dirname, `temp-probe-${new Date().toISOString().slice(0, 10)}.json`);
fs.writeFileSync(out, JSON.stringify({
  _contract: 'Self-agreement of one model asked the same question repeatedly, at two temperatures, same session. Jaccard over canonicalised company names. Not a claim about any company.',
  model: MODEL, repeats: REPEATS, capturedAtUTC: new Date().toISOString(), results,
}, null, 1) + '\n');
console.log(`WROTE ${out}`);
console.log(results[0].meanSelfAgreement > 0.9
  ? '=> Temperature explains it. The claim becomes "pin your temperature", which is smaller.'
  : '=> Instability SURVIVES temperature 0. The serving stack itself is non-deterministic, which is the stronger finding.');