#!/usr/bin/env node
/**
 * size-at-temp0.mjs · ISOLATE THE MODEL-SIZE EFFECT FROM SAMPLING NOISE (2026-09-23).
 *
 * ═══════════════════════════════════════════════════════════════════════════════════════════════
 * ⛔ THIS EXISTS TO CORRECT A CLAIM I MADE TOO FAST.
 * Earlier today I compared gpt-oss-120b against gpt-oss-20b at temperature 0.3, measured overlap
 * 0.202 against a within-model baseline of 0.408, and called the size effect real.
 * Then the temperature control showed the noise floor at 0.3 is itself enormous: the SAME model
 * asked the SAME question three times agrees with itself only 0.293. At temperature 0 it rises to
 * 0.752.
 *   ★ A DIFFERENCE MEASURED THROUGH THAT MUCH NOISE IS NOT YET A DIFFERENCE. Comparing two models
 *     at a temperature where each barely reproduces itself cannot separate "these models differ"
 *     from "neither model is repeating itself".
 *
 * THE CLEAN TEST. Run both sizes at temperature 0, where within-model agreement is 0.752, and
 * compare cross-size overlap against that floor:
 *   · cross-size ≈ 0.75  → NO size effect. The earlier 0.202 was sampling noise, and the honest
 *                          published finding is about temperature and run counts only.
 *   · cross-size ≪ 0.75  → size effect is REAL and now cleanly separated from noise.
 * Both outcomes are publishable. Only one of them is what I said this morning.
 *
 * ⚠️ Temperature 0 is not a determinism guarantee on batched MoE inference, which is why the floor
 *   is 0.752 rather than 1.0. The floor is measured, not assumed, and both arms share it.
 *
 * usage: node variance/size-at-temp0.mjs [--questions 20] [--repeats 2]
 */
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { createRequire } from 'node:module';

const require = createRequire(import.meta.url);
const { canon, isJunk } = require('../score-audit.js');
const arg = (k, d) => { const i = process.argv.indexOf(`--${k}`); return i > -1 ? process.argv[i + 1] : d; };
const NQ = Number(arg('questions', 20));
const REPEATS = Number(arg('repeats', 2));
const KEY = fs.readFileSync(path.join(os.homedir(), '.groq-api-key'), 'utf8').trim();
const MODELS = { '120b': 'openai/gpt-oss-120b', '20b': 'openai/gpt-oss-20b' };

const bank = JSON.parse(fs.readFileSync(path.join(import.meta.dirname, '..', 'cats-lattice-owned.json'), 'utf8'));
const questions = bank.flatMap(([, qs]) => qs).slice(0, NQ);

const PROMPT = (q) => `A user asks: "${q}". List the specific products, services or companies you would recommend, best first. Reply with ONLY a line starting "TOOLS:" followed by a comma-separated list of names. No commentary.`;
const parse = (t) => { const m = /TOOLS:\s*(.+)/i.exec(t || ''); return m ? [...new Set(m[1].split(',').map((s) => canon(s.trim())).filter((s) => s && !isJunk(s)))] : []; };

async function ask(model, q) {
  for (let a = 0; a < 3; a++) {
    const r = await fetch('https://api.groq.com/openai/v1/chat/completions', {
      method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + KEY },
      body: JSON.stringify({ model, messages: [{ role: 'user', content: PROMPT(q) }], temperature: 0 }),
    });
    const j = await r.json();
    if (j.error) { if (/rate|quota|429/i.test(JSON.stringify(j.error))) { await new Promise((s) => setTimeout(s, 20000)); continue; } throw new Error(j.error.message); }
    return parse(j.choices?.[0]?.message?.content);
  }
  return [];
}
const jac = (a, b) => { const sa = new Set(a), sb = new Set(b); if (!sa.size || !sb.size) return null; return [...sa].filter((x) => sb.has(x)).length / new Set([...a, ...b]).size; };

const data = [];
for (const q of questions) {
  const runs = { '120b': [], '20b': [] };
  for (const [tag, model] of Object.entries(MODELS)) {
    for (let i = 0; i < REPEATS; i++) { runs[tag].push(await ask(model, q)); await new Promise((s) => setTimeout(s, 700)); }
  }
  data.push({ q, runs });
  process.stdout.write('.');
}
console.log('\n');

const mean = (xs) => { const v = xs.filter((x) => x !== null); return v.length ? v.reduce((a, b) => a + b, 0) / v.length : null; };
const within120 = mean(data.map((d) => jac(d.runs['120b'][0], d.runs['120b'][1])));
const within20 = mean(data.map((d) => jac(d.runs['20b'][0], d.runs['20b'][1])));
const across = mean(data.flatMap((d) => d.runs['120b'].flatMap((a) => d.runs['20b'].map((b) => jac(a, b)))));
const floor = (within120 + within20) / 2;

console.log('ALL AT TEMPERATURE 0 · ' + data.length + ' questions x ' + REPEATS + ' runs per model');
console.log('  within 120B (noise floor)  :', within120.toFixed(3));
console.log('  within 20B  (noise floor)  :', within20.toFixed(3));
console.log('  ACROSS sizes               :', across.toFixed(3));
console.log('  across / floor ratio       :', (across / floor).toFixed(2));
console.log();
console.log(across < floor * 0.75
  ? '=> SIZE EFFECT CONFIRMED at temperature 0, cleanly separated from sampling noise.'
  : '=> NO SIZE EFFECT once noise is controlled. This morning\'s 0.202 was largely temperature. Publish the temperature/run-count finding ONLY.');

const out = path.join(import.meta.dirname, `size-at-temp0-${new Date().toISOString().slice(0, 10)}.json`);
fs.writeFileSync(out, JSON.stringify({
  _contract: 'Cross-model-size recommendation overlap at temperature 0, against a measured within-model noise floor. One model family (OpenAI gpt-oss), one host (Groq), one date. Not a claim about any company.',
  capturedAtUTC: new Date().toISOString(), models: MODELS, questions: data.length, repeats: REPEATS,
  within120, within20, floor, across, ratio: across / floor, data,
}, null, 1) + '\n');
console.log(`WROTE ${out}`);