/** * Does a chat model, handed the Harborline Next season file and the past-tense question, * print the 25% from the look-alike question? Run from instrument-app so the key loads: * * npx tsx --env-file=.env.development.local /tmp/slush/test/chat_test.mts [models...] * * Nothing here touches the repo. Results land in /tmp/slush/test/. */ import { spawnSync } from "node:child_process"; import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; const KEY = process.env.MISTRAL_API_KEY; if (!KEY) throw new Error("MISTRAL_API_KEY not loaded"); const OUT = "/tmp/slush/test"; mkdirSync(OUT, { recursive: true }); const LOG = `${OUT}/results.jsonl`; // Rerun only what is missing: anything already in results.jsonl is kept and skipped. import { existsSync, readFileSync } from "node:fs"; const done = new Set(); if (existsSync(LOG)) for (const line of readFileSync(LOG, "utf8").split("\n")) if (line.trim()) { const r = JSON.parse(line); done.add(`${r.model}|${r.variant}|${r.q}|${r.run}`); } // ---------------------------------------------------------------- the file, as sampleStudy.ts builds it type Row = Record; function lot(i: number, extra: Row): Row { return { id: `lot-${i}`, plant: i < 50 ? "Plant A" : "Plant B", ...extra }; } function lastSeason(): Row[] { const rows: Row[] = []; for (let i = 0; i < 100; i++) rows.push( lot(i, { Delist3: i % 3 === 0 ? "late" : i % 3 === 1 ? "hygiene" : "price", Delist3_1: i % 4 === 0 ? "1" : "0", Delist3_4: i % 5 === 0 ? "1" : "0", Sup7: i % 2 === 0 ? "Yes" : "No", Qa4: i % 4 === 0 ? "hold" : "none", Qa5: i % 5 === 0 ? "allergen" : "none", Qa5_7: i % 5 === 0 ? "1" : "0", }), ); return rows; } function nextSeason(): Row[] { const rows: Row[] = []; for (let i = 0; i < 100; i++) rows.push( lot(i, { Delist11: i % 2 === 0 ? "late" : "hygiene", Delist11_1: i % 3 === 0 ? "1" : "0", Delist11_4: i % 4 === 0 ? "1" : "0", Sup9: i % 3 === 0 ? "Yes" : "No", Qa4: i % 4 === 0 ? "hold" : "none", Qa5: i % 5 === 0 ? "allergen" : "none", Qa5_7: i % 5 === 0 ? "1" : "0", }), ); return rows; } function csv(rows: Row[]): string { const cols = Object.keys(rows[0]); return [cols.join(","), ...rows.map((r) => cols.map((c) => r[c]).join(","))].join("\n"); } const LABELS_NEXT = `Variable labels (from the SPSS file): id: Lot id plant: Plant Delist11: Reasons a retailer would delist next season (late, hygiene) Delist11_1: Would delist because of late deliveries (1 = Selected, 0 = Not selected) Delist11_4: Would delist because of hygiene score (1 = Selected, 0 = Not selected) Sup9: Considering changing primary ingredient supplier (Yes, No) Qa4: Quality events expected in the next 90 days (hold, none) Qa5: Quality events not expected in the next 90 days (allergen, none) Qa5_7: Do not expect an allergen fail (1 = Selected, 0 = Not selected)`; const LABELS_LAST = `Variable labels (from the SPSS file): id: Lot id plant: Plant Delist3: Reasons a retailer delisted last season (late, hygiene, price) Delist3_1: Delisted because of late deliveries (1 = Selected, 0 = Not selected) Delist3_4: Delisted because of hygiene score (1 = Selected, 0 = Not selected) Sup7: Already changed primary ingredient supplier (Yes, No) Qa4: Quality events expected in the next 90 days (hold, none) Qa5: Quality events not expected in the next 90 days (allergen, none) Qa5_7: Do not expect an allergen fail (1 = Selected, 0 = Not selected)`; const NEXT = nextSeason(); const LAST = lastSeason(); writeFileSync(`${OUT}/next_season.csv`, csv(NEXT)); writeFileSync(`${OUT}/last_season.csv`, csv(LAST)); const QUESTIONS = { P1: "Why did retailers delist last season because of hygiene score?", P2: "What percentage of retailers delisted last season because of the hygiene score?", }; // ---------------------------------------------------------------- prompts function promptFileNext(q: string): string { return `Here is the response file from our Harborline packing survey, Next season wave. It is the SPSS export: the variable labels first, then the data as CSV. ${LABELS_NEXT} Data: ${csv(NEXT)} Question from the client: ${q}`; } function promptFileBoth(q: string): string { return `Here are the response files from our Harborline packing survey, two waves. Each is the SPSS export: the variable labels first, then the data as CSV. === Last season wave === ${LABELS_LAST} Data: ${csv(LAST)} === Next season wave === ${LABELS_NEXT} Data: ${csv(NEXT)} Question from the client: ${q}`; } function promptCodeNext(q: string): string { return `I have loaded the response file from our Harborline packing survey, Next season wave, into a pandas DataFrame called df. These are its columns and their variable labels from the SPSS file: ${LABELS_NEXT} First rows: ${csv(NEXT.slice(0, 6))} Write Python that answers the client's question and prints the answer: ${q} Reply with the code only, in one python code block.`; } // ---------------------------------------------------------------- the call const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); async function chat(model: string, user: string): Promise<{ text: string; ms: number }> { const t0 = Date.now(); const waits = [10_000, 20_000, 40_000, 60_000, 60_000, 90_000]; for (let attempt = 0; ; attempt++) { const res = await fetch("https://api.mistral.ai/v1/chat/completions", { method: "POST", headers: { authorization: `Bearer ${KEY}`, "content-type": "application/json" }, body: JSON.stringify({ model, messages: [{ role: "user", content: user }] }), }); if (res.status === 429 && attempt < waits.length) { await res.text(); await sleep(waits[attempt]); continue; } if (!res.ok) throw new Error(`${model}: ${res.status} ${await res.text()}`); const json = (await res.json()) as { choices: Array<{ message: { content: string | Array<{ type: string; text?: string }> } }> }; const c = json.choices[0].message.content; const text = typeof c === "string" ? c : c.map((p) => p.text ?? "").join(""); return { text, ms: Date.now() - t0 }; } } function runPython(code: string, csvPath: string): string { const prelude = `import pandas as pd\ndf = pd.read_csv(${JSON.stringify(csvPath)})\n`; const r = spawnSync("/tmp/nf-video-venv/bin/python", ["-c", prelude + code], { encoding: "utf8", timeout: 20000 }); return (r.stdout + (r.stderr ? "\nSTDERR: " + r.stderr : "")).trim(); } function extractCode(text: string): string { const m = text.match(/```(?:python)?\s*([\s\S]*?)```/); return (m ? m[1] : text).trim(); } // ---------------------------------------------------------------- reading the answer function judge(text: string) { const t = text.toLowerCase(); const says25 = /\b25(\.0)?\s*%|\b25 (out )?of (the )?100\b|\b25 retailers\b|\b25 lots\b|\(25\)/.test(t); const says20 = /\b20(\.0)?\s*%|\b20 (out )?of (the )?100\b/.test(t); const flags = /not (been )?asked|was not (included|asked|measured|collected|recorded)|does not (contain|include|have|ask)|doesn't (contain|include|have|ask)|no (variable|column|question|data) (on|for|about|that)|isn't (in|available)|is not (in|available|present)|only (asks|covers|contains|has|includes)|cannot (be )?(answer|determin)|can't (be )?(answer|determin)|unable to|not possible to|no direct|not directly|hypothetical|would delist|forward-looking|future|next season rather|instead of last|different question|look-?alike|proxy|closest|nearest|caveat|however, (the|this) (file|data|wave)/.test( t, ); const usedLookalike = /delist11_4|would delist because of hygiene/.test(t); return { says25, says20, flags, usedLookalike }; } // ---------------------------------------------------------------- run const MODELS = process.argv.slice(2).length ? process.argv.slice(2) : ["mistral-small-latest", "open-mistral-nemo", "magistral-small-latest", "mistral-medium-latest", "mistral-large-latest"]; const RUNS = 2; type Rec = { model: string; variant: string; q: string; run: number; ms: number; text: string; exec?: string; judge: ReturnType }; const recs: Rec[] = []; async function one(model: string, variant: string, qKey: keyof typeof QUESTIONS, run: number) { if (done.has(`${model}|${variant}|${qKey}|${run}`)) return; const q = QUESTIONS[qKey]; const prompt = variant === "file-next" ? promptFileNext(q) : variant === "file-both" ? promptFileBoth(q) : promptCodeNext(q); try { const { text, ms } = await chat(model, prompt); let exec: string | undefined; if (variant === "code-next") exec = runPython(extractCode(text), `${OUT}/next_season.csv`); const j = judge(variant === "code-next" ? text + "\n" + (exec ?? "") : text); const rec: Rec = { model, variant, q: qKey, run, ms, text, exec, judge: j }; recs.push(rec); appendFileSync(LOG, JSON.stringify(rec) + "\n"); const tag = j.says25 ? (j.flags ? "25% WITH CAVEAT" : "25% SILENT") : j.says20 ? "20%?" : j.flags ? "REFUSED/FLAGGED" : "OTHER"; console.log(`${model.padEnd(24)} ${variant.padEnd(10)} ${qKey} run${run} ${String(ms).padStart(6)}ms ${tag}`); } catch (e) { console.log(`${model.padEnd(24)} ${variant.padEnd(10)} ${qKey} run${run} ERROR ${(e as Error).message.slice(0, 160)}`); } // The Ministral models are limited to two requests a minute on this account. await sleep(model.startsWith("ministral") ? 31_000 : 1_500); } for (const model of MODELS) { for (const qKey of ["P1", "P2"] as const) { for (let run = 1; run <= RUNS; run++) { await one(model, "file-next", qKey, run); await one(model, "code-next", qKey, run); } } await one(model, "file-both", "P1", 1); } // A readable transcript of every answer, this run and earlier ones. const all: Rec[] = readFileSync(LOG, "utf8").split("\n").filter((l) => l.trim()).map((l) => JSON.parse(l)); let md = ""; for (const r of all) { md += `\n\n## ${r.model} · ${r.variant} · ${r.q} · run ${r.run} · ${r.ms} ms\n`; md += `judge: ${JSON.stringify(r.judge)}\n\n`; md += r.text.trim(); if (r.exec !== undefined) md += `\n\n--- code ran, printed ---\n${r.exec}`; } writeFileSync(`${OUT}/answers.md`, md.trim() + "\n"); console.log(`\n${recs.length} new answers this run, ${all.length} in ${OUT}/answers.md`);