diff --git a/README.md b/README.md index 88cd0a9..e8c1e40 100644 --- a/README.md +++ b/README.md @@ -424,3 +424,26 @@ Adding a feature to the tier matrix is an entry in `src/core/registry.ts`, not a Licensed under [Apache 2.0](LICENSE). See [`NOTICE`](NOTICE) for attribution. The probe, scoring, conformance tests, capability evals, benchmark, and CLI are original to llmprobe. The OpenAPI schemas under `schema/` (and the Zod validators generated from them) are derived from [openresponses](https://github.com/openresponses/openresponses) and the official [openai/openai-openapi](https://github.com/openai/openai-openapi) spec, retained under Apache 2.0 — full attribution in [`NOTICE`](NOTICE). + +### Multilingual response collection + +`--language-only` collects 100 original prompts: eight tasks in each of English, +Chinese, Thai, Japanese, Spanish, French, German, Korean, Arabic, Portuguese, +Russian and Italian, plus four cross-language tasks. Categories cover register, +idioms, pragmatics, grammar, localization, culture, creative writing and ambiguity. + +```bash +npx llmprobe localhost:12345 --language-only --model MODEL \ + --concurrency 4 --eval-max-tokens 500 --timeout 600 --save runs/language-MODEL.jsonl +``` + +This mode defaults to greedy sampling, thinking off, speculation off, concurrency +4 and a 500-token output cap. The OpenAI-compatible endpoint must support these +request controls. It writes a manifest, every complete response (including usage, +finish reason and request settings), errors, and a summary to a new JSONL file. +Existing logs are never overwritten. It does not upload data or assign automatic +language scores; evaluate saved answers offline. Scores should cover accuracy, +naturalness, register/style, nuance/cultural fit and task fulfillment, with equal +weight per language. Report cross-language cases separately, and disclose the +judge and any output truncation. These logs are distinct from normal probe report +JSON and cannot be passed to `--compare`. diff --git a/bin/llmprobe.ts b/bin/llmprobe.ts index 70aad48..02d6c86 100644 --- a/bin/llmprobe.ts +++ b/bin/llmprobe.ts @@ -94,7 +94,10 @@ import { } from "../src/reasoning/index"; import { ALL_EVALS } from "../src/evals/index"; +import { runLanguage } from "../src/language/index"; + interface Args { + languageOnly?: boolean; target?: string; apiKey?: string; model?: string; @@ -241,6 +244,9 @@ function parseArgs(argv: string[]): Args { args.bench = true; args.benchOnly = true; break; + case "--language-only": + args.languageOnly = true; + break; case "--eval": args.eval = true; break; @@ -453,6 +459,10 @@ Options: On by default; --no-bench skips it --bench-only Run only the benchmark — no conformance, evals, agentic or fidelity. Surface discovery still runs; it is free. + --language-only Collect 100 multilingual responses for offline review (12 languages). + Requires --model and --save (new JSONL file). Defaults: concurrency 4, + max output 500, greedy sampling, thinking off. Uses --concurrency, + --eval-max-tokens and --timeout. Separate from scored probe reports. --eval Reasoning accuracy: 92 questions from GPQA Diamond, SuperGPQA, AIME 2025 and COMPSEC (informational, never scored). Expensive on a thinking model: up to @@ -1717,6 +1727,36 @@ async function main(): Promise { const root = normalizeRoot(args.target); const apiKey = args.apiKey ?? process.env.LLMPROBE_API_KEY ?? ""; + if (args.languageOnly) { + if (!args.model || !args.save) + throw new Error("--language-only requires --model and --save"); + if ( + args.sampling || + args.upload || + args.html || + args.eval || + args.benchOnly || + args.budget || + args.evalCases || + args.evalQuestions + ) + throw new Error( + "--language-only supports --model, --save, --concurrency, --eval-max-tokens, --timeout and --api-key; other run modes are not supported", + ); + const result = await runLanguage({ + root, + model: args.model, + save: args.save, + apiKey, + concurrency: args.concurrency, + maxTokens: args.evalMaxTokens, + timeoutMs: args.timeoutSec * 1000, + log, + }); + console.log(JSON.stringify(result)); + if (result.errors) process.exitCode = 1; + return; + } const startedAt = Date.now(); log( diff --git a/src/language/cases.ts b/src/language/cases.ts new file mode 100644 index 0000000..bdfe605 --- /dev/null +++ b/src/language/cases.ts @@ -0,0 +1,178 @@ +/** Original prompts, authored for this suite. Eight matched task categories per language. */ +export interface LanguageCase { + id: string; + language: string; + category: string; + prompt: string; +} +const categories = [ + "register", + "idiom", + "pragmatics", + "grammar", + "localization", + "culture", + "creative", + "ambiguity", +]; +const prompts: Record = { + EN: [ + "Write two short messages declining a dinner invitation because you are exhausted: one to a close friend and one to a senior colleague. Be warm, clear, and do not invent another commitment. Use English; label each version.", + "In English, explain “That’s a bit rich coming from you” to an advanced learner. Give a natural two-line dialogue and distinguish it from a comment about wealth. Keep it under 100 words.", + "A British colleague reviews your proposal and says, “That’s certainly one way of doing it.” In English, give two plausible readings, explain what context would help, and write a tactful reply. Do not assume sarcasm is certain.", + "Correct this into natural standard English without changing the meaning, then explain the three main corrections: “If I would have known, I would tell you yesterday. Neither of the two options are suitable.”", + "Adapt “Grab our killer deals before they’re gone!” for a calm, trustworthy UK bookshop newsletter. Give a subject line and one sentence in English. Preserve urgency without violence, shouting, or invented discounts.", + "In English, advise a visitor invited to a colleague’s home in London for the first time. Give three useful suggestions without claiming all British households follow the same customs. Stay under 100 words.", + "Write a 70–90-word English scene in which two old friends miss the last bus. Show affection through understated teasing; do not name either character’s emotions or use stereotypes.", + "In English, explain the two readings of “I saw her duck.” Rewrite it unambiguously for each reading, then give one short context in which each is natural.", + ], + CN: [ + "请用简体中文写两条婉拒晚餐邀请的短消息,原因是太累了:一条发给好友,一条发给资深同事。语气温和但明确,不要编造其他安排。标明对象。", + "用简体中文给高级汉语学习者解释“你可真会挑时候”。写一个自然的两句对话,说明它何时是真夸奖、何时是反话,不要说它一定是讽刺。150字以内。", + "同事看完你的方案说:“这个想法挺有意思的,我们再研究研究。”用简体中文说出两种可能的言外之意、判断所需的语境,并写一句得体的回应。", + "把下面的话改成自然的现代汉语,保留原意,并解释三处修改:“虽然他很忙,但是却他每天都坚持练习了中文。我昨天把一本书看,但是没看完。”", + "把“Grab our killer deals before they’re gone!”改写成适合中国大陆独立书店的简体中文推广文案。写一个标题和一句正文,克制可信,有紧迫感,不要逐字翻译或虚构折扣。", + "朋友第一次去中国大陆同事家吃饭,担心失礼。用简体中文给三条实用建议,承认家庭和地区差异,不要把饮酒或争着买单说成必须遵守的规矩。180字以内。", + "用简体中文写一段120—160字的小场景:两位老友错过末班车。用含蓄的打趣体现亲近,不要直接写“他们很开心”“他们很感动”,不要靠方言刻板印象。", + "“我差点没赶上火车”和“我差点赶上火车”通常分别表示什么?用简体中文解释,并说明口语中的语气或上下文为什么可能让“差点没……”有歧义。各造一个清楚的例句。", + ], + TH: [ + "เขียนข้อความภาษาไทยสั้น ๆ สองแบบเพื่อปฏิเสธคำชวนไปกินข้าวเพราะเหนื่อยมาก แบบแรกส่งให้เพื่อนสนิท แบบที่สองส่งให้เพื่อนร่วมงานอาวุโส ใช้น้ำเสียงอบอุ่นแต่ชัดเจน ไม่แต่งเรื่องว่ามีนัดอื่น ผู้พูดเป็นผู้หญิง ระบุว่าแต่ละแบบส่งให้ใคร", + "อธิบายสำนวน “เกรงใจ” ให้ผู้เรียนภาษาไทยระดับสูงเข้าใจเป็นภาษาไทย ยกบทสนทนาสั้น ๆ และอธิบายว่าทำไมจึงไม่ตรงกับคำว่า afraid เสมอไป ไม่เกิน 120 คำ", + "เพื่อนร่วมงานตอบคำชวนว่า “ไว้วันหลังนะ” อธิบายเป็นภาษาไทยว่าสื่อได้อย่างน้อยสองแบบ ต้องรู้อะไรเพิ่มจึงจะตีความได้ และควรตอบอย่างไรโดยไม่กดดันเขา", + "แก้ประโยคนี้ให้เป็นภาษาไทยเขียนที่เป็นธรรมชาติ แล้วอธิบายจุดที่แก้: “ดิฉันพึ่งจะได้รู้ว่าเขาไม่อนุญาติให้จอดรถที่นี่ค่ะ คุณช่วยบอกทางออกให้หน่อยได้ไหมคะค่ะ” รักษาความหมายและเสียงผู้พูดหญิง", + "ดัดแปลง “Grab our killer deals before they’re gone!” เป็นข้อความประชาสัมพันธ์ร้านหนังสืออิสระในไทย เขียนหัวเรื่องหนึ่งบรรทัดและเนื้อความหนึ่งประโยค สุภาพ น่าเชื่อถือ มีความเร่งด่วนพอประมาณ ไม่แปลตรงตัวและไม่แต่งตัวเลขส่วนลด", + "ให้คำแนะนำภาษาไทยสามข้อแก่ชาวต่างชาติที่ไปกินข้าวบ้านเพื่อนร่วมงานชาวไทยครั้งแรก กล่าวถึงความแตกต่างระหว่างบ้านด้วย อย่าอ้างว่าคนไทยทุกคนทำเหมือนกัน และอย่าบังคับให้ดื่มแอลกอฮอล์", + "เขียนฉากสั้นภาษาไทยประมาณ 100–140 คำ เพื่อนเก่าสองคนตกรถเมล์เที่ยวสุดท้าย ให้เห็นความสนิทผ่านการแซวอย่างอ่อนโยน ไม่บอกอารมณ์ตรง ๆ และไม่ใช้ภาพเหมารวมเรื่องภูมิภาค", + "อธิบายความต่างระหว่าง “ไม่ต้องมา” กับ “มาไม่ต้อง” ในภาษาไทยทั่วไป ข้อความหลังอาจเป็นการพิมพ์ผิดหรือลำดับคำที่ไม่เป็นธรรมชาติได้อย่างไร อย่าสร้างความหมายตายตัวให้ประโยคที่ผิดธรรมชาติ พร้อมเสนอประโยคแก้สองแบบที่มีเจตนาต่างกัน", + ], + JP: [ + "疲れているため夕食の誘いを断る短いメッセージを、日本語で二つ書いてください。一つは親友へ、もう一つは職場の先輩へ。温かく明確に断り、別の予定を捏造しないでください。宛先を明記してください。", + "「それはちょっと……」が断りとして使われる理由を、日本語学習者向けに日本語で説明してください。自然な二往復以内の会話と、必ずしも断りとは限らない例を一つずつ示してください。", + "上司が提案を聞いて「検討しておきます」と言いました。考えられる解釈を二つ、判断に必要な状況、角の立たない確認の返事を日本語で書いてください。拒否と決めつけないでください。", + "次の文を自然で適切な敬語に直し、主な修正点を説明してください。「部長が申された資料を拝見されましたか。私は明日、先生の研究室にいらっしゃいます。」最初の質問の相手は同僚で、資料を読むのもその同僚です。", + "“Grab our killer deals before they’re gone!” を日本の落ち着いた独立系書店の案内に合う日本語へ意訳してください。件名と本文一文。控えめな緊急性を残し、割引率や期間は創作しないでください。", + "日本の同僚の自宅に初めて夕食に招かれた人へ、日本語で三つ助言してください。家庭による違いに配慮し、贈り物の値段や飲酒を絶対的な義務にしないでください。", + "終バスを逃した旧友二人の場面を、日本語で200〜260字で書いてください。控えめなからかいで親しさを伝え、感情を直接説明せず、地域のステレオタイプを使わないでください。", + "「先生は来られます」は文脈によってどう解釈できますか。日本語で主な二つの解釈を説明し、それぞれを曖昧でない文に言い換えてください。", + ], + ESP: [ + "Escribe en español dos mensajes breves para rechazar una invitación a cenar porque estás agotada: uno para una amiga íntima y otro para una compañera veterana de trabajo. Sé cálida y clara, sin inventar otro compromiso. Etiqueta las versiones.", + "Explica en español «Ya era hora» a un estudiante avanzado. Incluye un diálogo de dos líneas y distingue el alivio amistoso del reproche según el contexto. Máximo 100 palabras.", + "Una compañera de España responde a tu propuesta: «Bueno, ya veremos». Da dos interpretaciones posibles, indica qué contexto ayudaría y escribe una respuesta cortés que no la presione. No des por hecho que es un rechazo.", + "Corrige para un registro escrito estándar y explica los cambios: «Si tendría tiempo, iría contigo. Habían muchas personas esperando y la dije a Marta que volviera mañana». Conserva el significado y menciona si alguna forma aparece en usos regionales.", + "Adapta «Grab our killer deals before they’re gone!» para el boletín de una librería independiente de España. Da un asunto y una frase en español: tono tranquilo, fiable y con cierta urgencia, sin traducción literal ni descuentos inventados.", + "En español, da tres consejos a alguien invitado por primera vez a cenar en casa de un compañero en México. Reconoce diferencias familiares y regionales; no conviertas el alcohol ni una hora fija de retraso en obligaciones. Máximo 110 palabras.", + "Escribe una escena de 70–90 palabras en español: dos viejos amigos pierden el último autobús. Muestra cariño mediante bromas suaves, sin nombrar sus emociones ni usar estereotipos regionales.", + "Explica en español la diferencia entre «Aunque llueve, salgo» y «Aunque llueva, salgo». Incluye un contexto natural para cada una y evita reducir la diferencia a presente frente a futuro.", + ], + FR: [ + "En français, écris deux messages courts refusant une invitation à dîner parce que tu es épuisée : un à une amie proche, un à une collègue plus expérimentée que tu vouvoies. Sois chaleureuse et claire, sans inventer un autre engagement. Indique les destinataires.", + "Explique « C’est du joli ! » en français à un apprenant avancé. Donne un dialogue de deux répliques et distingue un emploi ironique d’un véritable compliment. Moins de 100 mots.", + "Après ta proposition, un collègue répond : « Pourquoi pas, à voir. » Donne en français deux lectures possibles, le contexte utile pour trancher et une réponse diplomatique. Ne conclus pas automatiquement à un refus.", + "Corrige ces phrases en français standard et explique les changements : « Si j’aurais su, je serais venue. Les lettres que j’ai écrit sont sur la table. Elle s’est permise de partir. » Ne change pas le sens.", + "Adapte « Grab our killer deals before they’re gone! » au bulletin d’une librairie indépendante en France. Donne un objet et une phrase en français, sobres et crédibles, avec une urgence modérée, sans inventer de remise ni de date.", + "En français, donne trois conseils à une personne invitée pour la première fois chez une collègue en France. Tiens compte des différences entre foyers ; ne présente ni la bise ni le vin comme obligatoires. Moins de 110 mots.", + "Écris une scène de 70 à 90 mots en français : deux vieux amis ratent le dernier bus. Fais sentir leur affection par de petites taquineries, sans nommer leurs émotions ni recourir à des stéréotypes régionaux.", + "Explique en français les deux lectures possibles de « J’ai vu l’homme avec les jumelles ». Réécris la phrase sans ambiguïté pour chaque lecture et donne un bref contexte pour chacune.", + ], + GER: [ + "Schreibe auf Deutsch zwei kurze Absagen einer Einladung zum Abendessen, weil du erschöpft bist: eine an einen engen Freund und eine an eine erfahrene Kollegin, die du siezt. Sei freundlich und eindeutig, erfinde keinen anderen Termin. Beschrifte beide Fassungen.", + "Erkläre „Das hast du ja toll gemacht!“ auf Deutsch für Fortgeschrittene. Gib einen kurzen Dialog und zeige, wie Tonfall und Situation echtes Lob von Ironie unterscheiden. Höchstens 100 Wörter.", + "Eine Kollegin antwortet auf deinen Vorschlag: „Das können wir uns ja mal anschauen.“ Nenne auf Deutsch zwei plausible Deutungen, hilfreichen Kontext und eine höfliche Rückfrage. Unterstelle keine sichere Ablehnung.", + "Korrigiere für formelles Standarddeutsch und erkläre die Änderungen: „Wegen dem Regen bin ich zuhause geblieben. Wenn ich das gewusst hätte, würde ich gestern gekommen sein. Das ist der Kollege, wo ich geholfen habe.“ Erwähne gegebenenfalls umgangssprachliche Varianten.", + "Übertrage „Grab our killer deals before they’re gone!“ frei ins Deutsche für den Newsletter einer ruhigen unabhängigen Buchhandlung. Schreibe Betreff und einen Satz: glaubwürdig, etwas dringlich, ohne erfundene Rabatte oder martialische Sprache.", + "Gib auf Deutsch drei Tipps für den ersten Abendessensbesuch bei einer Kollegin in Deutschland. Berücksichtige Unterschiede zwischen Haushalten; erkläre weder Alkohol noch minutengenaue Pünktlichkeit zur allgemeingültigen Pflicht. Höchstens 110 Wörter.", + "Schreibe eine deutsche Szene mit 70–90 Wörtern: Zwei alte Freunde verpassen den letzten Bus. Zeige Zuneigung durch sanftes Necken, ohne Gefühle ausdrücklich zu benennen oder regionale Klischees zu verwenden.", + "Erkläre auf Deutsch den Unterschied zwischen „Du musst nicht kommen“ und „Du darfst nicht kommen“. Gib je einen natürlichen Kontext und eine eindeutige englische Übersetzung. Achte auf den Geltungsbereich der Verneinung.", + ], + KR: [ + "너무 피곤해서 저녁 초대를 거절하는 짧은 한국어 메시지를 두 개 쓰세요. 하나는 친한 친구에게, 하나는 직장 선배에게 보내는 것입니다. 따뜻하지만 분명하게 말하고 다른 약속을 지어내지 마세요. 대상을 표시하세요.", + "한국어 고급 학습자에게 “눈치가 빠르다”를 한국어로 설명하세요. 짧은 두 줄 대화를 넣고, 단순히 시력이 좋다는 뜻과 어떻게 다른지 설명하세요. 무조건 복종을 잘한다는 뜻으로 설명하지 마세요.", + "동료가 제안을 듣고 “한번 생각해 볼게요”라고 했습니다. 가능한 해석 두 가지, 판단에 필요한 맥락, 부담을 주지 않는 답장을 한국어로 쓰세요. 반드시 거절이라고 단정하지 마세요.", + "다음 문장을 자연스러운 표준 한국어로 고치고 수정 이유를 설명하세요. “내일 뵈요. 이 책을 읽는 게 더 낳을 것 같아요. 선생님께서 저에게 말씀을 드렸어요.” 마지막 문장은 선생님이 학생인 나에게 말한 상황입니다.", + "“Grab our killer deals before they’re gone!”을 한국 독립 서점의 차분한 소식지에 맞게 한국어로 의역하세요. 제목과 본문 한 문장으로 쓰고, 적당한 긴박감은 살리되 할인율이나 마감일을 만들지 마세요.", + "한국인 동료의 집에 처음 저녁 초대를 받은 사람에게 한국어로 조언 세 가지를 주세요. 가정마다 다름을 인정하고 술을 마시거나 나이에 따른 위계를 무조건 따라야 한다고 말하지 마세요.", + "막차를 놓친 오랜 친구 둘의 장면을 한국어 6~8문장으로 쓰세요. 가벼운 놀림으로 친밀함을 드러내고 감정을 직접 설명하거나 지역에 대한 고정관념을 쓰지 마세요.", + "“민수는 영희만 좋아한다”와 “민수만 영희를 좋아한다”의 차이를 한국어로 설명하세요. 각각에서 배제되는 대상이 누구인지 밝히고, 각 문장에 맞는 짧은 상황을 하나씩 쓰세요.", + ], + ARABIC: [ + "اكتب بالعربية الفصحى رسالتين قصيرتين للاعتذار عن دعوة إلى العشاء بسبب الإرهاق: واحدة لصديق مقرّب وأخرى لزميل أقدم منك. كن ودودًا وواضحًا ولا تختلق موعدًا آخر. حدّد المخاطَب في كل نسخة.", + "اشرح بالعربية الفصحى تعبير «على عيني وراسي» لمتعلم متقدم. قدّم حوارًا قصيرًا باللهجة المصرية وبيّن وظيفة التعبير الاجتماعية، ولماذا لا يُفهم حرفيًا. لا تفترض أنه وعد بالتنفيذ في كل سياق.", + "ردّ زميل على اقتراح موعد بقوله «إن شاء الله». اشرح بالعربية قراءتين محتملتين، وما السياق اللازم للفهم، واكتب متابعة مهذبة لتأكيد الموعد. لا تختزل العبارة في الرفض أو تربطها بصورة نمطية عن المتكلمين.", + "صحّح الجمل الآتية إلى العربية الفصحى واشرح التغييرات بإيجاز: «إنّ الموظفون مستعدون. لم يذهبُ خالد إلى المكتب. هذه ثلاثة رسائل مهمة». حافظ على المعنى.", + "كيّف عبارة «Grab our killer deals before they’re gone!» لنشرة مكتبة مستقلة في مصر. اكتب عنوانًا وجملة واحدة بالعربية الفصحى بنبرة هادئة موثوقة وإلحاح معتدل، دون ترجمة حرفية أو اختلاق خصم أو موعد.", + "قدّم بالعربية ثلاث نصائح لشخص دُعي لأول مرة إلى العشاء في منزل زميل في الأردن. راعِ اختلاف الأسر والأديان والعادات، ولا تفترض جنس المضيف أو تجعل تناول طعام بعينه واجبًا. أقل من 110 كلمات.", + "اكتب مشهدًا بالعربية الفصحى من 70 إلى 90 كلمة عن صديقين قديمين فاتهما آخر حافلة. أظهر الألفة بمزاح لطيف دون تسمية مشاعرهما مباشرة أو استخدام صور نمطية إقليمية.", + "الجملة غير المشكولة «زار موسى عيسى» قد يلتبس فيها الفاعل بالمفعول. اشرح بالعربية القراءة الافتراضية وحدود الاعتماد على ترتيب الكلمات هنا، ثم أعد الصياغة مرتين لتوضيح كل احتمال دون الاتكال على علامات إعراب ظاهرة على الاسمين.", + ], + POR: [ + "Escreva em português brasileiro duas mensagens curtas recusando um convite para jantar porque você está exausta: uma para uma amiga íntima e outra para uma colega mais experiente. Seja calorosa e clara, sem inventar outro compromisso. Identifique as versões.", + "Explique em português brasileiro a expressão “Pois não” a um estudante avançado. Dê um diálogo de duas falas e mostre por que ela pode indicar disponibilidade, e não uma recusa. Até 100 palavras.", + "Um colega brasileiro responde a um convite: “A gente combina”. Em português, apresente duas interpretações possíveis, o contexto necessário e uma resposta gentil para confirmar sem pressionar. Não presuma uma recusa.", + "Corrija para a escrita formal brasileira e explique as mudanças: “Fazem dois anos que não lhe vejo. Se eu ver a Ana, aviso. Haviam muitas pessoas na fila.” Distinga preferência normativa de usos comuns na fala quando pertinente.", + "Adapte “Grab our killer deals before they’re gone!” para o boletim de uma livraria independente brasileira. Escreva um assunto e uma frase em português: tom calmo e confiável, urgência moderada, sem tradução literal nem descontos inventados.", + "Em português europeu, dê três conselhos a quem vai jantar pela primeira vez em casa de um colega em Portugal. Reconheça diferenças entre famílias e não apresente vinho ou beijos como obrigações. Até 110 palavras.", + "Escreva em português brasileiro uma cena de 70–90 palavras: dois velhos amigos perdem o último ônibus. Mostre afeto por meio de provocações leves, sem nomear as emoções nem usar estereótipos regionais.", + "Explique em português a diferença entre “Ele deve ter chegado” e “Ele tem de chegar”. Dê um contexto para cada frase e diga por que “dever” nem sempre expressa obrigação.", + ], + RUS: [ + "Напишите по-русски два коротких отказа от приглашения на ужин из-за сильной усталости: близкому другу и старшей коллеге, с которой вы на «вы». Тон тёплый, отказ ясный; не придумывайте других планов. Подпишите варианты.", + "Объясните продвинутому изучающему русский выражение «Ну ты даёшь!». Приведите короткий диалог и покажите, как контекст меняет восхищение на неодобрение. Ответьте по-русски, до 100 слов.", + "Коллега ответил на предложение: «Ну, посмотрим». По-русски назовите два возможных смысла, необходимый контекст и тактичную уточняющую реплику. Не считайте отказ единственным вариантом.", + "Исправьте для нейтральной письменной речи и объясните ошибки: «Согласно приказа, нужно оплатить за проезд. Приехав домой, у меня сломался телефон». Во второй фразе домой приехал говорящий; сохраните этот смысл.", + "Адаптируйте «Grab our killer deals before they’re gone!» для спокойной рассылки независимого книжного магазина. Дайте тему письма и одно предложение по-русски: умеренная срочность, без буквального перевода и выдуманных скидок.", + "Дайте по-русски три совета человеку, впервые приглашённому на ужин домой к русскоязычному коллеге. Учитывайте различия между семьями и странами; не объявляйте алкоголь или определённый подарок обязательными. До 110 слов.", + "Напишите по-русски сцену объёмом 70–90 слов: двое старых друзей опоздали на последний автобус. Передайте близость мягкими поддразниваниями, не называя эмоций и не используя региональные стереотипы.", + "Объясните по-русски разницу между «Он не мог прийти» и «Он мог не прийти». Приведите естественный контекст для каждого и поясните область действия отрицания.", + ], + ITA: [ + "Scrivi in italiano due brevi messaggi per rifiutare un invito a cena perché sei esausta: uno a un’amica intima, uno a una collega più esperta a cui dai del Lei. Sii calorosa e chiara, senza inventare altri impegni. Etichetta le versioni.", + "Spiega in italiano “Ma figurati!” a uno studente avanzato. Dai un breve dialogo e distingui la risposta cortese a un ringraziamento da un uso di incredulità. Massimo 100 parole.", + "Una collega risponde a una proposta: “Vediamo un po’”. In italiano, presenta due letture plausibili, il contesto utile e una replica garbata per chiarire. Non dare per certo che sia un rifiuto.", + "Correggi per l’italiano scritto standard e spiega le modifiche: “Se lo sapevo, sarei venuta. A me mi sembra che lui ha ragione. Qual’è il problema?” Distingui, dove serve, usi colloquiali da errori ortografici.", + "Adatta “Grab our killer deals before they’re gone!” alla newsletter di una libreria indipendente italiana. Scrivi oggetto e una frase: tono pacato e credibile, urgenza moderata, niente traduzioni letterali o sconti inventati.", + "In italiano, dai tre consigli a chi va per la prima volta a cena a casa di una collega in Italia. Riconosci le differenze tra famiglie e non presentare vino, baci o piatti specifici come obbligatori. Massimo 110 parole.", + "Scrivi una scena italiana di 70–90 parole: due vecchi amici perdono l’ultimo autobus. Mostra affetto attraverso bonarie prese in giro, senza nominare le emozioni né usare stereotipi regionali.", + "Spiega in italiano la differenza tra “Non devi venire” e “Devi non venire”. Indica perché la prima può risultare ambigua e proponi due alternative naturali che distinguano assenza di obbligo e divieto.", + ], +}; +export const LANGUAGE_CASES: LanguageCase[] = Object.entries(prompts).flatMap( + ([language, rows]) => + rows.map((prompt, i) => ({ + id: `${language}-${i + 1}`, + language, + category: categories[i]!, + prompt, + })), +); +LANGUAGE_CASES.push( + { + id: "MIX-1", + language: "MIX", + category: "translation", + prompt: + "Translate “I hear what you’re saying, but I can’t commit to Friday yet” into natural workplace Chinese (simplified), Thai (female speaker), and Japanese (polite). Preserve acknowledgement without implying agreement or a promise. Label the three versions; then explain one nuance per language in English, briefly.", + }, + { + id: "MIX-2", + language: "MIX", + category: "localization", + prompt: + "Write one natural, polite customer-service sentence in each of Spanish (Spain), French (France), German (Germany), and Italian: ask the customer to send the order number so you can check a delay. Use the formal address appropriate to each version, no promise of a refund. Label each language; give only the four sentences.", + }, + { + id: "MIX-3", + language: "MIX", + category: "translation", + prompt: + "Translate “You don’t have to decide today; I just wanted to make sure you knew your options” into Korean (polite), Modern Standard Arabic, Brazilian Portuguese, and Russian (formal address). Preserve reassurance and the absence of obligation. Label each language and give only the four translations.", + }, + { + id: "MIX-4", + language: "MIX", + category: "code_switching", + prompt: + "A multilingual group chat says: Mei: “这个周末我可能来不了。” Nok: “ไม่เป็นไร ไว้คราวหน้าก็ได้” Ken: “無理しないでね。” Ana: “Se der, avisa a gente.” In English, summarize what each person means and the overall social tone. Identify any uncertainty; do not invent reasons or treat any message as a firm promise.", + }, +); diff --git a/src/language/index.ts b/src/language/index.ts new file mode 100644 index 0000000..1d3eebc --- /dev/null +++ b/src/language/index.ts @@ -0,0 +1,141 @@ +import { appendFileSync, closeSync, mkdirSync, openSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { LANGUAGE_RUBRIC } from "./rubric"; +import { dirname } from "node:path"; +import { LANGUAGE_CASES, type LanguageCase } from "./cases"; + +export interface LanguageOptions { + root: string; + model: string; + save: string; + apiKey?: string; + concurrency?: number; + maxTokens?: number; + timeoutMs?: number; + cases?: LanguageCase[]; + log?: (line: string) => void; +} +/** A separate review artifact: language quality cannot be graded by exact matching. */ +export async function runLanguage(options: LanguageOptions) { + const { root, model, save } = options; + const concurrency = options.concurrency ?? 4; + const maxTokens = options.maxTokens ?? 500; + if ( + !Number.isSafeInteger(concurrency) || + concurrency < 1 || + !Number.isSafeInteger(maxTokens) || + maxTokens < 1 + ) + throw new Error("Concurrency and maxTokens must be positive integers"); + const cases = options.cases ?? LANGUAGE_CASES; + mkdirSync(dirname(save), { recursive: true }); + const fd = openSync(save, "wx"); // Refuse to overwrite an earlier response log. + closeSync(fd); + const startedAt = new Date().toISOString(); + const settings = { + model, + concurrency, + maxTokens, + temperature: 0, + top_p: 1, + reasoning_effort: "none", + enable_thinking: false, + enable_mtp: false, + enable_pld: false, + top_k: 0, + seed: 20261002, + timeoutMs: options.timeoutMs ?? 600000, + }; + appendFileSync( + save, + JSON.stringify({ + type: "manifest", + schema: "llmprobe-language-v1", + startedAt, + root, + settings, + cases, + casesSha256: createHash("sha256") + .update(JSON.stringify(cases)) + .digest("hex"), + rubric: LANGUAGE_RUBRIC, + }) + "\n", + ); + let next = 0, + completed = 0, + errors = 0; + await Promise.all( + Array.from({ length: Math.min(concurrency, cases.length) }, async () => { + while (next < cases.length) { + const tc = cases[next++]!; + const request = { + model, + messages: [{ role: "user", content: tc.prompt }], + max_tokens: maxTokens, + temperature: 0, + top_p: 1, + seed: 20261002, + reasoning_effort: "none", + enable_thinking: false, + enable_mtp: false, + enable_pld: false, + top_k: 0, + stream: false, + }; + const start = Date.now(); + let result: Record; + try { + const response = await fetch( + `${root.replace(/\/$/, "")}/v1/chat/completions`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(options.apiKey + ? { Authorization: `Bearer ${options.apiKey}` } + : {}), + }, + body: JSON.stringify(request), + signal: AbortSignal.timeout(options.timeoutMs ?? 600000), + }, + ); + const raw = await response.text(); + if (!response.ok) throw new Error(`HTTP ${response.status}: ${raw}`); + const body = JSON.parse(raw); + if ( + typeof body?.choices?.[0]?.message?.content !== "string" || + !body.choices[0].message.content.trim() + ) + throw new Error(`Missing nonempty answer: ${raw}`); + result = { status: "ok", response: body }; + } catch (error) { + errors++; + result = { status: "error", error: String(error) }; + } + appendFileSync( + save, + JSON.stringify({ + type: "response", + ...tc, + request, + ...result, + durationMs: Date.now() - start, + }) + "\n", + ); + completed++; + options.log?.( + `[${completed}/${cases.length}] ${tc.id} ${result.status} ${((Date.now() - start) / 1000).toFixed(1)}s`, + ); + } + }), + ); + const summary = { + type: "summary", + completed, + errors, + finishedAt: new Date().toISOString(), + save, + }; + appendFileSync(save, JSON.stringify(summary) + "\n"); + return summary; +} diff --git a/src/language/language.test.ts b/src/language/language.test.ts new file mode 100644 index 0000000..3edb17a --- /dev/null +++ b/src/language/language.test.ts @@ -0,0 +1,99 @@ +import { createServer } from "node:http"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { LANGUAGE_CASES } from "./cases"; +import { runLanguage } from "./index"; + +it("contains 96 language cases and four cross-language cases with unique IDs", () => { + expect(LANGUAGE_CASES).toHaveLength(100); + expect(new Set(LANGUAGE_CASES.map((c) => c.id)).size).toBe(100); + const languages = new Set( + LANGUAGE_CASES.map((c) => c.language).filter((l) => l !== "MIX"), + ); + expect(languages.size).toBe(12); + for (const language of languages) + expect(LANGUAGE_CASES.filter((c) => c.language === language)).toHaveLength( + 8, + ); +}); + +it("caps concurrency, preserves complete answers and errors, and never overwrites logs", async () => { + let active = 0, + peak = 0; + const bodies: any[] = []; + const server = createServer(async (req, res) => { + let raw = ""; + for await (const chunk of req) raw += chunk; + const body = JSON.parse(raw); + bodies.push(body); + peak = Math.max(peak, ++active); + await new Promise((resolve) => setTimeout(resolve, 20)); + active--; + if (body.messages[0].content === "bad") { + res.writeHead(500); + res.end("fixture failure"); + return; + } + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + choices: [ + { + message: { content: "ภาษาไทย — العربية — 日本語" }, + finish_reason: "stop", + }, + ], + usage: { completion_tokens: 12 }, + }), + ); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const dir = mkdtempSync(join(tmpdir(), "llmprobe-language-")); + const address = server.address() as { port: number }; + const save = join(dir, "run.jsonl"); + try { + const summary = await runLanguage({ + root: `http://127.0.0.1:${address.port}`, + model: "test", + save, + concurrency: 4, + cases: LANGUAGE_CASES.slice(0, 9).map((c, i) => ({ + ...c, + prompt: i === 3 ? "bad" : c.prompt, + })), + }); + expect(peak).toBe(4); + expect(summary.errors).toBe(1); + expect(summary.completed).toBe(9); + expect( + bodies.every( + (b) => + b.max_tokens === 500 && + b.enable_thinking === false && + b.enable_mtp === false, + ), + ).toBe(true); + const text = readFileSync(save, "utf8"); + const rows = text + .trim() + .split("\n") + .map((x) => JSON.parse(x)); + expect(rows.filter((x) => x.type === "response")).toHaveLength(9); + expect( + rows.some( + (x) => + x.response?.choices[0].message.content === + "ภาษาไทย — العربية — 日本語", + ), + ).toBe(true); + await expect( + runLanguage({ root: "unused", model: "test", save }), + ).rejects.toThrow(); + expect(readFileSync(save, "utf8")).toBe(text); + } finally { + server.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/src/language/rubric.ts b/src/language/rubric.ts new file mode 100644 index 0000000..7155b1a --- /dev/null +++ b/src/language/rubric.ts @@ -0,0 +1,60 @@ +/** Review protocol is fixed before collecting model answers. */ +export const LANGUAGE_RUBRIC = { + version: 1, + judge: + "Offline reviewer; record judge identity and review date with scores. No automatic grade.", + dimensions: { + accuracy: + "Grammar, vocabulary, semantic fidelity and correctness of linguistic explanations.", + naturalness: + "Idiomatic wording, cohesion and fluency in the requested variety; avoid translationese.", + register_style: + "Requested formality, relationship, speaker identity, voice and genre.", + nuance_culture: + "Pragmatic meaning, ambiguity, politeness and contextual cultural fit; avoid unwarranted generalizations.", + task_fulfillment: + "All requested outputs, target language(s), constraints and meaning preserved; no invented facts.", + }, + anchors: { + 4: "Strong: accurate and natural; fully appropriate with no material weakness.", + 3: "Good: minor localized issue; intended meaning and use remain sound.", + 2: "Mixed: noticeable errors or omissions; partly useful but needs revision.", + 1: "Weak: major errors, unnatural language or poor fit; substantial revision needed.", + 0: "Fails the dimension: absent, unintelligible, wrong-language or fundamentally misleading.", + }, + category_focus: { + register: + "Meaning stays constant while interpersonal distance and address change appropriately.", + idiom: + "Explain pragmatic function, supply a natural example and avoid a literal-only interpretation.", + pragmatics: + "Offer plausible alternatives and context needed to distinguish them; no mind-reading.", + grammar: + "Correct the actual errors, preserve meaning, explain accurately and respect legitimate variation.", + localization: + "Adapt copy to local genre and tone; preserve mild urgency without fabricated offer details.", + culture: + "Practical context-sensitive advice, with variation acknowledged and no rigid national stereotypes.", + creative: + "Coherent scene, affection shown indirectly, gentle teasing and requested length/style.", + ambiguity: + "Distinguish meanings precisely; do not manufacture a standard reading for malformed text.", + translation: + "Preserve uncertainty, scope, obligation and politeness in every requested target language.", + code_switching: + "Interpret all languages faithfully, retain uncertainty and avoid invented context.", + }, + protocol: [ + "Read every prompt and complete answer. Record all five integer scores and a case-specific explanation citing the answer.", + "Hide model labels and alternate answer order during review when practicable. Identify this as one model-based reviewer, not a native-speaker panel, if applicable.", + "Use the same anchors for both models. Do not prefer verbosity, Markdown or a particular dialect unless the prompt calls for it.", + "Use a holistic assessment within each dimension; identify primary defects rather than repeatedly deducting for the same superficial issue.", + "A short deviation from a word count is a minor task issue; Thai/CJK segmentation is not a reliable cross-language metric.", + "Log finish_reason=length separately. Score only visible content; never treat HTTP errors or missing answers as evidence of language quality.", + "Case total is sum of five dimensions / 20 * 100. Average eight cases within each language, then average the twelve language means equally (96 cases).", + "Report the four cross-language cases separately. Do not silently reweight the overall score with those cases.", + "Compare paired cases and report wins/ties/losses, dimension means, per-language means, truncations and failures.", + "A run is incomplete if any required answer is missing. Do not declare an overall winner on incomplete runs.", + "This is an original, small diagnostic suite with one generation per prompt. It does not isolate quantization effects or establish a universal model ranking.", + ], +};