test.js

/**
 * test.js — Provider connectivity and agent smoke tests
 *
 * Usage:
 *   node test.js              # runs all tests for configured providers
 *   node test.js --provider anthropic   # test a specific provider
 *   node test.js --quick              # ping test only (no full agent run)
 *
 * What it tests:
 *   1. Ping   — can we reach the provider API?
 *   2. Agent  — can the agent parse agent.yaml and load skills?
 *   3. Review — run a mock PR review on a known diff (local, no GitHub needed)
 */

import { PROVIDER_CHAIN, isNvidiaProvider, runWithNvidia } from './providers.js';
import { query } from 'gitclaw';
import path from 'path';
import { fileURLToPath } from 'url';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const AGENT_DIR = path.resolve(__dirname, '.');
const args      = process.argv.slice(2);
const quickOnly = args.includes('--quick');
const onlyProv  = args.find((a, i) => args[i - 1] === '--provider');

// ─── Colors ──────────────────────────────────────────────────────────────────

const GREEN  = (s) => `\x1b[32m${s}\x1b[0m`;
const RED    = (s) => `\x1b[31m${s}\x1b[0m`;
const YELLOW = (s) => `\x1b[33m${s}\x1b[0m`;
const DIM    = (s) => `\x1b[2m${s}\x1b[0m`;
const BOLD   = (s) => `\x1b[1m${s}\x1b[0m`;

// ─── Mock diff for review testing ────────────────────────────────────────────

const MOCK_DIFF = `
diff --git a/src/auth.js b/src/auth.js
+++ b/src/auth.js
@@ -10,6 +10,12 @@ const express = require('express');
+const API_KEY = "sk-prod-abc123def456";  // TODO: move to env
+
+app.get('/users', async (req, res) => {
+  const id = req.query.id;
+  const result = await db.query("SELECT * FROM users WHERE id = " + id);
+  console.log("fetched user:", result);
+  res.json(result);
+});
`.trim();

// ─── Tests ───────────────────────────────────────────────────────────────────

async function testPing(provider) {
  const start = Date.now();

  if (isNvidiaProvider(provider)) {
    // Direct OpenAI-compat ping for NVIDIA NIM
    const { default: OpenAI } = await import('openai');
    const client = new OpenAI({ apiKey: process.env.NVIDIA_API_KEY, baseURL: provider.baseUrl });
    const res = await client.chat.completions.create({
      model: provider.modelId,
      messages: [{ role: 'user', content: 'Reply with the single word: PONG' }],
      max_tokens: 10,
    });
    const reply = res.choices[0]?.message?.content || '';
    return { ok: reply.toLowerCase().includes('pong'), ms: Date.now() - start, reply };
  }

  // For gitclaw-native providers, run a minimal query
  const session = query({ dir: AGENT_DIR, prompt: 'Reply with the single word: PONG', model: provider.model });
  let reply = '';
  for await (const msg of session) {
    if (msg.type === 'assistant') reply = msg.content;
  }
  return { ok: reply.toLowerCase().includes('pong'), ms: Date.now() - start, reply };
}

async function testAgentLoad() {
  // Just verify agent.yaml + skills parse without error
  try {
    const session = query({
      dir: AGENT_DIR,
      prompt: 'List the skills available to you in one line.',
      model: 'anthropic:claude-sonnet-4-5-20250929',
    });
    let loaded = false;
    for await (const msg of session) {
      if (msg.type === 'system' && msg.subtype === 'session_start') loaded = true;
      if (msg.type === 'assistant') break; // got a response, good enough
    }
    return { ok: loaded };
  } catch (err) {
    return { ok: false, error: err.message };
  }
}

async function testReview(provider) {
  const start = Date.now();
  const prompt = `
You are ReviewAgent. Review this diff and respond with:
1. One CRITICAL finding (hardcoded secret)
2. One HIGH finding (SQL injection)
3. Verdict: BLOCKED

Diff to review:
${MOCK_DIFF}
`.trim();

  let reply = '';
  try {
    if (isNvidiaProvider(provider)) {
      const sysPrompt = 'You are ReviewAgent, an AI code reviewer. Be concise.';
      for await (const msg of runWithNvidia(provider, sysPrompt, prompt)) {
        if (msg.type === 'assistant') reply = msg.content;
      }
    } else {
      const session = query({ dir: AGENT_DIR, prompt, model: provider.model });
      for await (const msg of session) {
        if (msg.type === 'assistant') reply += msg.content;
      }
    }

    const hasCritical = reply.toUpperCase().includes('CRITICAL') || reply.toLowerCase().includes('hardcoded');
    const hasHigh     = reply.toUpperCase().includes('HIGH') || reply.toLowerCase().includes('injection');
    const hasBlocked  = reply.toUpperCase().includes('BLOCKED');
    const ok          = hasCritical && hasHigh;

    return {
      ok,
      ms:          Date.now() - start,
      hasCritical,
      hasHigh,
      hasBlocked,
      preview:     reply.slice(0, 120).replace(/\n/g, ' '),
    };
  } catch (err) {
    return { ok: false, ms: Date.now() - start, error: err.message };
  }
}

// ─── Runner ──────────────────────────────────────────────────────────────────

async function runTests() {
  console.log(BOLD('\n╔════════════════════════════════════════════╗'));
  console.log(BOLD('║   ReviewAgent — Provider Test Suite        ║'));
  console.log(BOLD('╚════════════════════════════════════════════╝\n'));

  const providers = PROVIDER_CHAIN.filter(p => {
    if (onlyProv && p.name.toLowerCase() !== onlyProv.toLowerCase()) return false;
    return !!process.env[p.envKey];
  });

  if (providers.length === 0) {
    console.error(RED('  No configured providers found. Set at least one API key:'));
    for (const p of PROVIDER_CHAIN) console.error(DIM(`    ${p.envKey}=${p.name}`));
    process.exit(1);
  }

  const results = [];

  for (const provider of providers) {
    console.log(BOLD(`\n▶ ${provider.name}  ${DIM(`(tier ${provider.tier})`)}`));
    const row = { provider: provider.name, tier: provider.tier };

    // 1. Ping
    process.stdout.write('  📡  Ping    ... ');
    try {
      const r = await testPing(provider);
      row.ping = r.ok ? GREEN(`PASS ${r.ms}ms`) : YELLOW(`PARTIAL ${r.ms}ms`);
      console.log(row.ping, DIM(`"${r.reply?.trim().slice(0, 30)}"`));
    } catch (err) {
      row.ping = RED('FAIL');
      console.log(row.ping, DIM(err.message.slice(0, 60)));
      results.push(row);
      continue; // skip deeper tests if ping fails
    }

    if (quickOnly) { results.push(row); continue; }

    // 2. Review test
    process.stdout.write('  🔍  Review  ... ');
    try {
      const r = await testReview(provider);
      const label = r.ok ? GREEN('PASS') : YELLOW('PARTIAL');
      console.log(`${label} ${r.ms}ms  critical=${r.hasCritical} high=${r.hasHigh} blocked=${r.hasBlocked}`);
      if (r.error) console.log(DIM(`    error: ${r.error}`));
      if (r.preview) console.log(DIM(`    preview: "${r.preview}"`));
      row.review = r.ok ? 'PASS' : 'PARTIAL';
    } catch (err) {
      row.review = 'FAIL';
      console.log(RED('FAIL'), DIM(err.message.slice(0, 60)));
    }

    results.push(row);
  }

  // Summary table
  console.log(BOLD('\n╔════════════════════════════════════════════╗'));
  console.log(BOLD('║   Results                                  ║'));
  console.log(BOLD('╠════════════════════════════════════════════╣'));
  for (const r of results) {
    const review = r.review ? `review=${r.review}` : 'review=skipped';
    console.log(`║  Tier ${r.tier}  ${r.provider.padEnd(24)} ${review.padEnd(15)} ║`);
  }
  console.log(BOLD('╚════════════════════════════════════════════╝\n'));

  const passed = results.filter(r => r.review === 'PASS' || r.ping?.includes('PASS')).length;
  console.log(`  ${passed}/${results.length} providers operational\n`);
}

runTests().catch(err => {
  console.error(RED(`\n  Fatal: ${err.message}`));
  process.exit(1);
});