evalstone/bash/fingerprint/tools/llm-fingerprint-detector/test/api.integration.test.mjs

/**
 * End-to-end test against a local mock OpenAI-compatible server: adapter
 * detection, concurrent sampling, normalization, aggregation and verify().
 */
import assert from 'node:assert/strict'
import { createServer } from 'node:http'
import { after, before, test } from 'node:test'

import { compare, fingerprint, verify } from '../dist/api.js'

/** Two simulated "models" with different answer distributions. */
const MODEL_BEHAVIOR = {
  'mock-alpha': {
    'random-number-1-100': () => (Math.random() < 0.8 ? '42' : '73'),
    'random-number-1-10': () => '7',
    'random-letter': () => 'k',
    'random-color': () => (Math.random() < 0.6 ? 'blue' : 'teal'),
    'coin-flip': () => (Math.random() < 0.7 ? 'Heads' : 'Tails'),
    'random-animal': () => 'octopus',
    'random-city': () => 'Tokyo',
    'favorite-number': () => '42',
  },
  'mock-beta': {
    'random-number-1-100': () => (Math.random() < 0.8 ? '57' : '3'),
    'random-number-1-10': () => '3',
    'random-letter': () => 'm',
    'random-color': () => 'crimson',
    'coin-flip': () => 'Tails',
    'random-animal': () => 'giraffe',
    'random-city': () => 'Paris',
    'favorite-number': () => '9',
  },
}

/** Map a user prompt back to the battery task (rough but sufficient for the mock). */
function taskFromPrompt(prompt) {
  const p = prompt.toLowerCase()
  if (p.includes('100') || p.includes('1 到 100') || p.includes('1 至 100') || p.includes('1 到 100'))
    return 'random-number-1-100'
  if (p.includes('10') || p.includes('1 到 10')) return 'random-number-1-10'
  if (p.includes('letter') || p.includes('字母')) return 'random-letter'
  if (p.includes('color') || p.includes('颜色')) return 'random-color'
  if (p.includes('coin') || p.includes('硬币')) return 'coin-flip'
  if (p.includes('animal') || p.includes('动物')) return 'random-animal'
  if (p.includes('city') || p.includes('城市')) return 'random-city'
  if (p.includes('favorite') || p.includes('favourite') || p.includes('喜欢') || p.includes('最爱'))
    return 'favorite-number'
  return 'random-number-1-100'
}

let server
let baseUrl
let requestCount = 0
let sawReasoningField = 0

before(async () => {
  server = createServer((req, res) => {
    let body = ''
    req.on('data', (chunk) => (body += chunk))
    req.on('end', () => {
      requestCount += 1
      const payload = JSON.parse(body)

      if (req.url !== '/v1/chat/completions') {
        res.writeHead(404).end('not found')
        return
      }
      if (req.headers.authorization !== 'Bearer test-key') {
        res.writeHead(401, { 'content-type': 'application/json' })
        res.end(JSON.stringify({ error: { message: 'bad key' } }))
        return
      }
      // The mock accepts the OpenRouter-style reasoning field (counts it for assertions).
      if (payload.reasoning !== undefined) sawReasoningField += 1

      const behavior = MODEL_BEHAVIOR[payload.model]
      if (!behavior) {
        res.writeHead(404, { 'content-type': 'application/json' })
        res.end(JSON.stringify({ error: { message: `unknown model ${payload.model}` } }))
        return
      }
      const userPrompt = payload.messages.find((m) => m.role === 'user')?.content ?? ''
      const answer = behavior[taskFromPrompt(userPrompt)]()
      res.writeHead(200, { 'content-type': 'application/json' })
      res.end(
        JSON.stringify({
          choices: [{ message: { role: 'assistant', content: answer } }],
          usage: { prompt_tokens: 25, completion_tokens: 2 },
        }),
      )
    })
  })
  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
  baseUrl = `http://127.0.0.1:${server.address().port}/v1`
})

after(() => server.close())

test('fingerprint(): collects distributions from a live endpoint', async () => {
  const run = await fingerprint(
    { baseUrl, model: 'mock-alpha', apiKey: 'test-key' },
    { cells: 4, samplesPerCell: 12, concurrency: 6 },
  )
  assert.equal(run.errorCount, 0)
  assert.equal(run.fingerprint.model, 'mock-alpha')
  assert.equal(run.fingerprint.protocol, 'one-token/v1')
  assert.equal(Object.keys(run.fingerprint.cells).length, 4)

  const rn100 = run.fingerprint.cells['random-number-1-100:en']
  assert.ok(rn100, 'expected the top-priority cell to be probed')
  assert.equal(rn100.validCount, 12)
  assert.ok(rn100.counts['42'] > 0, 'mock-alpha answers 42 most of the time')
  assert.ok(run.adapter.postReasoning === false)
  assert.ok(sawReasoningField > 0, 'adapter probe should have tried the reasoning field')
})

test('fingerprint(): progress callback covers all requests', async () => {
  const events = []
  const run = await fingerprint(
    { baseUrl, model: 'mock-alpha', apiKey: 'test-key' },
    { cells: 2, samplesPerCell: 5, onProgress: (e) => events.push(e) },
  )
  const sampling = events.filter((e) => e.stage === 'sampling')
  assert.equal(sampling.length, 10)
  assert.equal(sampling.at(-1).done, 10)
  assert.equal(sampling.at(-1).total, 10)
  assert.equal(run.durationMs >= 0, true)
})

test('verify(): same mock model → match, different mock model → mismatch', async () => {
  const options = { cells: 6, samplesPerCell: 20, concurrency: 8 }
  const referenceRun = await fingerprint({ baseUrl, model: 'mock-alpha', apiKey: 'test-key' }, options)

  const same = await verify(
    { baseUrl, model: 'mock-alpha', apiKey: 'test-key' },
    referenceRun.fingerprint,
    options,
  )
  assert.equal(same.verdict, 'match')
  assert.ok(same.meanJsd < 0.25, `same model meanJsd should be small, got ${same.meanJsd}`)

  const different = await verify(
    { baseUrl, model: 'mock-beta', apiKey: 'test-key' },
    referenceRun.fingerprint,
    options,
  )
  assert.equal(different.verdict, 'mismatch')
  assert.ok(different.meanJsd > 0.35, `different model meanJsd should be large, got ${different.meanJsd}`)
})

test('compare(): flags protocol mismatch', async () => {
  const runA = await fingerprint(
    { baseUrl, model: 'mock-alpha', apiKey: 'test-key' },
    { cells: 4, samplesPerCell: 12 },
  )
  const foreign = { ...runA.fingerprint, protocol: 'someone-elses-protocol' }
  const result = compare(runA.fingerprint, foreign)
  assert.equal(result.protocolMismatch, true)
  assert.equal(result.verdict, 'match') // identical distributions still match
})

test('fingerprint(): invalid API key aborts the run with an auth error', async () => {
  await assert.rejects(
    fingerprint(
      { baseUrl, model: 'mock-alpha', apiKey: 'wrong-key' },
      { cells: 2, samplesPerCell: 3 },
    ),
    (error) => {
      assert.match(error.message, /401/)
      return true
    },
  )
})

test('fingerprint(): AbortSignal cancels the run', async () => {
  const controller = new AbortController()
  const promise = fingerprint(
    { baseUrl, model: 'mock-alpha', apiKey: 'test-key' },
    {
      cells: 8,
      samplesPerCell: 50,
      onProgress: (e) => {
        if (e.stage === 'sampling' && e.done >= 5) controller.abort()
      },
      signal: controller.signal,
    },
  )
  await assert.rejects(promise, (error) => {
    assert.equal(error.name, 'ProbeRunError')
    assert.equal(error.reason, 'aborted')
    return true
  })
})

test('endpoint validation: bad base URL rejects before any request', async () => {
  await assert.rejects(fingerprint({ baseUrl: '', model: 'x' }), /baseUrl is empty/)
  await assert.rejects(fingerprint({ baseUrl: 'https://ok.example/v1', model: ' ' }), /model is empty/)
  assert.ok(requestCount > 0)
})