Testing API

query GetTestCases($promptId: ID!) {
  testCases(promptId: $promptId) {
    id
    name
    description
    input
    expectedOutput
    totalRuns
    passRate
    avgScore
    lastRun
    lastPassed
    createdAt
    updatedAt
  }
}

query GetTestCase($id: ID!) {
  testCase(id: $id) {
    id
    name
    description
    input
    expectedOutput
    totalRuns
    passRate
    avgScore
    prompt {
      id
      name
    }
  }
}

query GetTestCaseExecutions($testCaseId: ID!, $limit: Int, $offset: Int) {
  testCaseExecutions(testCaseId: $testCaseId, limit: $limit, offset: $offset) {
    id
    passed
    score
    actualOutput
    expectedOutput
    exactMatch
    semanticScore
    lengthDiff
    tokenDiff
    isRegression
    regressionType
    promptVersion
    runDuration
    createdAt
  }
}

query GetTestCaseStats($testCaseId: ID!) {
  testCaseStats(testCaseId: $testCaseId) {
    totalRuns
    passRate
    avgScore
    recentRuns {
      id
      passed
      score
      promptVersion
      createdAt
    }
    regressions {
      id
      passed
      score
      regressionType
      promptVersion
      createdAt
    }
    trendData {
      date
      totalRuns
      passRate
      avgScore
    }
  }
}

mutation CreateTestCase($input: TestCaseInput!) {
  createTestCase(input: $input) {
    id
    name
    description
    input
    expectedOutput
    createdAt
  }
}

# Variables
{
  "input": {
    "promptId": "cm123456789",
    "name": "Positive sentiment detection",
    "description": "Tests detection of positive sentiment in customer feedback",
    "input": {
      "text": "I absolutely love this product! Best purchase ever."
    },
    "expectedOutput": "positive"
  }
}

mutation UpdateTestCase($id: ID!, $input: TestCaseInput!) {
  updateTestCase(id: $id, input: $input) {
    id
    name
    description
    input
    expectedOutput
    updatedAt
  }
}

mutation DeleteTestCase($id: ID!) {
  deleteTestCase(id: $id)
}

mutation RunTestCase($input: RunTestCaseInput!) {
  runTestCase(input: $input) {
    testCaseExecutionId
    executionId
    passed
    score
    actualOutput
    expectedOutput
    outputDiff
    exactMatch
    semanticScore
    lengthDiff
    tokenDiff
    latencyMs
    costUsd
    isRegression
    regressionType
  }
}

# Variables
{
  "input": {
    "testCaseId": "cm123456789",
    "promptVersion": 3  // Optional: defaults to latest
  }
}

mutation BatchRunTests($input: BatchRunTestsInput!) {
  batchRunTests(input: $input) {
    testCaseExecutionId
    executionId
    passed
    score
    actualOutput
    isRegression
    regressionType
  }
}

# Variables
{
  "input": {
    "promptId": "cm123456789",
    "testCaseIds": [
      "test1",
      "test2",
      "test3"
    ],
    "promptVersion": 4  // Optional
  }
}

mutation UpdateExpectedOutput($testCaseId: ID!, $output: String!) {
  updateExpectedOutput(testCaseId: $testCaseId, output: $output) {
    id
    expectedOutput
    updatedAt
  }
}

// Test fails when it previously passed
if (!currentPassed && historicalPassRate > 0.8) {
  regressionType = 'FAILED'
  isRegression = true
}

// Score drops significantly
if (currentScore < recentAvgScore * 0.9) {
  regressionType = 'SCORE_DROP'
  isRegression = true
}

// Output length changes dramatically
const lengthChange = Math.abs(currentLength - recentAvgLength) / recentAvgLength
if (lengthChange > 0.3) {
  regressionType = 'LENGTH_CHANGE'
  isRegression = true
}

// Create multiple test cases for different scenarios
const testCases = [
  {
    name: "Positive sentiment",
    input: { text: "I love this!" },
    expectedOutput: "positive"
  },
  {
    name: "Negative sentiment",
    input: { text: "This is terrible." },
    expectedOutput: "negative"
  },
  {
    name: "Neutral sentiment",
    input: { text: "It's okay." },
    expectedOutput: "neutral"
  }
]

for (const tc of testCases) {
  await createTestCase({ promptId, ...tc })
}

// Batch run all tests for a prompt
const results = await batchRunTests({
  promptId: "cm123456789",
  testCaseIds: ["test1", "test2", "test3"]
})

// Check for failures or regressions
const failures = results.filter(r => !r.passed)
const regressions = results.filter(r => r.isRegression)

if (failures.length > 0) {
  console.log('Failed tests:', failures.length)
}

if (regressions.length > 0) {
  console.log('Regressions detected:', regressions)
}

const stats = await getTestCaseStats({ testCaseId })

console.log('Pass rate:', stats.passRate + '%')
console.log('Average score:', stats.avgScore)
console.log('Recent regressions:', stats.regressions.length)

// Analyze trend data
stats.trendData.forEach(point => {
  console.log(`${point.date}: ${point.passRate}% pass rate`)
})

// Accept new output as the expected baseline
await updateExpectedOutput({
  testCaseId: "test123",
  output: newExpectedOutput
})

query GetTestBranches($testCaseId: ID!) {
  testBranches(testCaseId: $testCaseId) {
    id
    name
    description
    provider
    model
    temperature
    maxTokens
    topP
    isControl
    isActive
    totalRuns
    avgLatency
    avgCost
    avgQualityScore
    avgTokensPerSec
    consistencyScore
    passRate
    createdAt
  }
}

query GetBranchExecutions($branchId: ID!, $limit: Int, $offset: Int) {
  testBranchExecutions(branchId: $branchId, limit: $limit, offset: $offset) {
    id
    output
    status
    latencyMs
    timeToFirstToken
    tokensPerSecond
    inputTokens
    outputTokens
    estimatedCost
    qualityScore
    similarityScore
    clarityScore
    validJson
    promptVersion
    createdAt
  }
}

mutation CreateTestBranch($input: TestBranchInput!) {
  createTestBranch(input: $input) {
    id
    name
    model
    temperature
  }
}

# Variables
{
  "input": {
    "testCaseId": "tc_123",
    "name": "GPT-4 High Temperature",
    "description": "Testing GPT-4 with higher temperature for creativity",
    "provider": "openai",
    "model": "gpt-4-turbo-preview",
    "temperature": 0.9,
    "maxTokens": 1000,
    "topP": 0.95,
    "isControl": false
  }
}

mutation UpdateTestBranch($id: ID!, $input: TestBranchInput!) {
  updateTestBranch(id: $id, input: $input) {
    id
    name
    temperature
    maxTokens
  }
}

mutation DeleteTestBranch($id: ID!) {
  deleteTestBranch(id: $id)
}

mutation RunABTest($input: RunABTestInput!) {
  runABTest(input: $input) {
    branches {
      branchId
      branchName
      model
      execution {
        id
        output
        status
      }
      qualityScore
      latencyMs
      estimatedCost
      tokensPerSecond
      rank
    }
    winner {
      branchId
      branchName
      model
      qualityScore
      latencyMs
      estimatedCost
    }
    criteria
  }
}

# Variables
{
  "input": {
    "testCaseId": "tc_123",
    "branchIds": ["branch_1", "branch_2", "branch_3"],
    "promptVersion": 2,
    "winnerCriteria": "balanced"
  }
}

import { gql } from 'graphql-request'

// 1. Create a test case
const CREATE_TEST = gql`
  mutation CreateTestCase($input: TestCaseInput!) {
    createTestCase(input: $input) {
      id
    }
  }
`

const testCase = await client.request(CREATE_TEST, {
  input: {
    promptId: "prompt_123",
    name: "Sentiment Analysis Test",
    input: {
      text: "This product exceeded my expectations!"
    },
    expectedOutput: "positive"
  }
})

// 2. Create test branches with different configurations
const CREATE_BRANCH = gql`
  mutation CreateBranch($input: TestBranchInput!) {
    createTestBranch(input: $input) {
      id
    }
  }
`

const branches = await Promise.all([
  // GPT-4 Turbo
  client.request(CREATE_BRANCH, {
    input: {
      testCaseId: testCase.createTestCase.id,
      name: "GPT-4 Turbo",
      provider: "openai",
      model: "gpt-4-turbo-preview",
      temperature: 0.7
    }
  }),
  // Claude Sonnet
  client.request(CREATE_BRANCH, {
    input: {
      testCaseId: testCase.createTestCase.id,
      name: "Claude Sonnet",
      provider: "anthropic",
      model: "claude-3-sonnet-20240229",
      temperature: 0.7
    }
  }),
  // Gemini Pro
  client.request(CREATE_BRANCH, {
    input: {
      testCaseId: testCase.createTestCase.id,
      name: "Gemini Pro",
      provider: "gemini",
      model: "gemini-pro",
      temperature: 0.7
    }
  })
])

// 3. Run A/B test
const RUN_AB_TEST = gql`
  mutation RunABTest($input: RunABTestInput!) {
    runABTest(input: $input) {
      winner {
        branchName
        model
        qualityScore
        latencyMs
        estimatedCost
      }
      branches {
        branchName
        model
        qualityScore
        latencyMs
        estimatedCost
        rank
      }
      criteria
    }
  }
`

const result = await client.request(RUN_AB_TEST, {
  input: {
    testCaseId: testCase.createTestCase.id,
    branchIds: branches.map(b => b.createTestBranch.id),
    winnerCriteria: "balanced"
  }
})

// 4. Analyze results
console.log('Winner:', result.runABTest.winner.branchName)
console.log('  Model:', result.runABTest.winner.model)
console.log('  Quality:', result.runABTest.winner.qualityScore)
console.log('  Latency:', result.runABTest.winner.latencyMs + 'ms')
console.log('  Cost: $' + result.runABTest.winner.estimatedCost)

console.log('\nAll Results:')
result.runABTest.branches.forEach(branch => {
  console.log(`${branch.rank}. ${branch.branchName}`)
  console.log(`   Quality: ${branch.qualityScore}`)
  console.log(`   Speed: ${branch.latencyMs}ms`)
  console.log(`   Cost: $${branch.estimatedCost}`)
})

// 5. Get detailed execution history
const GET_EXECUTIONS = gql`
  query GetBranchExecutions($branchId: ID!, $limit: Int) {
    testBranchExecutions(branchId: $branchId, limit: $limit) {
      output
      qualityScore
      latencyMs
      estimatedCost
      createdAt
    }
  }
`

const winnerId = branches[0].createTestBranch.id
const executions = await client.request(GET_EXECUTIONS, {
  branchId: winnerId,
  limit: 10
})

console.log('\nRecent executions for winner:')
executions.testBranchExecutions.forEach((exec, i) => {
  console.log(`${i + 1}. Quality: ${exec.qualityScore}, Latency: ${exec.latencyMs}ms`)
})

// Run the same A/B test 5 times
const iterations = 5
const results = []

for (let i = 0; i < iterations; i++) {
  const result = await client.request(RUN_AB_TEST, {
    input: {
      testCaseId: testCase.id,
      branchIds: branchIds,
      winnerCriteria: "balanced"
    }
  })
  results.push(result.runABTest)

  // Small delay between iterations
  await new Promise(resolve => setTimeout(resolve, 1000))
}

// Aggregate results
const winnerCounts = {}
results.forEach(r => {
  const winner = r.winner.branchName
  winnerCounts[winner] = (winnerCounts[winner] || 0) + 1
})

console.log('Winner distribution across', iterations, 'runs:')
Object.entries(winnerCounts).forEach(([name, count]) => {
  console.log(`  ${name}: ${count}/${iterations} (${(count/iterations*100).toFixed(1)}%)`)
})

// Calculate average metrics per branch
const branchStats = {}
results.forEach(result => {
  result.branches.forEach(branch => {
    if (!branchStats[branch.branchName]) {
      branchStats[branch.branchName] = {
        quality: [],
        latency: [],
        cost: []
      }
    }
    branchStats[branch.branchName].quality.push(branch.qualityScore)
    branchStats[branch.branchName].latency.push(branch.latencyMs)
    branchStats[branch.branchName].cost.push(branch.estimatedCost)
  })
})

console.log('\nAverage metrics across all runs:')
Object.entries(branchStats).forEach(([name, stats]) => {
  const avgQuality = stats.quality.reduce((a, b) => a + b) / stats.quality.length
  const avgLatency = stats.latency.reduce((a, b) => a + b) / stats.latency.length
  const avgCost = stats.cost.reduce((a, b) => a + b) / stats.cost.length

  console.log(`${name}:`)
  console.log(`  Avg Quality: ${avgQuality.toFixed(3)}`)
  console.log(`  Avg Latency: ${avgLatency.toFixed(0)}ms`)
  console.log(`  Avg Cost: $${avgCost.toFixed(5)}`)
})

import { gql } from 'graphql-request'

// 1. Create test cases for a sentiment analysis prompt
const CREATE_TEST = gql`
  mutation CreateTestCase($input: TestCaseInput!) {
    createTestCase(input: $input) {
      id
      name
    }
  }
`

const testCases = [
  {
    promptId: "cm123",
    name: "Strongly positive",
    input: { text: "This is absolutely amazing! Best ever!" },
    expectedOutput: "positive"
  },
  {
    promptId: "cm123",
    name: "Subtly negative",
    input: { text: "I guess it's okay, but I expected better." },
    expectedOutput: "negative"
  }
]

const createdTests = await Promise.all(
  testCases.map(tc => client.request(CREATE_TEST, { input: tc }))
)

// 2. Make changes to the prompt
// ... update prompt content ...

// 3. Run all tests against the new version
const BATCH_RUN = gql`
  mutation BatchRun($input: BatchRunTestsInput!) {
    batchRunTests(input: $input) {
      testCaseExecutionId
      passed
      score
      isRegression
      regressionType
      actualOutput
    }
  }
`

const results = await client.request(BATCH_RUN, {
  input: {
    promptId: "cm123",
    testCaseIds: createdTests.map(t => t.createTestCase.id)
  }
})

// 4. Analyze results
const summary = {
  total: results.batchRunTests.length,
  passed: results.batchRunTests.filter(r => r.passed).length,
  regressions: results.batchRunTests.filter(r => r.isRegression)
}

console.log(`Test Results: ${summary.passed}/${summary.total} passed`)

if (summary.regressions.length > 0) {
  console.log('Regressions detected:')
  summary.regressions.forEach(r => {
    console.log(`  - Type: ${r.regressionType}, Score: ${r.score}`)
  })
}

// 5. Get detailed stats for monitoring
const STATS = gql`
  query GetStats($testCaseId: ID!) {
    testCaseStats(testCaseId: $testCaseId) {
      totalRuns
      passRate
      avgScore
      trendData {
        date
        passRate
        avgScore
      }
    }
  }
`

for (const test of createdTests) {
  const stats = await client.request(STATS, {
    testCaseId: test.createTestCase.id
  })
  console.log(`${test.createTestCase.name}:`)
  console.log(`  Pass rate: ${stats.testCaseStats.passRate}%`)
  console.log(`  Avg score: ${stats.testCaseStats.avgScore}`)
}