40.5 Contract tests for schema compliance

Contract Testing

Contract tests verify that model outputs match your schema. Run them in CI to catch regressions.

Implementation

// contract-tests.ts
describe('PromptAPI Contracts', () => {
  const testCases = loadContractTestCases();
  
  for (const testCase of testCases) {
    it(`${testCase.name} produces valid output`, async () => {
      const output = await api.execute(testCase.input);
      
      // Validate against schema
      const result = testCase.schema.safeParse(output);
      expect(result.success).toBe(true);
      
      // Validate business rules
      if (testCase.invariants) {
        for (const invariant of testCase.invariants) {
          expect(invariant(output)).toBe(true);
        }
      }
    });
  }
});

// Run with multiple temperatures to catch edge cases
async function fuzzSchemaCompliance(api, schema, n = 10) {
  const results = await Promise.all(
    Array(n).fill(null).map(() => 
      api.execute({ temperature: 0.9 })
    )
  );
  
  const failures = results.filter(r => !schema.safeParse(r).success);
  console.log(`Schema compliance: ${(1 - failures.length/n) * 100}%`);
}

Where to go next

• 41. Advanced RAG