diff --git a/src/index.ts b/src/index.ts index 6128a16..edcf042 100644 --- a/src/index.ts +++ b/src/index.ts @@ -125,7 +125,7 @@ program totalFailed += failed; totalWarnings += warnings; - const status = failed === 0 ? '✓' : '✗'; + const status = failed === 0 && warnings === 0 ? '✓' : '✗'; const warningStr = warnings > 0 ? `, ${warnings} warnings` : ''; console.log( `${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}` @@ -145,7 +145,7 @@ program console.log( `\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings` ); - process.exit(totalFailed > 0 ? 1 : 0); + process.exit(totalFailed > 0 || totalWarnings > 0 ? 1 : 0); } // Require either --scenario or --suite @@ -173,8 +173,12 @@ program timeout ); - const { failed } = printClientResults(result.checks, verbose); - process.exit(failed > 0 ? 1 : 0); + const { overallFailure } = printClientResults( + result.checks, + verbose, + result.clientOutput + ); + process.exit(overallFailure ? 1 : 0); } catch (error) { if (error instanceof ZodError) { console.error('Validation error:'); diff --git a/src/runner/client.ts b/src/runner/client.ts index cafb7c3..11a6115 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -150,8 +150,15 @@ export async function runConformanceTest( export function printClientResults( checks: ConformanceCheck[], - verbose: boolean = false -): { passed: number; failed: number; denominator: number; warnings: number } { + verbose: boolean = false, + clientOutput?: ClientExecutionResult +): { + passed: number; + failed: number; + denominator: number; + warnings: number; + overallFailure: boolean; +} { const denominator = checks.filter( (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' ).length; @@ -159,6 +166,14 @@ export function printClientResults( const failed = checks.filter((c) => c.status === 'FAILURE').length; const warnings = checks.filter((c) => c.status === 'WARNING').length; + // Determine if there's an overall failure (failures, warnings, client timeout, or exit failure) + const clientTimedOut = clientOutput?.timedOut ?? false; + const clientExitedWithError = clientOutput + ? clientOutput.exitCode !== 0 + : false; + const overallFailure = + failed > 0 || warnings > 0 || clientTimedOut || clientExitedWithError; + if (verbose) { // Verbose mode: JSON goes to stdout for piping to jq/jless console.log(JSON.stringify(checks, null, 2)); @@ -173,6 +188,16 @@ export function printClientResults( `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` ); + if (clientTimedOut) { + console.error(`\n⚠️ CLIENT TIMED OUT - Test incomplete`); + } + + if (clientExitedWithError && !clientTimedOut) { + console.error( + `\n⚠️ CLIENT EXITED WITH ERROR (code ${clientOutput?.exitCode}) - Test may be incomplete` + ); + } + if (failed > 0) { console.error('\nFailed Checks:'); checks @@ -185,7 +210,25 @@ export function printClientResults( }); } - return { passed, failed, denominator, warnings }; + if (warnings > 0) { + console.error('\nWarning Checks:'); + checks + .filter((c) => c.status === 'WARNING') + .forEach((c) => { + console.error(` - ${c.name}: ${c.description}`); + if (c.errorMessage) { + console.error(` Warning: ${c.errorMessage}`); + } + }); + } + + if (overallFailure) { + console.error('\n❌ OVERALL: FAILED'); + } else { + console.error('\n✅ OVERALL: PASSED'); + } + + return { passed, failed, denominator, warnings, overallFailure }; } export async function runInteractiveMode( diff --git a/src/scenarios/client/auth/scope-handling.ts b/src/scenarios/client/auth/scope-handling.ts index 137371a..d94760b 100644 --- a/src/scenarios/client/auth/scope-handling.ts +++ b/src/scenarios/client/auth/scope-handling.ts @@ -73,6 +73,21 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario { } getChecks(): ConformanceCheck[] { + // Emit failure check if expected scope check didn't run + const hasScopeCheck = this.checks.some( + (c) => c.id === 'scope-from-www-authenticate' + ); + if (!hasScopeCheck) { + this.checks.push({ + id: 'scope-from-www-authenticate', + name: 'Client scope selection from WWW-Authenticate header', + description: + 'Client did not complete authorization flow - scope check could not be performed', + status: 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY] + }); + } return this.checks; } } @@ -153,6 +168,21 @@ export class ScopeFromScopesSupportedScenario implements Scenario { } getChecks(): ConformanceCheck[] { + // Emit failure check if expected scope check didn't run + const hasScopeCheck = this.checks.some( + (c) => c.id === 'scope-from-scopes-supported' + ); + if (!hasScopeCheck) { + this.checks.push({ + id: 'scope-from-scopes-supported', + name: 'Client scope selection from scopes_supported', + description: + 'Client did not complete authorization flow - scope check could not be performed', + status: 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY] + }); + } return this.checks; } } @@ -221,6 +251,21 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario { } getChecks(): ConformanceCheck[] { + // Emit failure check if expected scope check didn't run + const hasScopeCheck = this.checks.some( + (c) => c.id === 'scope-omitted-when-undefined' + ); + if (!hasScopeCheck) { + this.checks.push({ + id: 'scope-omitted-when-undefined', + name: 'Client scope omission when scopes_supported undefined', + description: + 'Client did not complete authorization flow - scope check could not be performed', + status: 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY] + }); + } return this.checks; } }