fix: validate scope checks ran and report overall test status

src/index.ts

@@ -125,7 +125,7 @@ program
           totalFailed += failed;
           totalWarnings += warnings;
 
-          const status = failed === 0 ? '✓' : '✗';
+          const status = failed === 0 && warnings === 0 ? '✓' : '✗';
           const warningStr = warnings > 0 ? `, ${warnings} warnings` : '';
           console.log(
             `${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}`
@@ -145,7 +145,7 @@ program
         console.log(
           `\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings`
         );
-        process.exit(totalFailed > 0 ? 1 : 0);
+        process.exit(totalFailed > 0 || totalWarnings > 0 ? 1 : 0);
       }
 
       // Require either --scenario or --suite
@@ -173,8 +173,12 @@ program
         timeout
       );
 
-      const { failed } = printClientResults(result.checks, verbose);
-      process.exit(failed > 0 ? 1 : 0);
+      const { overallFailure } = printClientResults(
+        result.checks,
+        verbose,
+        result.clientOutput
+      );
+      process.exit(overallFailure ? 1 : 0);
     } catch (error) {
       if (error instanceof ZodError) {
         console.error('Validation error:');

src/runner/client.ts

@@ -150,15 +150,30 @@ export async function runConformanceTest(
 
 export function printClientResults(
   checks: ConformanceCheck[],
-  verbose: boolean = false
-): { passed: number; failed: number; denominator: number; warnings: number } {
+  verbose: boolean = false,
+  clientOutput?: ClientExecutionResult
+): {
+  passed: number;
+  failed: number;
+  denominator: number;
+  warnings: number;
+  overallFailure: boolean;
+} {
   const denominator = checks.filter(
     (c) => c.status === 'SUCCESS' || c.status === 'FAILURE'
   ).length;
   const passed = checks.filter((c) => c.status === 'SUCCESS').length;
   const failed = checks.filter((c) => c.status === 'FAILURE').length;
   const warnings = checks.filter((c) => c.status === 'WARNING').length;
 
+  // Determine if there's an overall failure (failures, warnings, client timeout, or exit failure)
+  const clientTimedOut = clientOutput?.timedOut ?? false;
+  const clientExitedWithError = clientOutput
+    ? clientOutput.exitCode !== 0
+    : false;
+  const overallFailure =
+    failed > 0 || warnings > 0 || clientTimedOut || clientExitedWithError;
+
   if (verbose) {
     // Verbose mode: JSON goes to stdout for piping to jq/jless
     console.log(JSON.stringify(checks, null, 2));
@@ -173,6 +188,16 @@ export function printClientResults(
     `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings`
   );
 
+  if (clientTimedOut) {
+    console.error(`\n⚠️  CLIENT TIMED OUT - Test incomplete`);
+  }
+
+  if (clientExitedWithError && !clientTimedOut) {
+    console.error(
+      `\n⚠️  CLIENT EXITED WITH ERROR (code ${clientOutput?.exitCode}) - Test may be incomplete`
+    );
+  }
+
   if (failed > 0) {
     console.error('\nFailed Checks:');
     checks
@@ -185,7 +210,25 @@ export function printClientResults(
       });
   }
 
-  return { passed, failed, denominator, warnings };
+  if (warnings > 0) {
+    console.error('\nWarning Checks:');
+    checks
+      .filter((c) => c.status === 'WARNING')
+      .forEach((c) => {
+        console.error(`  - ${c.name}: ${c.description}`);
+        if (c.errorMessage) {
+          console.error(`    Warning: ${c.errorMessage}`);
+        }
+      });
+  }
+
+  if (overallFailure) {
+    console.error('\n❌ OVERALL: FAILED');
+  } else {
+    console.error('\n✅ OVERALL: PASSED');
+  }
+
+  return { passed, failed, denominator, warnings, overallFailure };
 }
 
 export async function runInteractiveMode(

src/scenarios/client/auth/scope-handling.ts

@@ -73,6 +73,21 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario {
   }
 
   getChecks(): ConformanceCheck[] {
+    // Emit failure check if expected scope check didn't run
+    const hasScopeCheck = this.checks.some(
+      (c) => c.id === 'scope-from-www-authenticate'
+    );
+    if (!hasScopeCheck) {
+      this.checks.push({
+        id: 'scope-from-www-authenticate',
+        name: 'Client scope selection from WWW-Authenticate header',
+        description:
+          'Client did not complete authorization flow - scope check could not be performed',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY]
+      });
+    }
     return this.checks;
   }
 }
@@ -153,6 +168,21 @@ export class ScopeFromScopesSupportedScenario implements Scenario {
   }
 
   getChecks(): ConformanceCheck[] {
+    // Emit failure check if expected scope check didn't run
+    const hasScopeCheck = this.checks.some(
+      (c) => c.id === 'scope-from-scopes-supported'
+    );
+    if (!hasScopeCheck) {
+      this.checks.push({
+        id: 'scope-from-scopes-supported',
+        name: 'Client scope selection from scopes_supported',
+        description:
+          'Client did not complete authorization flow - scope check could not be performed',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY]
+      });
+    }
     return this.checks;
   }
 }
@@ -221,6 +251,21 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario {
   }
 
   getChecks(): ConformanceCheck[] {
+    // Emit failure check if expected scope check didn't run
+    const hasScopeCheck = this.checks.some(
+      (c) => c.id === 'scope-omitted-when-undefined'
+    );
+    if (!hasScopeCheck) {
+      this.checks.push({
+        id: 'scope-omitted-when-undefined',
+        name: 'Client scope omission when scopes_supported undefined',
+        description:
+          'Client did not complete authorization flow - scope check could not be performed',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY]
+      });
+    }
     return this.checks;
   }
 }