Add client conformance tests

ModelContextProtocol.slnx

@@ -70,6 +70,7 @@
   <Folder Name="/tests/">
     <Project Path="tests/ModelContextProtocol.Analyzers.Tests/ModelContextProtocol.Analyzers.Tests.csproj" />
     <Project Path="tests/ModelContextProtocol.AspNetCore.Tests/ModelContextProtocol.AspNetCore.Tests.csproj" />
+    <Project Path="tests/ModelContextProtocol.ConformanceClient/ModelContextProtocol.ConformanceClient.csproj" />
     <Project Path="tests/ModelContextProtocol.ConformanceServer/ModelContextProtocol.ConformanceServer.csproj" />
     <Project Path="tests/ModelContextProtocol.TestOAuthServer/ModelContextProtocol.TestOAuthServer.csproj" />
     <Project Path="tests/ModelContextProtocol.Tests/ModelContextProtocol.Tests.csproj" />

tests/ModelContextProtocol.AspNetCore.Tests/ClientConformanceTests.cs

@@ -0,0 +1,136 @@
+using System.Diagnostics;
+using System.Text;
+using ModelContextProtocol.Tests.Utils;
+
+namespace ModelContextProtocol.ConformanceTests;
+
+/// <summary>
+/// Runs the official MCP conformance tests against the ConformanceClient.
+/// This test runs the Node.js-based conformance test suite for the client
+/// and reports the results.
+/// </summary>
+public class ClientConformanceTests //: IAsyncLifetime
+{
+    private readonly ITestOutputHelper _output;
+
+    public ClientConformanceTests(ITestOutputHelper output)
+    {
+        _output = output;
+    }
+
+    [Theory]
+    [InlineData("initialize")]
+    [InlineData("tools_call")]
+    [InlineData("auth/metadata-default")]
+    [InlineData("auth/metadata-var1")]
+    [InlineData("auth/metadata-var2")]
+    [InlineData("auth/metadata-var3")]
+    [InlineData("auth/basic-cimd")]
+    // [InlineData("auth/2025-03-26-oauth-metadata-backcompat")]
+    // [InlineData("auth/2025-03-26-oauth-endpoint-fallback")]
+    [InlineData("auth/scope-from-www-authenticate")]
+    [InlineData("auth/scope-from-scopes-supported")]
+    [InlineData("auth/scope-omitted-when-undefined")]
+    [InlineData("auth/scope-step-up")]
+    public async Task RunConformanceTest(string scenario)
+    {
+        // Check if Node.js is installed
+        Assert.SkipWhen(!IsNodeInstalled(), "Node.js is not installed. Skipping conformance tests.");
+
+        // Run the conformance test suite
+        var result = await RunClientConformanceScenario(scenario);
+
+        // Report the results
+        Assert.True(result.Success,
+            $"Conformance test failed.\n\nStdout:\n{result.Output}\n\nStderr:\n{result.Error}");
+    }
+
+    private async Task<(bool Success, string Output, string Error)> RunClientConformanceScenario(string scenario)
+    {
+        // Construct an absolute path to the conformance client executable
+        var exeSuffix = OperatingSystem.IsWindows() ? ".exe" : "";
+        var conformanceClientPath = Path.GetFullPath($"./ModelContextProtocol.ConformanceClient{exeSuffix}");
+        // Replace AspNetCore.Tests with ConformanceClient in the path
+        conformanceClientPath = conformanceClientPath.Replace("AspNetCore.Tests", "ConformanceClient");
+
+        if (!File.Exists(conformanceClientPath))
+        {
+            throw new FileNotFoundException(
+                $"ConformanceClient executable not found at: {conformanceClientPath}");
+        }
+
+        var startInfo = new ProcessStartInfo
+        {
+            FileName = "npx",
+            Arguments = $"-y @modelcontextprotocol/conformance client --scenario {scenario} --command \"{conformanceClientPath} {scenario}\"",
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            UseShellExecute = false,
+            CreateNoWindow = true
+        };
+
+        var outputBuilder = new StringBuilder();
+        var errorBuilder = new StringBuilder();
+
+        var process = new Process { StartInfo = startInfo };
+
+        process.OutputDataReceived += (sender, e) =>
+        {
+            if (e.Data != null)
+            {
+                _output.WriteLine(e.Data);
+                outputBuilder.AppendLine(e.Data);
+            }
+        };
+
+        process.ErrorDataReceived += (sender, e) =>
+        {
+            if (e.Data != null)
+            {
+                _output.WriteLine(e.Data);
+                errorBuilder.AppendLine(e.Data);
+            }
+        };
+
+        process.Start();
+        process.BeginOutputReadLine();
+        process.BeginErrorReadLine();
+
+        await process.WaitForExitAsync();
+
+        return (
+            Success: process.ExitCode == 0,
+            Output: outputBuilder.ToString(),
+            Error: errorBuilder.ToString()
+        );
+    }
+
+    private static bool IsNodeInstalled()
+    {
+        try
+        {
+            var startInfo = new ProcessStartInfo
+            {
+                FileName = "npx",   // Check specifically for npx because windows seems unable to find it
+                Arguments = "--version",
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            };
+
+            using var process = Process.Start(startInfo);
+            if (process == null)
+            {
+                return false;
+            }
+
+            process.WaitForExit(5000);
+            return process.ExitCode == 0;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+}

tests/ModelContextProtocol.ConformanceClient/ModelContextProtocol.ConformanceClient.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>net10.0;net9.0;net8.0</TargetFrameworks>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TargetFramework)' == 'net9.0'">
+    <!-- For better test coverage, only disable reflection in one of the targets -->
+    <JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../../src/ModelContextProtocol.Core/ModelContextProtocol.Core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+  </ItemGroup>
+
+</Project>

tests/ModelContextProtocol.ConformanceClient/Program.cs

@@ -0,0 +1,189 @@
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.Web;
+using Microsoft.Extensions.Logging;
+using ModelContextProtocol.Client;
+
+// This program expects the following command-line arguments:
+// 1. The client conformance test scenario to run (e.g., "tools_call")
+// 2. The endpoint URL (e.g., "http://localhost:3001")
+
+if (args.Length < 2)
+{
+    Console.WriteLine("Usage: dotnet run --project ModelContextProtocol.ConformanceClient.csproj <scenario> [endpoint]");
+    return 1;
+}
+
+var scenario = args[0];
+var endpoint =  args[1];
+
+McpClientOptions options = new()
+{
+    ClientInfo = new()
+    {
+        Name = "ConformanceClient",
+        Version = "1.0.0"
+    }
+};
+
+var consoleLoggerFactory = LoggerFactory.Create(builder =>
+{
+    builder.AddConsole();
+});
+
+// Configure OAuth callback port via environment or pick an ephemeral port.
+var callbackPortEnv = Environment.GetEnvironmentVariable("OAUTH_CALLBACK_PORT");
+int callbackPort = 0;
+if (!string.IsNullOrEmpty(callbackPortEnv) && int.TryParse(callbackPortEnv, out var parsedPort))
+{
+    callbackPort = parsedPort;
+}
+
+if (callbackPort == 0)
+{
+    var tcp = new TcpListener(IPAddress.Loopback, 0);
+    tcp.Start();
+    callbackPort = ((IPEndPoint)tcp.LocalEndpoint).Port;
+    tcp.Stop();
+}
+
+var listenerPrefix = $"http://localhost:{callbackPort}/";
+var preStartedListener = new HttpListener();
+preStartedListener.Prefixes.Add(listenerPrefix);
+preStartedListener.Start();
+
+var clientRedirectUri = new Uri($"http://localhost:{callbackPort}/callback");
+
+var clientTransport = new HttpClientTransport(new()
+{
+    Endpoint = new Uri(endpoint),
+    TransportMode = HttpTransportMode.StreamableHttp,
+    OAuth = new()
+    {
+        RedirectUri = clientRedirectUri,
+        AuthorizationRedirectDelegate = (authUrl, redirectUri, ct) => HandleAuthorizationUrlWithListenerAsync(authUrl, redirectUri, preStartedListener, ct),
+        DynamicClientRegistration = new()
+        {
+            ClientName = "ProtectedMcpClient",
+        },
+    }
+}, loggerFactory: consoleLoggerFactory);
+
+await using var mcpClient = await McpClient.CreateAsync(clientTransport, options, loggerFactory: consoleLoggerFactory);
+
+bool success = true;
+
+switch (scenario)
+{
+    case "tools_call":
+    {
+        var tools = await mcpClient.ListToolsAsync();
+        Console.WriteLine($"Available tools: {string.Join(", ", tools.Select(t => t.Name))}");
+
+        // Call the "add_numbers" tool
+        var toolName = "add_numbers";
+        Console.WriteLine($"Calling tool: {toolName}");
+        var result = await mcpClient.CallToolAsync(toolName: toolName, arguments: new Dictionary<string, object?>
+        {
+            { "a", 5 },
+            { "b", 10 }
+        });
+        success &= !(result.IsError == true);
+        break;
+    }
+    case "auth/scope-step-up":
+    {
+        // Just testing that we can authenticate and list tools
+        var tools = await mcpClient.ListToolsAsync();
+        Console.WriteLine($"Available tools: {string.Join(", ", tools.Select(t => t.Name))}");
+
+        // Call the "test_tool" tool
+        var toolName = tools.FirstOrDefault()?.Name ?? "test-tool";
+        Console.WriteLine($"Calling tool: {toolName}");
+        var result = await mcpClient.CallToolAsync(toolName: toolName, arguments: new Dictionary<string, object?>
+        {
+            { "foo", "bar" },
+        });
+        success &= !(result.IsError == true);
+        break;
+    }
+    default:
+        // No extra processing for other scenarios
+        break;
+}
+
+// Exit code 0 on success, 1 on failure
+return success ? 0 : 1;
+
+// Copied from ProtectedMcpClient sample
+static async Task<string?> HandleAuthorizationUrlWithListenerAsync(Uri authorizationUrl, Uri redirectUri, HttpListener listener, CancellationToken cancellationToken)
+{
+    Console.WriteLine("Starting OAuth authorization flow...");
+    Console.WriteLine($"Opening browser to: {authorizationUrl}");
+
+    try
+    {
+        _ = OpenBrowserAsync(authorizationUrl);
+
+        Console.WriteLine($"Listening for OAuth callback on: {listener.Prefixes.Cast<string>().FirstOrDefault()}");
+        var contextTask = listener.GetContextAsync();
+        var context = await contextTask.WaitAsync(cancellationToken);
+        var query = HttpUtility.ParseQueryString(context.Request.Url?.Query ?? string.Empty);
+        var code = query["code"];
+        var error = query["error"];
+
+        string responseHtml = "<html><body><h1>Authentication complete</h1><p>You can close this window now.</p></body></html>";
+        byte[] buffer = Encoding.UTF8.GetBytes(responseHtml);
+        context.Response.ContentLength64 = buffer.Length;
+        context.Response.ContentType = "text/html";
+        context.Response.OutputStream.Write(buffer, 0, buffer.Length);
+        context.Response.Close();
+
+        if (!string.IsNullOrEmpty(error))
+        {
+            Console.WriteLine($"Auth error: {error}");
+            return null;
+        }
+
+        if (string.IsNullOrEmpty(code))
+        {
+            Console.WriteLine("No authorization code received");
+            return null;
+        }
+
+        Console.WriteLine("Authorization code received successfully.");
+        return code;
+    }
+    catch (Exception ex)
+    {
+        Console.WriteLine($"Error getting auth code: {ex.Message}");
+        return null;
+    }
+    finally
+    {
+        try { if (listener.IsListening) listener.Stop(); } catch { }
+    }
+}
+
+// Simulate a user opening the browser and logging in
+static async Task OpenBrowserAsync(Uri url)
+{
+    // Validate the URI scheme - only allow safe protocols
+    if (url.Scheme != Uri.UriSchemeHttp && url.Scheme != Uri.UriSchemeHttps)
+    {
+        Console.WriteLine($"Error: Only HTTP and HTTPS URLs are allowed.");
+        return;
+    }
+
+    try
+    {
+        using var httpClient = new HttpClient();
+        using var authResponse = await httpClient.GetAsync(url);
+    }
+    catch (Exception ex)
+    {
+        Console.WriteLine($"Error opening browser: {ex.Message}");
+        Console.WriteLine($"Please manually open this URL: {url}");
+    }
+}

tests/ModelContextProtocol.ConformanceServer/ModelContextProtocol.ConformanceServer.csproj

@@ -5,7 +5,6 @@
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <OutputType>Exe</OutputType>
-    <AssemblyName>ConformanceServer</AssemblyName>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(TargetFramework)' == 'net9.0'">