From 66a0b872e42a7f516b5fb5d6ea440517226e27dc Mon Sep 17 00:00:00 2001 From: James Braza Date: Thu, 23 Oct 2025 12:09:53 -0700 Subject: [PATCH 1/3] Pinned new lower version of litellm --- packages/lmi/pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lmi/pyproject.toml b/packages/lmi/pyproject.toml index 4c3bcf26..115a1634 100644 --- a/packages/lmi/pyproject.toml +++ b/packages/lmi/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "coredis", "fhaviary>=0.14.0", # For multi-image support "limits>=4.8", # Pin for RedisBridge.key_prefix - "litellm>=1.63.5", # Lower pin for reasoning + streaming fix + "litellm>=1.75.6", # Lower pin for OpenAI reasoning effort support "pydantic~=2.0,>=2.10.1", "tiktoken>=0.4.0", "typing-extensions; python_version <= '3.11'", # for typing.override diff --git a/uv.lock b/uv.lock index 51c236b6..f547af39 100644 --- a/uv.lock +++ b/uv.lock @@ -864,7 +864,7 @@ requires-dist = [ { name = "httpx-aiohttp", marker = "extra == 'dev'" }, { name = "ipython", marker = "extra == 'dev'", specifier = ">=8" }, { name = "limits", specifier = ">=4.8" }, - { name = "litellm", specifier = ">=1.63.5" }, + { name = "litellm", specifier = ">=1.75.6" }, { name = "litellm", marker = "extra == 'dev'", specifier = ">=1.71" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8" }, { name = "numpy", marker = "extra == 'local'" }, From e293954f5f407aab9fc1bc71335536abd759b1f5 Mon Sep 17 00:00:00 2001 From: James Braza Date: Thu, 23 Oct 2025 11:51:37 -0700 Subject: [PATCH 2/3] Upgraded Anthropic reasoning test to include GPT-5 --- .../TestReasoning.test_anthropic_model.yaml | 110 --------------- ...cted_len1-claude-3-7-sonnet-20250219].yaml | 118 ++++++++++++++++ ...[high-expected_len1-gpt-5-2025-08-07].yaml | 128 ++++++++++++++++++ ...cted_len0-claude-3-7-sonnet-20250219].yaml | 110 +++++++++++++++ ...t[low-expected_len0-gpt-5-2025-08-07].yaml | 126 +++++++++++++++++ packages/lmi/tests/test_llms.py | 40 ++++-- 6 files changed, 509 insertions(+), 123 deletions(-) delete mode 100644 packages/lmi/tests/cassettes/TestReasoning.test_anthropic_model.yaml create mode 100644 packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-3-7-sonnet-20250219].yaml create mode 100644 packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-gpt-5-2025-08-07].yaml create mode 100644 packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-3-7-sonnet-20250219].yaml create mode 100644 packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-gpt-5-2025-08-07].yaml diff --git a/packages/lmi/tests/cassettes/TestReasoning.test_anthropic_model.yaml b/packages/lmi/tests/cassettes/TestReasoning.test_anthropic_model.yaml deleted file mode 100644 index 393eb8bc..00000000 --- a/packages/lmi/tests/cassettes/TestReasoning.test_anthropic_model.yaml +++ /dev/null @@ -1,110 +0,0 @@ -interactions: - - request: - body: - '{"model":"claude-3-7-sonnet-20250219","messages":[{"role":"user","content":[{"type":"text","text":"What - is the meaning of life?"}]}],"thinking":{"type":"enabled","budget_tokens":1024},"max_tokens":5120,"system":[{"type":"text","text":"Think - deeply about the following question and answer it."}]}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - anthropic-version: - - "2023-06-01" - connection: - - keep-alive - content-length: - - "295" - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - litellm/1.70.4 - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: !!binary | - H4sIAAAAAAAAA4RW21IcNxD9lS69+CHDbW2w2ZdUwBgImGzAxpCQorQzPaNmNdKglnZ2cLkqH5Ev - zJekpNkbOJU87awufT19jr4KKsRQ1Fzdb+9czEp18TTOy+ODvZtr3nlrfz27YpEJ3zUYTyGzrFBk - wlkdFyQzsZfGi0zUtkAthiLXMhS48Xrj7QZbY9BvDLYHu9uDnX2Ridwaj8aL4e9fF0a9IjMhU0U3 - i8+h+KSIgRgkNM6WNpgCGkXasm0U5VLDY0D2ZA14JT0oyTBGNFDgWHoswCtnQ6Vs8KBCLQ0oYm9d - twmfFDoEYvPKgwQmU2mEO5Fb5zD3dwKk4RZdb7e1QRfA0hOXHeAUXWcNZiAZvEKoURoyFdgSNJXR - KiiqlO6Aw/gBc09TBGkKmEpHyDCWjAVYA3nQPjipM3CoqSIbOHueYJbuNejYGqnTR9Mb5M07c2dO - gVUKrnF2SgWCBK9iyr4MGhxyYw1jn4XMJ8a2GosKGQoqS3RoPEwJ28aS8Zx8ocltcDKeibn1rsFb - yK1hKtBBG43VVCkP42Xy0Z238Uq9DFd3m7AMUE4tFTDGVKgpOt1B45BzR01fn3HsUrL9bzW9E8R3 - IjYqj+n0sAjaUy09xkrbGiNwKkCZq0XYdWAPBXp0NRmE0roUIaNeFTCXBmo0CUXRCuS2rq15ATTZ - NM7KXCEP78wGXC769awlYEuPBjhUFbJPgb/iZSpkpjb6jZU08YapoCWvUr4FTWOEsQOl1dq2cZcb - cuSD1OBjVmQqjs6PZsRxfkhqYr8KFB2DpgnClXTe4bxH0lUBoUXIHUqPYIMD25q1sEAaIKMSHHS3 - 2NDIDMHQFB1jdPvZkyYvHUnDc9uLTGs5o5qeojklm4ZMvBtzcViEPKUSIt7iV5UqlZo69xSNX8XK - 90ZLMsV6dEv0O9QytokVNZz1+UToNMFxIM8ZWBeL6x2NQyqvt8A2J/Rd9PGLTyX6Hy+Vs61XGeCs - QUdoUvixRa11ukg+GDESFCznKSHpldZrYziWWpoci/8awoTj3NaNxhn5DlpFGuczsjbG62MZt9YK - UuoIJGs2RSaYKiN9cJFOj9zng8PJ7enBZXV4Ojl4c3nyud0dH18f+73S2MvH3x5ms6uPO09hZ/R2 - kI/3zz8MvL7hydVj8/p6uz3fbau3o61iXD2Y2y83s7xrHt//fPNwfqkoPz+t1dH5+clR1e7fcGC9 - t33RvrkYnU1+le8vzk5Ho+PZYH+0daZLdXN12t58+ens6LePlxeMdDg5Orw533+4vbrY+nR98kFN - 86fgB1vm6B19OP58sbUzeHrPO+EdvtFH0x9ODn13cqYGZm9E74rj4+1re4rl6HavHHU/dyf6fEfd - HrYff/C7Xw4+7B5Uh+JbtpIUnEVFSj9RSv6VqK3B+DepA/kujqtlD2iKkNCa6HehPQu14U14v+TP - FwRQoks8H7mh15DIGHcmAfwZ6ObitOL+BK3lyDfSK84i1uLhFK9kaKTzMWC5YIxGS7O5gDbWYyfz - SCOAz0kisvxczP6HCWw01RIjkFmpxIIINuGjNF2fSBNcY9O5ubouiM2av//8axxIF9Hsy7F9MaCR - bkOsfRotjXKa6g6NZUrjrbGSeZfYerROdevi3ss6KDnFBSnhqtK5rZGhdLZetoZtcDnycE4e68SV - Lee7JS5svWAaU8EYZYiBSq1xSv3aktl6aqC60d335FEGrcFJU63gtmhRjn1u6JRseC6vcrLgh+UT - h1OtGtQ6NYv7dkbj87dK/5qZd0rq9Mj5DvNL4f4vxSTOo0YvIeoVUo+VBxucwS6aY9TlxoqCMtA0 - xWKVOvKL18tU6jCX3S+JC9Pc8DK0cZq4zoZeERGbqC2MkTxtucxErqD3o/j2RybY2+beoWRrxFCg - Ke59cEbMNxgfQ4xGDE3QOhMhPV6HXwWZJvh7bydoWAx3X2cijwJ/33fbmvvnB7YX+w5l8f2eDX59 - 5c1gLxOMbko53ntCJ4YivpEL6Qrx7ds/AAAA//8DAOGZBu9xCwAA - headers: - CF-RAY: - - 958988ccc9ebf97f-SJC - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Tue, 01 Jul 2025 23:03:10 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Robots-Tag: - - none - anthropic-organization-id: - - f2c99ed9-038a-406f-9cb5-1f840b758a20 - anthropic-ratelimit-input-tokens-limit: - - "20000000" - anthropic-ratelimit-input-tokens-remaining: - - "20000000" - anthropic-ratelimit-input-tokens-reset: - - "2025-07-01T23:03:02Z" - anthropic-ratelimit-output-tokens-limit: - - "2000000" - anthropic-ratelimit-output-tokens-remaining: - - "2000000" - anthropic-ratelimit-output-tokens-reset: - - "2025-07-01T23:03:10Z" - anthropic-ratelimit-requests-limit: - - "7000" - anthropic-ratelimit-requests-remaining: - - "6999" - anthropic-ratelimit-requests-reset: - - "2025-07-01T23:03:02Z" - anthropic-ratelimit-tokens-limit: - - "22000000" - anthropic-ratelimit-tokens-remaining: - - "22000000" - anthropic-ratelimit-tokens-reset: - - "2025-07-01T23:03:02Z" - cf-cache-status: - - DYNAMIC - request-id: - - req_011CQh8gct8KS1oFHc4oisQg - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - via: - - 1.1 google - status: - code: 200 - message: OK -version: 1 diff --git a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-3-7-sonnet-20250219].yaml b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-3-7-sonnet-20250219].yaml new file mode 100644 index 00000000..81f464ac --- /dev/null +++ b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-3-7-sonnet-20250219].yaml @@ -0,0 +1,118 @@ +interactions: + - request: + body: + '{"model":"claude-3-7-sonnet-20250219","messages":[{"role":"user","content":[{"type":"text","text":"What + is the meaning of life?"}]}],"thinking":{"type":"enabled","budget_tokens":4096},"max_tokens":8192,"system":[{"type":"text","text":"Think + deeply about the following question and answer it."}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "295" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.78.3 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//dFfrctQ6En6VLu8P9oADuR7Y2R9bSSAhIRxCIGFh2Upp7LbVjKz2UUue + OBRV+xD7hPskW5LtyQ1+JTNqtfry9dfffM8aLtFks6wwKpS4trX2fE3YWvRrm+ubO+ubG3/L8ozK + bJY1Ul+ub5yGvfOgLhRSwBenu282yt/r83mWZ75vMVqhiKoxyzPHJn6hREi8sj7Ls4KtR+uz2b++ + T/Zek12QraOH6d9Z9lGTAAkoaB1XHGwJrSbDwq2mQhn4M6B4YgteKw9aCcwRLZQ4Vx5LmPc39ugk + B6+RDdekrOSgbAnpMXQCXjsOtebgQYdGWdAknl3/FD5qdBijsAxCtjaYQ7DUoRNlTA+qdojlWmjZ + grKyRPcUTtBDg4BXrWGH0ClHHARadNJi4alDmX21X+0ZGqp/drQGb5XtwQ3nVkBCXaP4IVGvERpU + lmwNXIGhKgXoGQRdh3DIJbADBZpqjQ5aXqLLoWJjeAkldWQRCm4aZcsGrR+LoQpN2CGIMp1KZWUH + aA3V2qONhjGwfe1IPClLvp/FNxeWlzkY7nBws4ohWh+JUU0yW7ITTS3sGqN0MjTUIaiiYFfGTDyn + xN4Hp2y8uhfKUpMMtzE6DlWFLprO+zHY+OFBhK/JlmG6OCXV8EK0gr8amqMbsqscN+nJoi8Mxko6 + nJPz+rfYm9M7SHvQn1dXJBHFpAyJl+h4gfBBOe8wZbevmiC/zeIDK/yQjR+sn7r3d1giNEE8FA6V + R+DggJd2Oo8P/UGa0hOzodER5ZaB59+GcG7bvsaS7WD7a4w06ooaukZoDSoJDuPNc0+GvHJxNGZ3 + rLRqW7IokrJaojFrc4xeK3bplTqFLh5saOboorcPnqkY/KQmd+R84CCmB7Jjy5UtEJbkNVjlw1g0 + h0rYxvK/5RKdhVb6QqeZ/WkXLmjh2cGBU3ZhZlCRTVCaEh+nGtrgWhasgokwXNxGa8HBqRpjXJUq + 4qVYqkcChVbGoK1R4kOnLJSqvQqon93CoALxqX/V7Qqtnp8uY8MReZID2lrVGBGbxylPgIzjIfkU + /DSTBTetIdETul91bEI0V66/E00c/+C6NFSplK3jMhQYq/nOJx64X74P3CAoVwcE8o8E1DwSoEE1 + JrUK3GCtij5eSZ5u+OjWteidrTJQO156PXKBqdZU4YMydJ2SXHFb7NWqUWTvVmFsjbUx2sh/CSic + 3l4Fft9DmqFU5qvWoQglRk5+vKN5GJ7/ao9ANAcTixu5y2BZ40SsJDdLhQRKxNb0q9SmpXF/I8DX + rGDnsPBfs9USeEmRruK0t8iteRhvuTKIa6gWmCvBEtI6Q3LQKRNQcpijIaxGksarFh2hLVCewiqT + 1nFHJYICryPkfBVMDnNl4pRFLEjLVsYs4ycsvNwKYMnOlB3hUmCpKZHhSLUSK12jDXFnkJVItfI0 + yzOhehjcbJa9cud7+4vPR/t79f7RYu/i8OPnNwdbF82Ben/Fu/xalq/Pdq82xextbZ683/5Me7zR + 7l7/c30/mJfdp3b34tlic2dxeHZ97l4efHRPnnw5fvLEPFs/PNfHzcnns92z41f1lVnuhYa/vQ8O + j82H5wv18lV3tHVxzYsw5+Pynd3sj5Z4tv7py+56ub0jm5snhXpnFi3bA+b1g7D4dlxdq2dH3VW/ + 2W0fN/7L3Ibndbsh+jps9y+36HD5Rm8WZvtEt9fv95r1+vqQ/3hRdfvP3ZuT/Xfqxeed5bPtQh29 + Ubrez37kNxIGr6K4SX9m2V+iboC3NwR8QlWaxKRpVnJmhbXI6hRhWgcsBwlCvr8jSwZBko+cuYLe + Sov8FHg3Aw9kCxNKTMLjZ/OTGOHx47MHcxjJ8vFjWEvT8HAqK9WQ6XOoIixLmWi1aUJMYXB6OhDw + g3FMbhu1uEc3EzCLxMvD2D9Kq0RGf3e5JrmJfskGZSKBuZRUfG6OBTfxQx+Xqwpex61dJGoanTkU + tH6t4cixoJbKYVx4yeu0Ur5xP7pTwac1hh26vlT97ZG8K+rUA8HqnSppqF2asCQNsRxb91Pl1zJF + hhi03VCnSdaNRWU3abrWDNLpvjiZ2HqJvxAaaS3blT4x/XRg4uIfIYYr5r2tqiOYUxhpyWN+Ixjy + GNmSpOQGlECB1rsIU2pSnU7RadVK0hBjVQ0lnKavBJUrNJCPjfrff/6bqAtVoUc2HnRTSVJwhw6W + 8XjuEo96jQ1UwVRkzLBib0A7rlUQjHzI1e0KDLQby/KNg7PYP41xfoqe6zRCCUJJSk23UouHPvwj + +/HvPBPP7eUgY7JZhra89MHZbDwQ/DNEoGQzG4zJs5B+Jc2+Z2Tb4C89L9BKNtvZyrNCFRovh53G + 9vKuwfp07lCVvzqb7kb/2Gps0ClzudM8tL853dD3T3/kGQd/+6vfN7bzLKGxwEtP6LJZFn/blcqV + 2Y8f/wcAAP//AwCVT2Z/Tg4AAA== + headers: + CF-RAY: + - 99336b1bd8291742-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 23 Oct 2025 18:50:19 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + Via: + - 1.1 google + X-Robots-Tag: + - none + anthropic-organization-id: + - f2c99ed9-038a-406f-9cb5-1f840b758a20 + anthropic-ratelimit-input-tokens-limit: + - "20000000" + anthropic-ratelimit-input-tokens-remaining: + - "20000000" + anthropic-ratelimit-input-tokens-reset: + - "2025-10-23T18:50:09Z" + anthropic-ratelimit-output-tokens-limit: + - "2000000" + anthropic-ratelimit-output-tokens-remaining: + - "1999000" + anthropic-ratelimit-output-tokens-reset: + - "2025-10-23T18:50:19Z" + anthropic-ratelimit-requests-limit: + - "7000" + anthropic-ratelimit-requests-remaining: + - "6999" + anthropic-ratelimit-requests-reset: + - "2025-10-23T18:50:08Z" + anthropic-ratelimit-tokens-limit: + - "22000000" + anthropic-ratelimit-tokens-remaining: + - "21999000" + anthropic-ratelimit-tokens-reset: + - "2025-10-23T18:50:09Z" + cf-cache-status: + - DYNAMIC + request-id: + - req_011CUQdcX9gAmBe8qxd6k626 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "10343" + status: + code: 200 + message: OK +version: 1 diff --git a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-gpt-5-2025-08-07].yaml b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-gpt-5-2025-08-07].yaml new file mode 100644 index 00000000..b6fd3e0e --- /dev/null +++ b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-gpt-5-2025-08-07].yaml @@ -0,0 +1,128 @@ +interactions: + - request: + body: + '{"messages":[{"role":"system","content":"Think deeply about the following + question and answer it."},{"role":"user","content":"What is the meaning of life?"}],"model":"gpt-5-2025-08-07","n":1,"reasoning_effort":"high"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "217" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.5.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.5.0 + x-stainless-raw-response: + - "true" + x-stainless-read-timeout: + - "60.0" + x-stainless-retry-count: + - "0" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.8 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA3RW244URwx936+w+gmk2WF3ybJk9olLQlBAWWAjgQJa1VS5p52tLrfKrhlGCInf + 4Pf4ksjVc0OBl5G6fD8+tufTEUBDoZlB4zunvh/i8ZPr0r4dVk9fvb667t6++z3+FW+fv3q3XK66 + /o9mYhY8/xe9bq2mnvshohKnUewzOkXzenrx4PTsl/P7D+9XQc8Bo5ktBj0+Pz47OTs/Pnl4fHKx + seuYPEozg3+OAAA+1V/LMAX82MzgZLJ96VHELbCZ7ZQAmszRXhonQqIuaTPZCz0nxVSTvu4QenSJ + 0gK4hUgtAkn69uWrggOhtIgIrfMKK0datYqCdpgRlGGO0HJJ4RJIv335KuBgcKqYE6y5wFg93HEp + gHCPSj0KBBLPS8x3QbvMZdHBqnNaDSIvcbL/FM20tBB5AuZjK/j25WtGWFGMlpEyqLtFyCgDJ6E5 + RdK1WU3hKbUtZkwKEZOgANs3hN2zS7LCLLP36X06hsfEkRfkXZyBlLwkSyfjkDkUj2MSgxMBTkCp + 5dw76/XUbF9jpAVxkRncJl5VXasHnnG4BMG8RGBDTsAJuAT4ccgoQpwMerXaTL86e5RJlBUjuXTv + jTL5GUTDYklZCxeJ60toI5dM0u1w9J3LzivmES2SwH1197caJi6bt670Lm0+ZpDRKgMpBoihaZaU + rHOCsMIYj+dIaVHd/PaRxJhDLt5zcyk5kOjM6AAl0RKzGHtAKGLSy5ECHbNg9bphw5Zuu6S570kV + AzhvYEJAGUixuh2j1OCPSwhdjYdGp13C8zUIWoqQ0dXO+4gux/UYtESlpavUteF0FfCptftK1r6z + fq+hpRRkbMEuPaxPDIvMK2gz97OaBUZOC1O446NVljFWDkhHg9SQFS9IiAHD3YkZXZU8mO6dBbu4 + iTMSok5FV8vANadQixaM7Wj4hG3SkrcZAlHO602S7hYFxCht3FlzyXV2R6vr7JJ4TGG07LnHpEZ9 + cCucgDifMSQUmQBnCIgDGBNN+25F5hEMxiObA5B10g6FpJb/wgg9IA8Rq2FcT434hwjksXOimdMC + RSsmPYsaVOTmcaSVDHmzdDaIT8eCk2aaF8UtIFabATKFq6Jjpa5OOKEApYoj+QrDHswKEheNTlTM + qDp/Zp2kNM7QyEm5pRin8AJdTpMNQ8cx/8FKuYSICp3LwUoF6dyAkJ3NtEVMIF2mdLuL90bdGnjA + ZDRacQqYp/Dyu27UUIvslLQEi+ddpHm2ORkwy4BeabMS/rQ+ZVySbeUpvBxRE8AlxyWC85nFOOGE + k2yXeW3n88oQWLl0sNQF7Qh4nMELqkTcgDZuqHG8IKJsOe2KrieguWg32e82tI0APWfrl3dFcAxl + 98GoOz28OxnbIs6uXioxHghcSqwjh+zifdhIPu9uXEuJpLvJtTi7W6I8NFX6+QjgQ72Z5bsz2AyZ + +0FvlG+xuj27GN01+xt9IDy9f74RK6uLh5IHZ5MfeLwJqI6iHNzdxjvfYdjb7o+0K4H4QHB0UN// + E/qR77F2Sou9l9OLX89+GmEv8B4HxXAzZAzkv697r5bR/sj8TG0Hdc252QzdjRJma0fA1pU4/slo + ZC2K/U1LaYHZplw3HT/6fPQfAAAA//8DAINfyMNmCQAA + headers: + Access-Control-Expose-Headers: + - X-Request-ID + CF-RAY: + - 99336a7dbe6ceb26-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 23 Oct 2025 18:50:08 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=DjysHB9qTMZORX7QYyklZkQ1gpCLqiF_4Oc3s4I8ipY-1761245408-1.0.1.1-o1.0znf3QrMBHHkWGdL6H8DlcQnOUBlYMBnIgPIQbfD4x_sGH1NYYRKZdaDWYYgKSCKsA55Y_WKWdc3JL2YOUVOAxVcxpifd7KTjej8UtC0; + path=/; expires=Thu, 23-Oct-25 19:20:08 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=EoJIYMSxi_5wB664HTnmjMc0JkyL5rOMzF69BXv6NQw-1761245408465-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "25008" + openai-project: + - proj_RpeV6PrPclPHBb5GlExPXSBj + openai-version: + - "2020-10-01" + x-envoy-upstream-service-time: + - "25087" + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - "15000" + x-ratelimit-limit-tokens: + - "40000000" + x-ratelimit-remaining-requests: + - "14999" + x-ratelimit-remaining-tokens: + - "39999976" + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_79da4032ef5c40fda551af38c74aa897 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-3-7-sonnet-20250219].yaml b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-3-7-sonnet-20250219].yaml new file mode 100644 index 00000000..b4214387 --- /dev/null +++ b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-3-7-sonnet-20250219].yaml @@ -0,0 +1,110 @@ +interactions: + - request: + body: + '{"model":"claude-3-7-sonnet-20250219","messages":[{"role":"user","content":[{"type":"text","text":"What + is the meaning of life?"}]}],"thinking":{"type":"enabled","budget_tokens":1024},"max_tokens":5120,"system":[{"type":"text","text":"Think + deeply about the following question and answer it."}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "295" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.78.3 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//fFbbchs5Dv0VFF9mttKWZSu+6WUrvsVyxk4i32KPp1xUN7qJEZto8yJZ + SblqPmK+cL5ki2zJkp3dfZKaBAng4OCAP0TNBWrRF7mWocC13trOmmNj0K9tdje3upsbeyITVIi+ + qF310N0Y3vVOP13+dnrDo5vrD9f+bEZXp15kws8ajFbonKxQZMKyjgvSOXJemmiTs/FovOj//mNh + 7xWZMZkq3rD42xeXihw8BnSe2IAccfDgFUKN0pCpgEvQVCKQAzYYP9MuOw+N5ZKDKaBRpNlxoyiX + +uUyByrU0jiQbtyBgf/FwYi9ggKx0TNo0Do2UoM0BQRDE7ROaj2DFHrdaOmx6Nybe3Op0CJIi1BL + 0x5sMPc0QQcDyKWBglwenOtH640ODFFTRRzcumvIkg9Svz7FpUcDLlQVutfpkpmwjiZ5LE2eQPEM + EgqakEFogm3YYQYO7SQe+MhFBmxB5oowrSx9otFUKY+mRuNTKpsdOHoiF0tDUlMEcYEdWgeaxggX + 0nqLK9FJ35ZASQeGgUzEw/hFzBmMgocpQm5R+mUqXlkOlQIOFnLFlKNLWMuUlUvx9DpwEotEzlP+ + GqOF/8V1OdfooLRcr2DjYEpeAfsYfpZKZ2kUfPLP4Dgn9LMs+X0peGV56lXy/74D+8Saq8ScV/7r + CB2UVtYI92Iexb0AMuDR1rGI4IKd0GTOIYuN5SKkuNLlWx04kU1DBp1bKzkPDle5inO8jxrKg433 + uzpWMnjS5KWldqWN4/9ypZZPVNP3uDZFrddGSKZKIQzAKQ46gj42PNVYVNhW1Me+IwdBe6qlRz0D + uYTopSETvIbBkal0BCJnazH39wKkcVO0HRjAVBof4W4sT6hAkOBVLL0vgwaLrmHj5m7jF+beQUFl + 2dJoylYXE8JpSw80OQcrK3RgsdRtnTsiE44qI32wUUuO7NX+wfh2cLBfHQzG+91GUugNH83eDQ56 + +8ePo/NPF3ty42DEo+v3p99wfHgre7fHd38eHJ0dedvsXXc/V+piuP/lu7kd4U733d1336zr2+ZD + c3c9PJwUT/lRNf3y4brUzbGhsfnYTD+8l4cHV8fbk+2wXm7vnn8e6pvZYNpdv9pS9sx+/kLb+vGd + 9Sd08qnZOf0qy5uvd/zn7bWfDfzl0/nI2PPe2fm32beDq3p8c7l7slutH158Ut1vl7tbGvF0d/to + Nty53FjXh9vq6+Hex/2j4/dhui5nZzvvhsPqQDxnSz3Fp6TG8eetjsZWpdgLVVjlXGoSrzAxXhrX + NkZS49j+846NEpykExQ5z3a2MEOLv7gVNryo5oILkXIXXCOUZIoVni6JtdK6//z1d2SzRS3Tt6LG + RRkZBYqEbT2C5gnGVaeiAK90ewc+p98kvjxBu9DGF+FJehQDaNhRbGrIlTQVxib7WSlqjDhUoKWt + 0Ea2mhhA7TD2WErtLMq/XYg7eCsLalWII5lbsZiyHTso2f4khEthXqKQwVSRxjcj7JUQ2RRzEj5c + 1e4afl2KbhRZnpqFz3/FnGq2CAVGtaK5wqY2S7l8QatkBDx2ZS3H6FpJeKGQ4+WAJbcQDZyXOi69 + GaQduPwvc7uVrxHC/KYEKrCpOM0qlDZXQN6hLv/56++F9zRWFoVNH5EPy4K+qfTKiMlWKdYew6cG + LaHJ52W8ianIuQ5xCbN4fD7hJgglat8+MOa5RBXzHM3+LZ7/yITz3DxYlI6N6As0xYMP1oj5hsPH + EH2JvglaZyKkR1L/hyDTBP/geYzGif5WLxO5zBU+tFmxeXht0F3sW5TF/9pbnI33Y6OwRiv1w1b9 + s/1yd0O93X3OBAe/utTb2cxEemLk+OAJreiL+LQrpC3E8/N/AAAA//8DACWxE7dNCgAA + headers: + CF-RAY: + - 99336a4a7e6015eb-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 23 Oct 2025 18:49:43 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + Via: + - 1.1 google + X-Robots-Tag: + - none + anthropic-organization-id: + - f2c99ed9-038a-406f-9cb5-1f840b758a20 + anthropic-ratelimit-input-tokens-limit: + - "20000000" + anthropic-ratelimit-input-tokens-remaining: + - "20000000" + anthropic-ratelimit-input-tokens-reset: + - "2025-10-23T18:49:36Z" + anthropic-ratelimit-output-tokens-limit: + - "2000000" + anthropic-ratelimit-output-tokens-remaining: + - "2000000" + anthropic-ratelimit-output-tokens-reset: + - "2025-10-23T18:49:43Z" + anthropic-ratelimit-requests-limit: + - "7000" + anthropic-ratelimit-requests-remaining: + - "6999" + anthropic-ratelimit-requests-reset: + - "2025-10-23T18:49:35Z" + anthropic-ratelimit-tokens-limit: + - "22000000" + anthropic-ratelimit-tokens-remaining: + - "22000000" + anthropic-ratelimit-tokens-reset: + - "2025-10-23T18:49:36Z" + cf-cache-status: + - DYNAMIC + request-id: + - req_011CUQda41UKyvZxRwgf29iN + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "7987" + status: + code: 200 + message: OK +version: 1 diff --git a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-gpt-5-2025-08-07].yaml b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-gpt-5-2025-08-07].yaml new file mode 100644 index 00000000..035aeec6 --- /dev/null +++ b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-gpt-5-2025-08-07].yaml @@ -0,0 +1,126 @@ +interactions: + - request: + body: + '{"messages":[{"role":"system","content":"Think deeply about the following + question and answer it."},{"role":"user","content":"What is the meaning of life?"}],"model":"gpt-5-2025-08-07","n":1,"reasoning_effort":"low"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "216" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.5.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.5.0 + x-stainless-raw-response: + - "true" + x-stainless-read-timeout: + - "60.0" + x-stainless-retry-count: + - "0" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.8 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA3RVwW4URxC9+ytKczLSLLENxrA5RIgghYggEkyiKCCrtqdmunBP96i7escjhOSP + yAUp+Tl/SVQ9Y++iwGW1mqp6/eq96uqPBwAVN9UaKmNRTD+41bPz3P78x7k/a56GP48//Pr72+nq + +YOrt29+OT+3Va0VYfOBjNxW3TehHxwJBz+HTSQUUtTjs0fHJw9PHzx6UgJ9aMhpWTfI6nR1cnRy + ujp6vDo6W+psYEOpWsNfBwAAH8uvMvQNXVVrOKpvv/SUEnZUre+SAKoYnH6pMCVOgl6qehc0wQv5 + Qvo1hcERWNwSoE8jRWpALCdgDz36CUacUg3oGyA0Fix31nFnJQFCi4YEQgujRYGb6396Qs++u7n+ + Fwx62ND6nX/nV/AbOe445PRdGjiyZHRrOA/gwpZqGENMlocaQgR03HkYWSyk0JNY9h04jB1FEIse + ppBjItfeV9yfco+ek7ApcJGabAhSbluKWqi02asJiWAk51YbKvz+bkO8gyppQSzFVFCfX3FSgVhZ + vuRW9UngAwyREgksXX6vADA7DCwgNobcWVi8K6gm9D1LT14W6G1wWccD41Qopxy3XMRvYMCUIHhg + 34bYo6bVsMkCVttMRVKJ6JMh30BPkZZydAX8FcaIwlsqyJjFqqCQJMRJud5cf44EQwy5gdDWs8om + WIrkBbboMqUauhhGsbPlkVyhofZoA+/8UxNDSkqj4RKptW8iEEs9JbgkGiCSyVENWIMJ3pPRTDj8 + Aq0u2mTPMt3T/14ib/KcmChu2VA9i1tkMBhJ8JJ9d2+mNtOEQ0cY1Y0aekxCcarBWIxohOK9QvlF + W4wa0cveTA2awgadSsyuuXUVNtNaxXzmMHI76ZdZGTgcIwuVqQEJA7S3vo120vYn6FGWU1fwwm8p + iZazh2ZWZa/9UphsGDUjlyDjxk2l9sdQviZqs9PbcamjL6CyUCo9BE8QafH99dyK1iwa6JmaMrca + thRhIlzG+w3RZbkcIy1KorDkhuDQo+RINWCU2RPqh5n03NR5cQAipSH4xBt2LBPoXdLTLMZmOVIF + b7jxN9efBQzmRKX+JeG2iO/w9oYM8wZC2LDAhlTAu4sObci+KaM1Dx8k1v0KQknWsBjreHu7tEac + 2Rwfzf3qbsmuKWlKmoxeU4KBYgrzERsy2M9CaETBUglIyMZS88P+5ozU5oS6t312bi+A3geZ3dWd + /X6JfLrb0i17TvZCN1HwunmThKEq0U8HAO/L1s9fLPJqiKEf5ELCJRXYk7MZrtq9MrvggyePl6gE + QbcLPDw5rb+Cd9GQILu0925UBrXjXenukcHccNgLHOx19386X8OeO2ff7VCOTx5/84BdwBgahJqL + IVLD5sumd2mR9B3+VtqdzoVyteyXC2GK6kVDLWY3v5FVmpJQf9Gy7ygOkctDqXYffDr4DwAA//8D + AGEqaaElCAAA + headers: + Access-Control-Expose-Headers: + - X-Request-ID + CF-RAY: + - 99336a248e26f9e0-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 23 Oct 2025 18:49:35 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=IEfUxr6qz04fheqdJ9TXvDtuHw5DWtNDNH1y8WvLGYw-1761245375-1.0.1.1-3D2UF8kvGhlq5mUgqubbbyKd0fe7wXf7WJjLR0QWdEH500jcCtJx97ko2E8MjnB_EbXU.Z65HdJbs71QdAHYa4Ssz0I6wjhG_JEN6O1qXwQ; + path=/; expires=Thu, 23-Oct-25 19:19:35 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=gD__Dj8uH38zQg6EoElBHvq.2Mz5NvBBj3m.fAKEhg8-1761245375036-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "5790" + openai-project: + - proj_RpeV6PrPclPHBb5GlExPXSBj + openai-version: + - "2020-10-01" + x-envoy-upstream-service-time: + - "5888" + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - "15000" + x-ratelimit-limit-tokens: + - "40000000" + x-ratelimit-remaining-requests: + - "14999" + x-ratelimit-remaining-tokens: + - "39999976" + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_b30a25a833cd4ce28fe17fa05f78e109 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/lmi/tests/test_llms.py b/packages/lmi/tests/test_llms.py index 7d5d7310..9456b490 100644 --- a/packages/lmi/tests/test_llms.py +++ b/packages/lmi/tests/test_llms.py @@ -927,35 +927,49 @@ async def test_deepseek_model(self, llm_name: str) -> None: assert result.reasoning_content assert outputs[i] == result.text - @pytest.mark.vcr(match_on=[*VCR_DEFAULT_MATCH_ON, "body"]) + @pytest.mark.vcr @pytest.mark.asyncio - async def test_anthropic_model(self) -> None: + @pytest.mark.parametrize( + "model", [CommonLLMNames.GPT_5, CommonLLMNames.CLAUDE_37_SONNET] + ) + @pytest.mark.parametrize( + ("reasoning_effort", "expected_len"), + [("low", (200, 2_500)), ("high", (900, 10_000))], + ) + async def test_reasoning_effort( + self, + model: CommonLLMNames, + reasoning_effort: str, + expected_len: tuple[int, int], + ) -> None: llm = LiteLLMModel( - # Using 3.7 sonnet for its reasoning capabilities - name=CommonLLMNames.CLAUDE_37_SONNET.value, + name=model.value, config={ "model_list": [ { - "model_name": CommonLLMNames.CLAUDE_37_SONNET.value, + "model_name": model.value, "litellm_params": { - "model": CommonLLMNames.CLAUDE_37_SONNET.value, - "reasoning_effort": "low", + "model": model.value, + "reasoning_effort": reasoning_effort, }, } ] }, ) - messages = [ + + result = await llm.call_single([ Message( role="system", content="Think deeply about the following question and answer it.", ), Message(content="What is the meaning of life?"), - ] - results = await llm.call(messages) - for result in results: - assert result.reasoning_content is not None, "Should have reasoning content" - assert result.text is not None + ]) + assert result.text + assert expected_len[0] <= len(result.text) <= expected_len[1] + assert result.prompt_count > 0 + assert result.completion_count > 0 + if litellm.get_llm_provider(model=model.value)[1] == "anthropic": + assert result.reasoning_content def test_json_schema_validation() -> None: From 858cc79e74bbb156760df2d2f33f64526138f0d0 Mon Sep 17 00:00:00 2001 From: James Braza Date: Thu, 23 Oct 2025 12:37:24 -0700 Subject: [PATCH 3/3] Bumped from Claude 3.7 to Claude 4.5 Sonnet in reasoning effort test --- packages/lmi/src/lmi/llms.py | 1 + ...cted_len1-claude-3-7-sonnet-20250219].yaml | 118 ---------------- ...cted_len1-claude-sonnet-4-5-20250929].yaml | 127 ++++++++++++++++++ ...cted_len0-claude-3-7-sonnet-20250219].yaml | 110 --------------- ...cted_len0-claude-sonnet-4-5-20250929].yaml | 114 ++++++++++++++++ packages/lmi/tests/test_llms.py | 2 +- 6 files changed, 243 insertions(+), 229 deletions(-) delete mode 100644 packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-3-7-sonnet-20250219].yaml create mode 100644 packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-sonnet-4-5-20250929].yaml delete mode 100644 packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-3-7-sonnet-20250219].yaml create mode 100644 packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-sonnet-4-5-20250929].yaml diff --git a/packages/lmi/src/lmi/llms.py b/packages/lmi/src/lmi/llms.py index 481cb221..116e9dc6 100644 --- a/packages/lmi/src/lmi/llms.py +++ b/packages/lmi/src/lmi/llms.py @@ -83,6 +83,7 @@ class CommonLLMNames(StrEnum): GPT_35_TURBO = "gpt-3.5-turbo-0125" CLAUDE_35_SONNET = "claude-3-5-sonnet-20241022" CLAUDE_37_SONNET = "claude-3-7-sonnet-20250219" + CLAUDE_45_SONNET = "claude-sonnet-4-5-20250929" # Use these when trying to think of a somewhat opinionated default OPENAI_BASELINE = "gpt-4o-2024-11-20" # Fast and decent diff --git a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-3-7-sonnet-20250219].yaml b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-3-7-sonnet-20250219].yaml deleted file mode 100644 index 81f464ac..00000000 --- a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-3-7-sonnet-20250219].yaml +++ /dev/null @@ -1,118 +0,0 @@ -interactions: - - request: - body: - '{"model":"claude-3-7-sonnet-20250219","messages":[{"role":"user","content":[{"type":"text","text":"What - is the meaning of life?"}]}],"thinking":{"type":"enabled","budget_tokens":4096},"max_tokens":8192,"system":[{"type":"text","text":"Think - deeply about the following question and answer it."}]}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - anthropic-version: - - "2023-06-01" - connection: - - keep-alive - content-length: - - "295" - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - litellm/1.78.3 - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: !!binary | - H4sIAAAAAAAAAwAAAP//dFfrctQ6En6VLu8P9oADuR7Y2R9bSSAhIRxCIGFh2Upp7LbVjKz2UUue - OBRV+xD7hPskW5LtyQ1+JTNqtfry9dfffM8aLtFks6wwKpS4trX2fE3YWvRrm+ubO+ubG3/L8ozK - bJY1Ul+ub5yGvfOgLhRSwBenu282yt/r83mWZ75vMVqhiKoxyzPHJn6hREi8sj7Ls4KtR+uz2b++ - T/Zek12QraOH6d9Z9lGTAAkoaB1XHGwJrSbDwq2mQhn4M6B4YgteKw9aCcwRLZQ4Vx5LmPc39ugk - B6+RDdekrOSgbAnpMXQCXjsOtebgQYdGWdAknl3/FD5qdBijsAxCtjaYQ7DUoRNlTA+qdojlWmjZ - grKyRPcUTtBDg4BXrWGH0ClHHARadNJi4alDmX21X+0ZGqp/drQGb5XtwQ3nVkBCXaP4IVGvERpU - lmwNXIGhKgXoGQRdh3DIJbADBZpqjQ5aXqLLoWJjeAkldWQRCm4aZcsGrR+LoQpN2CGIMp1KZWUH - aA3V2qONhjGwfe1IPClLvp/FNxeWlzkY7nBws4ohWh+JUU0yW7ITTS3sGqN0MjTUIaiiYFfGTDyn - xN4Hp2y8uhfKUpMMtzE6DlWFLprO+zHY+OFBhK/JlmG6OCXV8EK0gr8amqMbsqscN+nJoi8Mxko6 - nJPz+rfYm9M7SHvQn1dXJBHFpAyJl+h4gfBBOe8wZbevmiC/zeIDK/yQjR+sn7r3d1giNEE8FA6V - R+DggJd2Oo8P/UGa0hOzodER5ZaB59+GcG7bvsaS7WD7a4w06ooaukZoDSoJDuPNc0+GvHJxNGZ3 - rLRqW7IokrJaojFrc4xeK3bplTqFLh5saOboorcPnqkY/KQmd+R84CCmB7Jjy5UtEJbkNVjlw1g0 - h0rYxvK/5RKdhVb6QqeZ/WkXLmjh2cGBU3ZhZlCRTVCaEh+nGtrgWhasgokwXNxGa8HBqRpjXJUq - 4qVYqkcChVbGoK1R4kOnLJSqvQqon93CoALxqX/V7Qqtnp8uY8MReZID2lrVGBGbxylPgIzjIfkU - /DSTBTetIdETul91bEI0V66/E00c/+C6NFSplK3jMhQYq/nOJx64X74P3CAoVwcE8o8E1DwSoEE1 - JrUK3GCtij5eSZ5u+OjWteidrTJQO156PXKBqdZU4YMydJ2SXHFb7NWqUWTvVmFsjbUx2sh/CSic - 3l4Fft9DmqFU5qvWoQglRk5+vKN5GJ7/ao9ANAcTixu5y2BZ40SsJDdLhQRKxNb0q9SmpXF/I8DX - rGDnsPBfs9USeEmRruK0t8iteRhvuTKIa6gWmCvBEtI6Q3LQKRNQcpijIaxGksarFh2hLVCewiqT - 1nFHJYICryPkfBVMDnNl4pRFLEjLVsYs4ycsvNwKYMnOlB3hUmCpKZHhSLUSK12jDXFnkJVItfI0 - yzOhehjcbJa9cud7+4vPR/t79f7RYu/i8OPnNwdbF82Ben/Fu/xalq/Pdq82xextbZ683/5Me7zR - 7l7/c30/mJfdp3b34tlic2dxeHZ97l4efHRPnnw5fvLEPFs/PNfHzcnns92z41f1lVnuhYa/vQ8O - j82H5wv18lV3tHVxzYsw5+Pynd3sj5Z4tv7py+56ub0jm5snhXpnFi3bA+b1g7D4dlxdq2dH3VW/ - 2W0fN/7L3Ibndbsh+jps9y+36HD5Rm8WZvtEt9fv95r1+vqQ/3hRdfvP3ZuT/Xfqxeed5bPtQh29 - Ubrez37kNxIGr6K4SX9m2V+iboC3NwR8QlWaxKRpVnJmhbXI6hRhWgcsBwlCvr8jSwZBko+cuYLe - Sov8FHg3Aw9kCxNKTMLjZ/OTGOHx47MHcxjJ8vFjWEvT8HAqK9WQ6XOoIixLmWi1aUJMYXB6OhDw - g3FMbhu1uEc3EzCLxMvD2D9Kq0RGf3e5JrmJfskGZSKBuZRUfG6OBTfxQx+Xqwpex61dJGoanTkU - tH6t4cixoJbKYVx4yeu0Ur5xP7pTwac1hh26vlT97ZG8K+rUA8HqnSppqF2asCQNsRxb91Pl1zJF - hhi03VCnSdaNRWU3abrWDNLpvjiZ2HqJvxAaaS3blT4x/XRg4uIfIYYr5r2tqiOYUxhpyWN+Ixjy - GNmSpOQGlECB1rsIU2pSnU7RadVK0hBjVQ0lnKavBJUrNJCPjfrff/6bqAtVoUc2HnRTSVJwhw6W - 8XjuEo96jQ1UwVRkzLBib0A7rlUQjHzI1e0KDLQby/KNg7PYP41xfoqe6zRCCUJJSk23UouHPvwj - +/HvPBPP7eUgY7JZhra89MHZbDwQ/DNEoGQzG4zJs5B+Jc2+Z2Tb4C89L9BKNtvZyrNCFRovh53G - 9vKuwfp07lCVvzqb7kb/2Gps0ClzudM8tL853dD3T3/kGQd/+6vfN7bzLKGxwEtP6LJZFn/blcqV - 2Y8f/wcAAP//AwCVT2Z/Tg4AAA== - headers: - CF-RAY: - - 99336b1bd8291742-SJC - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Thu, 23 Oct 2025 18:50:19 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - Via: - - 1.1 google - X-Robots-Tag: - - none - anthropic-organization-id: - - f2c99ed9-038a-406f-9cb5-1f840b758a20 - anthropic-ratelimit-input-tokens-limit: - - "20000000" - anthropic-ratelimit-input-tokens-remaining: - - "20000000" - anthropic-ratelimit-input-tokens-reset: - - "2025-10-23T18:50:09Z" - anthropic-ratelimit-output-tokens-limit: - - "2000000" - anthropic-ratelimit-output-tokens-remaining: - - "1999000" - anthropic-ratelimit-output-tokens-reset: - - "2025-10-23T18:50:19Z" - anthropic-ratelimit-requests-limit: - - "7000" - anthropic-ratelimit-requests-remaining: - - "6999" - anthropic-ratelimit-requests-reset: - - "2025-10-23T18:50:08Z" - anthropic-ratelimit-tokens-limit: - - "22000000" - anthropic-ratelimit-tokens-remaining: - - "21999000" - anthropic-ratelimit-tokens-reset: - - "2025-10-23T18:50:09Z" - cf-cache-status: - - DYNAMIC - request-id: - - req_011CUQdcX9gAmBe8qxd6k626 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "10343" - status: - code: 200 - message: OK -version: 1 diff --git a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-sonnet-4-5-20250929].yaml b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-sonnet-4-5-20250929].yaml new file mode 100644 index 00000000..fa4e8622 --- /dev/null +++ b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[high-expected_len1-claude-sonnet-4-5-20250929].yaml @@ -0,0 +1,127 @@ +interactions: + - request: + body: + '{"model":"claude-sonnet-4-5-20250929","messages":[{"role":"user","content":[{"type":"text","text":"What + is the meaning of life?"}]}],"thinking":{"type":"enabled","budget_tokens":4096},"max_tokens":8192,"system":[{"type":"text","text":"Think + deeply about the following question and answer it."}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "295" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.75.6 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//dJjpjqs4FoBfxWKkvlKRutm3+jPKQjYg+z53VHLAgAPYxDYQ0rpSP0Q/ + YT/JyCSpqtuj/hGFYPvsB76T35WQ2ihQ3hQrgLGNXjklBInX2mv9tVKq1EvtSlspKNhW3pSQu++l + sm6EXlqzzhNuN1abegUvN7XrRSkoIouQ3IU4hy5SCgqjgbwBOcdcQCKUgmJRIhARytt/fn/uFx4m + PiaulPC8fFPWHuYAc0AJAtQBwkMgpFyAiFGHxsQGMP9YGBEBLjHiAlPCASYg8nBAOY287DtYe4ih + bxwQCjgmboAAJDxFDAgPCoASxDKpALoMIamrkIu1seMgJgVHiPEIWQInclne/bKICceuJ/h3YCAB + QgRy84HwGI1dDySQYRpzAKOIUWh5iL/9ID9I+Tt4eZk/bcQWDH7R8vby8oO8Au2KuQwUhgHmIeCx + 6yIuOEgRsBiCAgEaM0BTAkIECSauPLREAXZznYJBG98jQh2BCIgoJgIICjhiCSZu0ZJptgQmLkix + 8PIA2zjBBElJU+zhXPFnZFMPCQ8xgIl3d/+hGCBpKpenOiceM1seQ+GJQQtxEGAHARvxCAsEAmj5 + Mpl/FyHDUsnDwqAl/ikkvYfFlORJYiiAuWUejvjdByoNzC2ZI8YpgQFwGU2Flx/gKHBeoSViGOBb + fvQhVTB8ivNIyPjQEMk8uiCAzL1XCpFVKI/fMxMhhhGx5J4zzQogoAm6F84JwVhk91TYcb6Dx7Jc + Hk5WpZOrvGaxg63iCdOAurnDCUbpw9EBoyFACQ1iaeQ3/jUYBcBjluAEBo8gRIzasfX0phsL4MUh + JBx4MEHAghG0sMiAQ9kz2q8hlD0GTiij0ubchkyaNwbco3FgAxcnCEAgPFnIwomDAvBkCMQv7QMt + n9A0QLaLuKyfv3XaD8WijCFL/FCex1IPB+jeR9KCmCMnDoDDYIhSynyem/l8CAB4orGQP/lHGX5X + CgrHLoEiZvLZoV34pOcfxr2u29P9bjxqkvGyuOODFW7N4dFO46EeEqfKw7JuztDwFNYm3pKUs1sK + q71lvPWzejcjjdnYb9Os6Pq19bwFL+VxZ90mt8aCDI+a5l79xoEE89qpwvZlHvR82Nf2xdHEtAcH + p37wRlM9G6fL1l6NiuYiYEOatSp7u1Q5bp0DVZu9xizuJhd1pCa0Uke9U60+6FfxWq+5K7pc7t3x + bFKkejQe141440eVxfJ83O0uw5qxnnRn2XWVucgul/slYZl7M5uMRqVi0bS2J1vt6NdBq47Thq2b + ++2hd0z5pmFParOEj0pnZ1u6hiVvYYaTcp/OrN75MgxraDtZ9s67+CYaA6PRNZpklV3PRU+f+8vm + 3g931sZddS47AYdZY+UnGta1sEx27WLadU+dze1a5qmYDpYV98p2ndRoiQM9mCSMVMPzm+ZoR2A9 + 5maCx/IR2V2UTDZraelUzconaGZismtOmVa7HqE+5DRj5+1Ub613FZO1m/P2vNbIeldx4roQw9Lh + JvrWPu45qXG5bo7hbKpNh/t0X2Q3l1f5VYvCzmV6jTOrUXXtKB2SbrNWnczStqM2+S6ENStxQr3H + d77duFTKkFC0rxPjsuwMU6cDB/Cgina5VI4aC7tWbS2b0zZOtvMqdduu113o6kxfXviwYi0nPuq4 + Z79P+7MLL4viRW1UbnVaguWzV+PBIhvCToWL8/k0pxtjcg4t2ui0x3qnPXcCxsimXMfWLLZGVti9 + aVcXqsX6ruKuOiV7uB8ilRhb9eye1qdBqg/sQRtHXkl4aX3cq5TOQXAtFsMhXjfC1bo0a5eaSb83 + rVnD9U6gzs1I1I49JerAmQ+SJquIOOnWQjfQi5fbcV3t+4ZpLAbrQ3Pspnu6MxMt3JzYkVcrtK5P + +RXjxaw0I91p5zobVQ7evr+G/ZAtztdWu7kmi2addkm0LhuiWkq0WmVdWeNeJSqny7PTtkfrQ2iF + Jl83T0lrUYqmIR9rxbK/TrPZtrjoatNGy6ium44fGRc0ZNk69SfNVt1Q+x4y4268qFQ3l33jqDlu + B2+czmYDK7fZ2dWJRkR2bq4CjNtbbe24u/1h4ZR2rfM8FIdasKu29qWg1hhvw1VzO90lK2u9uNS0 + 0q0aqy1Ratcuh2t5HLJ+vNiumvt9Npl4bXKuV+aZdp3jYSD8BexdgsU12ztI3x2JU66z6nY73O68 + RnNgRuOx0VSLwcHZbg6Xy2rWSsNqdTrXg0oPp73KtWY7sbVJpkHUPkV8xFvrhM0m5shNI2E75+1Y + m6nJxtj1V1pw7GzZyFAN7dQ06aKmkaNqDbpNa+RYmh4U3WZl263V0bWb9tBxfFthxyybmkvnjglJ + aQqn2e7mROUmauyG7HZxvKnfIPYquflTd3hTd1m9c12Yc2M5wBjveQp7TRPqmtnWydwsJed4Mhkb + eK8OqzU6GRh8bqFJgDTLOKzP9Z7e9Oet4oRsK9621XXTo826q5m78a5D9TKqjpDbcPoD4zDRujZf + F2nQEmXr0ujPV+lFv6xuC42b+5nqtdW+htRWYuy8auY3taSj120r3aQx9gXfH7ZtZ9EdWe3b6ugc + F2w1b1/1xkEfrcxS19zGq3DfzdZLzZ6Pzpt11i4ZtzgsnlSVJatDMLkOezgaTTfNmIwWu82odt0u + PRQecNUfHdkV0omf8Lh26Q+hOjpfYrWKrXAaQq062VcrxblRbzaipXpaul3lZ+GTQ9FVEmr+9ab8 + S7IjMB+MQx1gYAfJl+Q/IWUBnOTLFzEEIEM5R/yKj/k7M4QkAw4m9vNt7MRBjoUvLz3JdVLZQaLd + LCVP7ZIKTHnug2wR408kzAnrG/+gMczJN0nJyEYCsRATZP/1x5+f2IjFB6BKNZZHsYV4ASQwiOW3 + pAqLhiEWISKCf3/Y9slev0nM/CQvaZyRA5AjqciWNz+lxETCxwNBGU2wLWkQRcCJAwcHgdSRb45i + FlGOHuqGd2r77QudUXKHoz5KUEAj6WomHYjoA5Pl4ggF+codBAFlQGLFnexSyOx/oDsp6Il3BoJJ + fkDu4uCEhPiyDdxnD+Gh8G7pBw0i8BuYM8TlpbR0gIkt5XzkhQArZphyLLLCAxQLd368k6Pc5MRB + kAFEXOgi+860OUBjIc17BEfW5eLBQ2CcrwATSjPzZKxlmWU0/iYLkft3Bn1MD5/u4zCiTM5kb09e + 5AiFICb4EqMgAzbDCSL3gQH5H+mRYO1BydsMcRF8mR6+QNpff/zJYD4r5GFzHpGA8irnVtkuf/3x + J+YggkzI5krvveE/J4bP5sidfn19vfu+k/sYkmwv0IP6Mxr/++UF9D/msrsCDhwsfhnlaJQ369f5 + LlfGBXQRz4fFj3TluZZtKqgNMxDKQS9ncgnGPF+2aW7b+MG+AObeYCsOIAMw73vp2lMiZc8oPrOD + rlFAcxBmuXhC038rP/9bULig0TtDkFOivCmI2O8iZkR5LHB0iWWVKW8kDoKCEufT9tvvCiZRLN4F + 9RHhylu9WlAsOXa+552PKXn/dUPpuc4QtP9p7XlWykeRh0LEYPBeD/9//+dq2fv76s+CQmPx9Vat + Xi8o+SxqoXeBEVPeFPkfgQ2Zrfz8+T8AAAD//wMAZxvKv5YQAAA= + headers: + CF-RAY: + - 9933aec89c3a58ac-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 23 Oct 2025 19:36:32 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + Via: + - 1.1 google + X-Robots-Tag: + - none + anthropic-organization-id: + - f2c99ed9-038a-406f-9cb5-1f840b758a20 + anthropic-ratelimit-input-tokens-limit: + - "25000000" + anthropic-ratelimit-input-tokens-remaining: + - "25000000" + anthropic-ratelimit-input-tokens-reset: + - "2025-10-23T19:36:22Z" + anthropic-ratelimit-output-tokens-limit: + - "5000000" + anthropic-ratelimit-output-tokens-remaining: + - "5000000" + anthropic-ratelimit-output-tokens-reset: + - "2025-10-23T19:36:32Z" + anthropic-ratelimit-tokens-limit: + - "30000000" + anthropic-ratelimit-tokens-remaining: + - "30000000" + anthropic-ratelimit-tokens-reset: + - "2025-10-23T19:36:22Z" + cf-cache-status: + - DYNAMIC + request-id: + - req_011CUQh8rN1Ro5LL1uE9Ps9V + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "12001" + status: + code: 200 + message: OK +version: 1 diff --git a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-3-7-sonnet-20250219].yaml b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-3-7-sonnet-20250219].yaml deleted file mode 100644 index b4214387..00000000 --- a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-3-7-sonnet-20250219].yaml +++ /dev/null @@ -1,110 +0,0 @@ -interactions: - - request: - body: - '{"model":"claude-3-7-sonnet-20250219","messages":[{"role":"user","content":[{"type":"text","text":"What - is the meaning of life?"}]}],"thinking":{"type":"enabled","budget_tokens":1024},"max_tokens":5120,"system":[{"type":"text","text":"Think - deeply about the following question and answer it."}]}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - anthropic-version: - - "2023-06-01" - connection: - - keep-alive - content-length: - - "295" - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - litellm/1.78.3 - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: !!binary | - H4sIAAAAAAAAAwAAAP//fFbbchs5Dv0VFF9mttKWZSu+6WUrvsVyxk4i32KPp1xUN7qJEZto8yJZ - SblqPmK+cL5ki2zJkp3dfZKaBAng4OCAP0TNBWrRF7mWocC13trOmmNj0K9tdje3upsbeyITVIi+ - qF310N0Y3vVOP13+dnrDo5vrD9f+bEZXp15kws8ajFbonKxQZMKyjgvSOXJemmiTs/FovOj//mNh - 7xWZMZkq3rD42xeXihw8BnSe2IAccfDgFUKN0pCpgEvQVCKQAzYYP9MuOw+N5ZKDKaBRpNlxoyiX - +uUyByrU0jiQbtyBgf/FwYi9ggKx0TNo0Do2UoM0BQRDE7ROaj2DFHrdaOmx6Nybe3Op0CJIi1BL - 0x5sMPc0QQcDyKWBglwenOtH640ODFFTRRzcumvIkg9Svz7FpUcDLlQVutfpkpmwjiZ5LE2eQPEM - EgqakEFogm3YYQYO7SQe+MhFBmxB5oowrSx9otFUKY+mRuNTKpsdOHoiF0tDUlMEcYEdWgeaxggX - 0nqLK9FJ35ZASQeGgUzEw/hFzBmMgocpQm5R+mUqXlkOlQIOFnLFlKNLWMuUlUvx9DpwEotEzlP+ - GqOF/8V1OdfooLRcr2DjYEpeAfsYfpZKZ2kUfPLP4Dgn9LMs+X0peGV56lXy/74D+8Saq8ScV/7r - CB2UVtYI92Iexb0AMuDR1rGI4IKd0GTOIYuN5SKkuNLlWx04kU1DBp1bKzkPDle5inO8jxrKg433 - uzpWMnjS5KWldqWN4/9ypZZPVNP3uDZFrddGSKZKIQzAKQ46gj42PNVYVNhW1Me+IwdBe6qlRz0D - uYTopSETvIbBkal0BCJnazH39wKkcVO0HRjAVBof4W4sT6hAkOBVLL0vgwaLrmHj5m7jF+beQUFl - 2dJoylYXE8JpSw80OQcrK3RgsdRtnTsiE44qI32wUUuO7NX+wfh2cLBfHQzG+91GUugNH83eDQ56 - +8ePo/NPF3ty42DEo+v3p99wfHgre7fHd38eHJ0dedvsXXc/V+piuP/lu7kd4U733d1336zr2+ZD - c3c9PJwUT/lRNf3y4brUzbGhsfnYTD+8l4cHV8fbk+2wXm7vnn8e6pvZYNpdv9pS9sx+/kLb+vGd - 9Sd08qnZOf0qy5uvd/zn7bWfDfzl0/nI2PPe2fm32beDq3p8c7l7slutH158Ut1vl7tbGvF0d/to - Nty53FjXh9vq6+Hex/2j4/dhui5nZzvvhsPqQDxnSz3Fp6TG8eetjsZWpdgLVVjlXGoSrzAxXhrX - NkZS49j+846NEpykExQ5z3a2MEOLv7gVNryo5oILkXIXXCOUZIoVni6JtdK6//z1d2SzRS3Tt6LG - RRkZBYqEbT2C5gnGVaeiAK90ewc+p98kvjxBu9DGF+FJehQDaNhRbGrIlTQVxib7WSlqjDhUoKWt - 0Ea2mhhA7TD2WErtLMq/XYg7eCsLalWII5lbsZiyHTso2f4khEthXqKQwVSRxjcj7JUQ2RRzEj5c - 1e4afl2KbhRZnpqFz3/FnGq2CAVGtaK5wqY2S7l8QatkBDx2ZS3H6FpJeKGQ4+WAJbcQDZyXOi69 - GaQduPwvc7uVrxHC/KYEKrCpOM0qlDZXQN6hLv/56++F9zRWFoVNH5EPy4K+qfTKiMlWKdYew6cG - LaHJ52W8ianIuQ5xCbN4fD7hJgglat8+MOa5RBXzHM3+LZ7/yITz3DxYlI6N6As0xYMP1oj5hsPH - EH2JvglaZyKkR1L/hyDTBP/geYzGif5WLxO5zBU+tFmxeXht0F3sW5TF/9pbnI33Y6OwRiv1w1b9 - s/1yd0O93X3OBAe/utTb2cxEemLk+OAJreiL+LQrpC3E8/N/AAAA//8DACWxE7dNCgAA - headers: - CF-RAY: - - 99336a4a7e6015eb-SJC - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Thu, 23 Oct 2025 18:49:43 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - Via: - - 1.1 google - X-Robots-Tag: - - none - anthropic-organization-id: - - f2c99ed9-038a-406f-9cb5-1f840b758a20 - anthropic-ratelimit-input-tokens-limit: - - "20000000" - anthropic-ratelimit-input-tokens-remaining: - - "20000000" - anthropic-ratelimit-input-tokens-reset: - - "2025-10-23T18:49:36Z" - anthropic-ratelimit-output-tokens-limit: - - "2000000" - anthropic-ratelimit-output-tokens-remaining: - - "2000000" - anthropic-ratelimit-output-tokens-reset: - - "2025-10-23T18:49:43Z" - anthropic-ratelimit-requests-limit: - - "7000" - anthropic-ratelimit-requests-remaining: - - "6999" - anthropic-ratelimit-requests-reset: - - "2025-10-23T18:49:35Z" - anthropic-ratelimit-tokens-limit: - - "22000000" - anthropic-ratelimit-tokens-remaining: - - "22000000" - anthropic-ratelimit-tokens-reset: - - "2025-10-23T18:49:36Z" - cf-cache-status: - - DYNAMIC - request-id: - - req_011CUQda41UKyvZxRwgf29iN - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "7987" - status: - code: 200 - message: OK -version: 1 diff --git a/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-sonnet-4-5-20250929].yaml b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-sonnet-4-5-20250929].yaml new file mode 100644 index 00000000..115e2c29 --- /dev/null +++ b/packages/lmi/tests/cassettes/TestReasoning.test_reasoning_effort[low-expected_len0-claude-sonnet-4-5-20250929].yaml @@ -0,0 +1,114 @@ +interactions: + - request: + body: + '{"model":"claude-sonnet-4-5-20250929","messages":[{"role":"user","content":[{"type":"text","text":"What + is the meaning of life?"}]}],"thinking":{"type":"enabled","budget_tokens":1024},"max_tokens":5120,"system":[{"type":"text","text":"Think + deeply about the following question and answer it."}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "295" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.75.6 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA2RWaXPqOBb9Kyq+vJ5Awg4mVVNTQMweNrOEzKuaku1rWyBLRgvGdPV/n5JJ0un3 + PiGk66Oro6tz7p+FmPtAC88Fj2Ltw6PkjIF6bDw2H2uVWrPSqXUKpQLxC8+FWIb/q1TXp1bT70nL + aTmnVzcWrFc/qmmhVFBZAiYKpMQhFEoFwamZwFISqTBThVLB40wBU4Xn//75Ga8iwk6EhQbhc/hc + 2EREIiIRZ4B4gFQEKOZSoUAzH8fAFKYoiQjlkicR8TBFZw1SEc4kinSMmURYnp7QJgIBPyRiHEnC + QgolpBm5gJCY0gxhz4NEgY8wkymIEnK1QmPkYYZ4EIBAkseAVMR1GKlAU5SAkAl4ilxAPv9kP1n1 + Cb0QEwpM/ZIRZj4SQElIuJZICeyTe4J36AsW+cJ9a/mT1Z7QQPAYYSTB0xSLcn4SIhW6EEhLKAbM + CAtzXgIFLM/OkBaiFJAnACv4yepPqM/jmDPDWgwSEeZR7cOzSQbnGUQkkSUUCp6qqIQE+NozIFKb + xAgLS3ewfATXBAQB5uX/KL8QFpqDj5GMuKY+CskFEEYupph54Je+0yVAJpxJQyFWCHsnxlMKfggS + xZoqklD4B6UojQgF5IJJJwSmCYOcxwhoEmj6VCgVJAkZVlqY4rGFHPRPh3G/F/anp15cbarVPOqv + b+d0Wl3vF8dOE2NWd5LRYp8kw9mhXSGTY+NmjS+VuX3kztS9VnvkuGjaB7tuNYIDnx+O16UHjcNU + 1Obj675jh9m4zlYXYLc3EtJG2uvilwWtL/S1OWkce3vZtovZOG3eWsui+yq8zLbbrVPXx9V5D5rb + VvNAN6RfkZPaQHXUupc6zmQb1V8z3erErdt+ceC99taZnsf2daK9Shy/TC/rCjnTnZhMUnuRWjt3 + OKvELbIK36X19k7w0mmsbWu2SHp1fZ6OZjNVbFyH3bolwsPt1Hd2pLw97g51lxc5XSa8/Op1kmqt + 17lyL9x3Tq2Q4PKeNix7u4luyzoUw5Z961vJad+qLYqWW7W6w3YWJMNULNZErsqTfja81dWi37Xm + wc2LB4GTzM7nwyxxT4moEX8RBBmMSHJk4+2GOsPzLhXB+2Y0rpddOhtP3lZFnOFNb/pylcH+Zp2m + rVlbFLeOlg4eLl8Pnd31rbU4NazmPmoEtZV/2Fh0MXIiuonUmG+Ie2vf1qvBuTnxTzscqYO76wWD + RtLXfXc6SazUDSt6XrM4H0WzolqPYzuZx7EoTvrpsDm07Ncwqb/Hw3F1uBngS/9VX31ZtZtLq61E + z57v0+idX+q9tzcvHYzxcGM1jtkyIrbYdNkmxGp0S61ZfN55V396Pifj1oyO2ns2b/fZxlu9v1dv + ycvkvTqry9d1uVOc6aO9WGLg2+2akmZ/u7HilmtdKukR1lkyDcoduzjiXt8dT7JD87AdnJybGJVf + 3sqdG7MXuNU/8Jn0V9wZW83Kud4b9cUgXnetbNvbS7seNIrr98mpgY9vK0gbq2jGiFVtXvTstE2E + PrV2tU5lEuDdsVJu1dd64yxdjy44m8wwXW6HIa6ugtd5vekoq7gsD4rhS8iyln+9vhD1Fjd6+0H/ + 0B3YC0x02ynD4jjvznvj9TVYquHuIFlzU1mUhWf1ZOxNgrfmnNDl2nmzK0Fr/DKN59NZUGzOVv1h + 1/534a/S36IPV2MH+c9vYn9XPZX9kIhTH6T6W9tLuRioX0X9Q0PvKpNycZIo4ALBBUTGGTyhEQhA + WMBdz38T8YeH5T+U28itfH54+Mke0cODfSXSuBbBlEj18PCM9p9qi7gWiKfsS5pVJIz4IS/ixAOZ + p4u9u+z/4WChBJRQH8da/usO3nWlFv4H7owEgCKcn4yw6G4qH9B3bzI6jxkKCPPRkWcIsyzF2R1q + /ek1ZSchgiiNqQF9/UjN48YMAuMw340ApURFucH65GLklgsUcEp5mlsC9gT4KNEi4RLuXH24i+Ra + mCMmwI2M5yl9ORT7YO8FIEGe6Sk+SMh34+YGpVlf/+Y9X3JvxndsE9j/MKRvnvcHFqqEiA9YllCA + Y0KznNQlCMkZph8GlwNq5oMwXYif29cjsr/ZGnIBa5XlgSk3kfmOnClBXJ3vqvi3jSkW4b3aGMq4 + FhJocKemixJhrtsUkcInyFn4vICYhJFCjCvkwje0jGv0YNh7yG/4lwVXE+o/fNbVs0lsb6rcLHqm + orHLtTLTI57ms8ow9Y3ir/DcqE2+CKu8nk2LwE3IJgJ0wVSDzAOpCXQzc6T8YwGGTwUft2dC7v2H + Dwkw3zzcO+5XrwCyhFygBIKPJ5sanACASoS1iszuniE1+3xAlATwlPcV8v6+EUYJForknRDC+TPP + exoiv/TAfP5DAEoFSEXzPoio6D/IMRwSU+6mrr+iiTI3hQRcAFN5zykFAyAgbwclYPF/OstuBWEY + BqPv8l0XUca86LtIKG1kw5nWtPVG+u6y6fAPb3O+BBIIHL/c/hh1g3YwyCUmUnY5CixYApWqgifI + fKnzurBSp8mgLuprbxgl1UIlnlgybN8ZeOcHpodXRaHPwHblyi78Y2vvPJ/T7HbqJurPv/kX3Q3f + tBnEWt5L3b43yKzX0TOVkRUWy6s4DWjtDgAA//8DACamS/wjDAAA + headers: + CF-RAY: + - 9933ae8c7b9f159e-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 23 Oct 2025 19:36:20 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + Via: + - 1.1 google + X-Robots-Tag: + - none + anthropic-organization-id: + - f2c99ed9-038a-406f-9cb5-1f840b758a20 + anthropic-ratelimit-input-tokens-limit: + - "25000000" + anthropic-ratelimit-input-tokens-remaining: + - "25000000" + anthropic-ratelimit-input-tokens-reset: + - "2025-10-23T19:36:12Z" + anthropic-ratelimit-output-tokens-limit: + - "5000000" + anthropic-ratelimit-output-tokens-remaining: + - "5000000" + anthropic-ratelimit-output-tokens-reset: + - "2025-10-23T19:36:20Z" + anthropic-ratelimit-tokens-limit: + - "30000000" + anthropic-ratelimit-tokens-remaining: + - "30000000" + anthropic-ratelimit-tokens-reset: + - "2025-10-23T19:36:12Z" + cf-cache-status: + - DYNAMIC + request-id: + - req_011CUQh89EVEmzVStReZFrpd + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "9432" + status: + code: 200 + message: OK +version: 1 diff --git a/packages/lmi/tests/test_llms.py b/packages/lmi/tests/test_llms.py index 9456b490..4d8f5992 100644 --- a/packages/lmi/tests/test_llms.py +++ b/packages/lmi/tests/test_llms.py @@ -930,7 +930,7 @@ async def test_deepseek_model(self, llm_name: str) -> None: @pytest.mark.vcr @pytest.mark.asyncio @pytest.mark.parametrize( - "model", [CommonLLMNames.GPT_5, CommonLLMNames.CLAUDE_37_SONNET] + "model", [CommonLLMNames.GPT_5, CommonLLMNames.CLAUDE_45_SONNET] ) @pytest.mark.parametrize( ("reasoning_effort", "expected_len"),