OpenAI API Verification #71
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: Apache-2.0 | |
| name: OpenAI API Verification | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "21 01 * * *" | |
| env: | |
| LC_ALL: en_US.UTF-8 | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| contents: read | |
| jobs: | |
| verification: | |
| name: openai-api-verification | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: "Harden Runner" | |
| uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 | |
| with: | |
| egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs | |
| - name: Checkout meta-llama/llama-stack | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: "meta-llama/llama-stack" | |
| ref: "main" | |
| # https://github.com/actions/checkout/issues/249 | |
| fetch-depth: 0 | |
| - name: Setup Python 3.10 | |
| uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 | |
| with: | |
| python-version: "3.10" | |
| cache: pip | |
| cache-dependency-path: | | |
| **/pyproject.toml | |
| **/requirements*.txt | |
| **/uv.lock | |
| - name: Install dependencies | |
| run: | | |
| export UV_PYTHON_PREFERENCE=only-system | |
| python -m pip install uv | |
| uv sync --extra dev | |
| uv pip install -e . | |
| uv pip install pytest-json-report | |
| uv run llama stack build --template fireworks --image-type venv | |
| uv run llama stack build --template groq --image-type venv | |
| uv run llama stack build --template together --image-type venv | |
| # Temporarily pin litellm to 1.67.2 or lower | |
| # See https://github.com/meta-llama/llama-stack/issues/2043 | |
| uv pip install "litellm<=1.67.2" | |
| - name: Run Llama Stack | |
| run: | | |
| uv run llama stack run --image-type venv --port 8321 tests/verifications/openai-api-verification-run.yaml 2>&1 | tee llama_stack.log & | |
| curl --retry-connrefused --retry 5 --retry-delay 3 http://localhost:8321/v1/openai/v1/models | |
| env: | |
| FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} | |
| - name: Run verification tests | |
| run: | | |
| source .venv/bin/activate | |
| mkdir -p tests/verifications/test_results | |
| # Clear out any results from the upstream Llama Stack repo | |
| rm tests/verifications/test_results/*.json | |
| tail -f llama_stack.log & | |
| for provider in together fireworks groq openai together-llama-stack fireworks-llama-stack groq-llama-stack openai-llama-stack; do | |
| python -m pytest -s -v tests/verifications/openai_api/test_chat_completion.py --provider="${provider}" --json-report --json-report-file="tests/verifications/test_results/${provider}.json" || echo "some ${provider} verification tests failed" | |
| done | |
| env: | |
| FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} | |
| - name: Upload test artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: openai-api-verification-results | |
| path: tests/verifications/test_results | |
| publish-results: | |
| name: publish-results | |
| runs-on: ubuntu-latest | |
| needs: verification | |
| permissions: | |
| # allow pushing to the github repo | |
| contents: write | |
| steps: | |
| - name: Checkout this repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| ref: "main" | |
| # https://github.com/actions/checkout/issues/249 | |
| fetch-depth: 0 | |
| - name: Download test results | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: openai-api-verification-results | |
| path: artifacts | |
| - name: Add the new verification results | |
| run: | | |
| TODAY="$(date -Idate)" | |
| export TODAY | |
| ls -l artifacts/ | |
| mkdir -p openai-api-verification/raw_results/latest | |
| mkdir -p "openai-api-verification/raw_results/${TODAY}" | |
| cp artifacts/*.json openai-api-verification/raw_results/latest/ | |
| cp artifacts/*.json "openai-api-verification/raw_results/${TODAY}/" | |
| - name: Push new results | |
| run: | | |
| git config --global user.name "Ben Browning" | |
| git config --global user.email "[email protected]" | |
| git add openai-api-verification | |
| git commit -m "Verification results for $(date -Idate)" | |
| git push | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| verification-workflow-complete: | |
| needs: ["verification", "publish-results"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Verification Workflow Complete | |
| run: echo "Verification Workflow Complete" |