OpenAI API Verification Report #37
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: Apache-2.0 | |
| name: OpenAI API Verification Report | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "23 02 * * *" | |
| env: | |
| LC_ALL: en_US.UTF-8 | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| contents: read | |
| jobs: | |
| generate-report: | |
| name: generate-report | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: "Harden Runner" | |
| uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 | |
| with: | |
| egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs | |
| - name: Checkout meta-llama/llama-stack | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: "meta-llama/llama-stack" | |
| ref: "main" | |
| # https://github.com/actions/checkout/issues/249 | |
| fetch-depth: 0 | |
| - name: Setup Python 3.10 | |
| uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0 | |
| with: | |
| python-version: "3.10" | |
| cache: pip | |
| cache-dependency-path: | | |
| **/pyproject.toml | |
| **/requirements*.txt | |
| **/uv.lock | |
| - name: Install dependencies | |
| run: | | |
| export UV_PYTHON_PREFERENCE=only-system | |
| python -m pip install uv | |
| uv sync --extra dev | |
| uv pip install -e . | |
| uv pip install pytest-json-report | |
| - name: Checkout results | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| ref: "main" | |
| path: test_results | |
| - name: Generate report | |
| run: | | |
| source .venv/bin/activate | |
| mkdir -p tests/verifications/test_results/ | |
| # Clear out any results from the upstream Llama Stack repo | |
| rm tests/verifications/test_results/*.json | |
| cp test_results/openai-api-verification/raw_results/latest/*.json tests/verifications/test_results/ | |
| providers="$(ls test_results/openai-api-verification/raw_results/latest | sed 's/.json$//' | paste -sd ' ')" | |
| echo "Generating report for providers: ${providers}" | |
| python tests/verifications/generate_report.py \ | |
| --providers $providers \ | |
| --output verification_report.md | |
| # Fail the step if we don't see any results in the generated report | |
| cat verification_report.md | grep "Generated on" | |
| # Add the verification report to the step summary | |
| cat verification_report.md >> $GITHUB_STEP_SUMMARY | |
| - name: Upload report artifact | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: openai-api-verification-report | |
| path: verification_report.md | |
| publish-report: | |
| name: publish-report | |
| runs-on: ubuntu-latest | |
| needs: generate-report | |
| permissions: | |
| # allow pushing to the github repo | |
| contents: write | |
| steps: | |
| - name: Checkout this repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| ref: "main" | |
| # https://github.com/actions/checkout/issues/249 | |
| fetch-depth: 0 | |
| - name: Download report artifact | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: openai-api-verification-report | |
| path: artifacts | |
| - name: Add the new verification report | |
| run: | | |
| ls -l artifacts/ | |
| mkdir -p openai-api-verification | |
| cp artifacts/verification_report.md openai-api-verification/latest.md | |
| cp artifacts/verification_report.md openai-api-verification/$(date -Idate).md | |
| - name: Push new reports | |
| run: | | |
| git config --global user.name "Ben Browning" | |
| git config --global user.email "[email protected]" | |
| git add openai-api-verification | |
| git commit -m "Verification report for $(date -Idate)" | |
| git push | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| verification-workflow-complete: | |
| needs: ["generate-report", "publish-report"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Verification Workflow Complete | |
| run: echo "Verification Workflow Complete" |