E2E (NVIDIA L4 x1) #816
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: Apache-2.0 | |
| name: E2E (NVIDIA L4 x1) | |
| on: | |
| schedule: | |
| - cron: '0 14 * * 1,2,3,4' # Runs at 2PM UTC every Mon-Thurs | |
| # run against every merge commit to 'main' and release branches | |
| push: | |
| branches: | |
| - main | |
| - release-* | |
| # only run on PRs that touch certain regex paths | |
| pull_request_target: | |
| branches: | |
| - main | |
| - release-* | |
| paths: | |
| # note this should match the merging criteria in 'mergify.yml' | |
| - '**.py' | |
| - 'pyproject.toml' | |
| - 'requirements**.txt' | |
| - '.github/workflows/e2e-nvidia-l4-x1.yml' # This workflow | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| LC_ALL: en_US.UTF-8 | |
| TMPDIR: /home/tmp | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| contents: read | |
| jobs: | |
| start-medium-ec2-runner: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.launch-ec2-instance-with-fallback.outputs.label }} | |
| ec2-instance-id: ${{ steps.launch-ec2-instance-with-fallback.outputs.ec2-instance-id }} | |
| ec2-instance-region: ${{ steps.launch-ec2-instance-with-fallback.outputs.ec2-instance-region }} | |
| steps: | |
| - name: Checkout "launch-ec2-runner-with-fallback" in-house CI action | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: instructlab/ci-actions | |
| # clone the "ci-actions" repo to a local directory called "ci-actions", instead of | |
| # overwriting the current WORKDIR contents | |
| path: ci-actions | |
| ref: release-v0.1 | |
| sparse-checkout: | | |
| actions/launch-ec2-runner-with-fallback | |
| - name: Launch EC2 Runner with Fallback | |
| id: launch-ec2-instance-with-fallback | |
| uses: ./ci-actions/actions/launch-ec2-runner-with-fallback | |
| env: | |
| TMPDIR: "/tmp" | |
| with: | |
| aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| github_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| regions_config: > | |
| [ | |
| { | |
| "region": "us-east-2", | |
| "subnets": { | |
| "us-east-2a": "${{ vars.SUBNET_US_EAST_2A }}", | |
| "us-east-2b": "${{ vars.SUBNET_US_EAST_2B }}", | |
| "us-east-2c": "${{ vars.SUBNET_US_EAST_2C }}" | |
| }, | |
| "ec2-ami": "${{ vars.AWS_EC2_AMI_US_EAST_2 }}", | |
| "security-group-id": "${{ vars.SECURITY_GROUP_ID_US_EAST_2 }}" | |
| }, | |
| { | |
| "region": "us-east-1", | |
| "subnets": { | |
| "us-east-1a": "${{ vars.SUBNET_US_EAST_1A }}", | |
| "us-east-1b": "${{ vars.SUBNET_US_EAST_1B }}", | |
| "us-east-1c": "${{ vars.SUBNET_US_EAST_1C }}", | |
| "us-east-1d": "${{ vars.SUBNET_US_EAST_1D }}", | |
| "us-east-1e": "${{ vars.SUBNET_US_EAST_1E }}", | |
| "us-east-1f": "${{ vars.SUBNET_US_EAST_1F }}" | |
| }, | |
| "ec2-ami": "${{ vars.AWS_EC2_AMI_US_EAST_1 }}", | |
| "security-group-id": "${{ vars.SECURITY_GROUP_ID_US_EAST_1 }}" | |
| } | |
| ] | |
| try_spot_instance_first: false | |
| ec2_instance_type: g6e.12xlarge | |
| aws_resource_tags: > | |
| [ | |
| {"Key": "Name", "Value": "instructlab-ci-github-medium-runner"}, | |
| {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}, | |
| {"Key": "GitHubRef", "Value": "${{ github.ref }}"}, | |
| {"Key": "GitHubPR", "Value": "${{ github.event.number }}"} | |
| ] | |
| e2e-medium-test: | |
| needs: | |
| - start-medium-ec2-runner | |
| runs-on: ${{ needs.start-medium-ec2-runner.outputs.label }} | |
| # It is important that this job has no write permissions and has | |
| # no access to any secrets. This part (e2e) is where we are running | |
| # untrusted code from PRs. | |
| permissions: {} | |
| steps: | |
| - name: Install Packages | |
| run: | | |
| cat /etc/os-release | |
| mkdir -p /home/tmp | |
| sudo dnf install -y gcc gcc-c++ make git-core python3.11 python3.11-devel util-linux | |
| - name: Set XFS retries for root disk | |
| run: | | |
| # Locate our root block device - typically something like /dev/nvme2n1p2 | |
| ROOT_DEV="$(findmnt -nvo SOURCE --mountpoint /)" | |
| # Strip the lead "/dev/" off this device string | |
| ROOT_DEV=${ROOT_DEV#"/dev/"} | |
| MAX_RETRIES=10 | |
| RETRY_TIMEOUT_SECONDS=10 | |
| echo "Setting XFS retries for $ROOT_DEV to $MAX_RETRIES with a timeout of $RETRY_TIMEOUT_SECONDS" | |
| echo "$MAX_RETRIES" > "/sys/fs/xfs/$ROOT_DEV/error/metadata/ENOSPC/max_retries" | |
| echo "$RETRY_TIMEOUT_SECONDS" > "/sys/fs/xfs/$ROOT_DEV/error/metadata/ENOSPC/retry_timeout_seconds" | |
| - name: Checkout instructlab/instructlab | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: "instructlab/instructlab" | |
| path: "instructlab" | |
| # https://github.com/actions/checkout/issues/249 | |
| fetch-depth: 0 | |
| - name: Checkout instructlab/sdg | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| path: "sdg" | |
| # https://github.com/actions/checkout/issues/249 | |
| fetch-depth: 0 | |
| - name: Fetch and checkout PR | |
| if: ${{ github.event_name == 'pull_request_target' }} | |
| working-directory: ./sdg | |
| run: | | |
| git fetch origin pull/${{ github.event.pull_request.number }}/merge:pr-merge-${{ github.event.pull_request.number }} | |
| git checkout pr-merge-${{ github.event.pull_request.number }} | |
| git log -1 --format="%H %s" | |
| - name: Install ilab | |
| working-directory: ./instructlab | |
| run: | | |
| PYTHON=python3.11 ./scripts/install-ilab-with-cuda.sh | |
| - name: Update instructlab-sdg library | |
| working-directory: ./sdg | |
| run: | | |
| . ../instructlab/venv/bin/activate | |
| # Patch out our own pin from the ilab repo constraints file | |
| ilab_constraints=../instructlab/constraints-dev.txt | |
| sed -i '/instructlab-sdg==/d' $ilab_constraints | |
| # Since we reuse the virtual environment prepared using ilab | |
| # constraints, we should stick to the same constraints when | |
| # installing latest sdg. | |
| # | |
| # FIX: this is not ideal; a proper fix would require decoupling the | |
| # two repos in CI: either by removing the job completely and relying | |
| # on "sdk" (no ilab) test runs; or by preparing a separate | |
| # constraints file that would consider both the requirements files | |
| # for the sdg library AND for the ilab - so that they are | |
| # consistent. | |
| pip_install="pip install -c $ilab_constraints" | |
| $pip_install . | |
| - name: Check disk before tests | |
| run: | | |
| df -h | |
| df -i | |
| - name: Run e2e test | |
| working-directory: ./instructlab | |
| run: | | |
| . venv/bin/activate | |
| ./scripts/e2e-ci.sh -m -p | |
| - name: Check disk after tests | |
| if: ${{ always() }} | |
| run: | | |
| df -h | |
| df -i | |
| stop-medium-ec2-runner: | |
| needs: | |
| - start-medium-ec2-runner | |
| - e2e-medium-test | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ needs.start-medium-ec2-runner.outputs.ec2-instance-region }} | |
| - name: Stop EC2 runner | |
| uses: machulav/ec2-github-runner@fb91019e71385fb10dfcbec812b4de8c61589f7b # v2.4.1 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ needs.start-medium-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-medium-ec2-runner.outputs.ec2-instance-id }} | |
| e2e-medium-workflow-complete: | |
| # we don't want to block PRs on failed EC2 cleanup | |
| # so not requiring "stop-runner" as well | |
| needs: ["start-medium-ec2-runner", "e2e-medium-test"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: E2E Workflow Complete | |
| run: echo "E2E Workflow Complete" |