Skip to content
Python Build

Python Build #3010

Sign in to view logs

Python Build

Python Build #3010

Workflow file for this run

.github/workflows/build.yaml at 22360ae
# The following workflow provides an opinionated template you can customize for your own needs.
#
# If you are not an Octopus user, the "Push to Octopus", "Generate Octopus Deploy build information",
# and "Create Octopus Release" steps can be safely deleted.
#
# To configure Octopus, set the OCTOPUS_API_TOKEN secret to the Octopus API key, and
# set the OCTOPUS_SERVER_URL secret to the Octopus URL.
#
# Double check the "project" and "deploy_to" properties in the "Create Octopus Release" step
# match your Octopus projects and environments.
#
# Get a trial Octopus instance from https://octopus.com/start
name: Python Build
'on':
workflow_dispatch: { }
push:
paths-ignore:
- '.octopus/**'
- 'context/**'
- '.github/agents/**'
schedule:
- cron: '0 0 * * *'
jobs:
validate:
runs-on: ubuntu-latest
steps:
- name: Set up Python
uses: actions/setup-python@v5
with:
# Need 3.11 because of https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean
python-version: 3.11
- uses: actions/checkout@v4
- name: Install tools
run: |
pip install pytest coverage setuptools radon xenon
sudo apt-get install -y pycodestyle mypy
- name: Linting
shell: bash
run: find . -path ./domain/sanitizers/stringlifier -prune -o -path ./venv -prune -o -name "*.py" -print0 | xargs -0 -n1 pycodestyle --ignore=E501,W503,E126
- name: Complexity Enforcement
shell: bash
run: find . -path ./domain/sanitizers/stringlifier -prune -o -path ./tests -prune -o -path ./venv -prune -o -path ./.venv -prune -o -name "*.py" -print0 | xargs -0 -n1 xenon --max-absolute C
test:
name: Test
runs-on: ubuntu-latest
needs: validate
strategy:
matrix:
group: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 ]
steps:
- name: Install terraform
uses: hashicorp/setup-terraform@v3
- name: Set up Python
uses: actions/setup-python@v5
with:
# Need 3.11 because of https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean
python-version: 3.11
- uses: actions/checkout@v4
# The --skipApiVersionCheck might need to be added to the Azurite command line to skip the error:
# The API version 2025-07-05 is not supported by Azurite. Please upgrade Azurite to latest version and retry. If you are using Azurite in Visual Studio, please check you have installed latest Visual Studio patch. Azurite command line parameter "--skipApiVersionCheck" or Visual Studio Code configuration "Skip Api Version Check" can skip this error.
# https://github.com/Azure/Azurite/issues/2562
- name: Start Azurite
run: docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 --restart unless-stopped mcr.microsoft.com/azure-storage/azurite
shell: bash
- uses: robinraju/[email protected]
with:
repository: "OctopusSolutionsEngineering/OctopusTerraformExport"
latest: true
fileName: "octoterra_linux_amd64_azure.zip"
- uses: robinraju/[email protected]
with:
repository: "OctopusSolutionsEngineering/OctopusRecommendationEngine"
latest: true
fileName: "octolint_linux_amd64_azure.zip"
- uses: robinraju/[email protected]
with:
repository: "OctopusSolutionsEngineering/OctoAISpaceBuilder"
latest: true
fileName: "spacebuilder_linux_amd64_azure.zip"
- name: Start Octoterra
run: |
# Start the octoterra microservice
mkdir octoterra
mv octoterra_linux_amd64_azure.zip octoterra
cd octoterra
unzip octoterra_linux_amd64_azure.zip
chmod +x octoterra_linux_amd64_azure
./octoterra_linux_amd64_azure &
env:
# The port used when running the octoterra microservice
FUNCTIONS_CUSTOMHANDLER_PORT: 8081
shell: bash
- name: Start Octolint
run: |
# Start the octolint microservice
mkdir octolint
mv octolint_linux_amd64_azure.zip octolint
cd octolint
unzip octolint_linux_amd64_azure.zip
chmod +x octolint_linux_amd64_azure
./octolint_linux_amd64_azure &
env:
# The port used when running the octolint microservice
FUNCTIONS_CUSTOMHANDLER_PORT: 8082
shell: bash
- name: Start SpaceBuilder
run: |
# Start the spacebuilder microservice
mkdir spacebuilder
mv spacebuilder_linux_amd64_azure.zip spacebuilder
cd spacebuilder
unzip spacebuilder_linux_amd64_azure.zip
chmod +x spacebuilder_linux_amd64_azure
chmod +x binaries/opa_linux_amd64
chmod +x binaries/tofu
./spacebuilder_linux_amd64_azure &
env:
SPACEBUILDER_INSTALLATION_DIRECTORY: "${{ github.workspace }}/spacebuilder"
# The directory containing the policy files
SPACEBUILDER_OPA_POLICY_PATH: "${{ github.workspace }}/spacebuilder/policy"
# The directory containing the Terraform providers
SPACEBUILDER_TERRAFORM_PROVIDERS: "${{ github.workspace }}/spacebuilder/provider"
# The location of the OPA binary
SPACEBUILDER_OPA_PATH: "${{ github.workspace }}/spacebuilder/binaries/opa_linux_amd64"
# The location of the tofu binary
SPACEBUILDER_TOFU_PATH: "${{ github.workspace }}/spacebuilder/binaries/tofu"
# Don't try and make the binaries executable
DISABLE_BINARIES_EXECUTABLE: true
# Force the use of the built in providers
SPACEBUILDER_DISABLE_TERRAFORM_CLI_CONFIG: false
# The port used when running the spacebuilder microservice
FUNCTIONS_CUSTOMHANDLER_PORT: 8083
# This is the connection string to Azurite, the test Aure Storage emulator
AzureWebJobsStorage: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
shell: bash
- name: Install Dependencies
run: pip install -r requirements.txt
shell: bash
# To run tests locally, use these instructions:
# 1. source <venv>/bin/activate
# 2. pip install coverage pytest
# 3. pip install -r requirements.txt
# 4. coverage run --omit="./domain/sanitizers/stringlifier/*" -m pytest --junitxml=results.xml
- name: Test
run: |
pip install pytest pytest-split coverage setuptools
PYTHONPATH=$(pwd) coverage run --omit="./tests/*" --omit="./domain/sanitizers/stringlifier/*" -m pytest --junitxml=results.xml --splits 20 --group ${{ matrix.group }}
# See https://github.com/pytest-dev/pytest/issues/2393
ret=$?
if [ "$ret" = 5 ]; then
echo "No tests collected. Exiting with 0 (instead of 5)."
exit 0
fi
# These executables and container should be running. If there was an error, this will help diagnose it.
ps aux | grep spacebuilder_linux_amd64_azure
ps aux | grep octolint_linux_amd64_azure
ps aux | grep octoterra_linux_amd64_azure
docker container ps
exit "$ret"
env:
# These are the details of Slack
SLACK_CLIENT_ID: ${{ secrets.SLACK_CLIENT_ID }}
SLACK_CLIENT_REDIRECT: ${{ secrets.SLACK_CLIENT_REDIRECT }}
# These are the details for zendesk
ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }}
ZENDESK_TOKEN: ${{ secrets.ZENDESK_TOKEN }}
# These are the connection details to Azure OpenAI
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_ENDPOINT: ${{ secrets.OPENAI_ENDPOINT }}
# This is a base 64 encoded version of the Octopus license
LICENSE: ${{ secrets.OCTOPUS_LICENSE }}
# This is the slack webhook used to send messages
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
# This is the connection string to Azurite, the test Aure Storage emulator
AzureWebJobsStorage: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
# This is the list of admin users
APPLICATION_USERS_ADMIN: "[160104]"
# This is the test user that is used to simulate a login
TEST_GH_USER: "160104"
# This is the GitHub token passed to the chat tests
GH_TEST_TOKEN: ${{ secrets.GH_TEST_TOKEN }}
# This is the encryption salt used when saving encrypted values to Azure Storage
ENCRYPTION_PASSWORD: password
# This is the encryption salt used when saving encrypted values to Azure Storage
ENCRYPTION_SALT: salt
# The location of the octoterra microservice
APPLICATION_OCTOTERRA_URL: http://localhost:8081
# The location of the octolint microservice
APPLICATION_OCTOLINT_URL: http://localhost:8082
# The location of the spacebuilder microservice
APPLICATION_SPACEBUILDER_URL: http://localhost:8083
GITHUB_CLIENT_REDIRECT: https://aiagent.octopus.com/api/oauth_callback
GITHUB_CLIENT_ID: 4ae1ef506301902a41a5
REDIRECTION_HOST: https://redirection.example.org
CODEFRESH_URL: ${{ secrets.CODEFRESH_URL }}
# This is the deployment used to handle prompts
AISERVICES_DEPLOYMENT: gpt-4.1
# This is the deployment used to generate project Terraform configuration
AISERVICES_DEPLOYMENT_PROJECT_GEN: gpt-4.1
# This is the deployment used to handle function calls
AISERVICES_DEPLOYMENT_FUNCTIONS: gpt-4.1
AISERVICES_KEY: ${{ secrets.AISERVICES_KEY }}
AISERVICES_ENDPOINT: ${{ secrets.AISERVICES_ENDPOINT }}
# REDIRECTION_DISABLE: !!str "true"
# REDIRECTION_FORCE: !!str "false"
shell: /usr/bin/bash --noprofile --norc {0}
- name: Upload coverage
uses: actions/upload-artifact@v4
with:
name: coverage${{ matrix.group }}
include-hidden-files: true
path: .coverage
- name: Upload test artifacts
uses: actions/upload-artifact@v4
with:
name: junit-test-summary-${{ matrix.group }}
path: results.xml
retention-days: 1
test-results:
needs: test
runs-on: ubuntu-latest
steps:
- name: Set up Python
uses: actions/setup-python@v5
with:
# Need 3.11 because of https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean
python-version: 3.11
- uses: actions/checkout@v4
- name: Download artifacts
uses: actions/download-artifact@v4
- name: Run coverage
run: |
pip install pytest coverage setuptools
coverage combine coverage*/.coverage*
coverage xml --omit="./domain/sanitizers/stringlifier/*"
coverage html --omit="./domain/sanitizers/stringlifier/*" --fail-under=80
- name: Upload combined coverage
uses: actions/upload-artifact@v4
with:
name: coverage
include-hidden-files: true
path: .coverage
- name: Upload coverage xml
uses: actions/upload-artifact@v4
with:
name: coverage.xml
include-hidden-files: true
path: coverage.xml
- name: Coverage Badge
uses: tj-actions/coverage-badge-py@v2
- name: Verify Changed files
uses: tj-actions/verify-changed-files@v17
id: verify-changed-files
with:
files: coverage.svg
- name: Commit files
if: steps.verify-changed-files.outputs.files_changed == 'true'
run: |
git config --local user.email "github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
git add coverage.svg
git commit -m "Updated coverage.svg"
- name: Push changes
if: steps.verify-changed-files.outputs.files_changed == 'true'
uses: ad-m/github-push-action@master
continue-on-error: true
with:
github_token: ${{ secrets.github_token }}
branch: ${{ github.ref }}
- uses: actions/setup-node@v3
with:
node-version: 16
- name: Install junit-report-merger
run: npm install -g junit-report-merger
- name: Merge reports
run: >
jrm ./junit-test-summary.xml
"junit-test-summary-1/*.xml"
"junit-test-summary-2/*.xml"
"junit-test-summary-3/*.xml"
"junit-test-summary-4/*.xml"
"junit-test-summary-5/*.xml"
"junit-test-summary-6/*.xml"
"junit-test-summary-7/*.xml"
"junit-test-summary-8/*.xml"
"junit-test-summary-9/*.xml"
"junit-test-summary-10/*.xml"
"junit-test-summary-11/*.xml"
"junit-test-summary-12/*.xml"
"junit-test-summary-13/*.xml"
"junit-test-summary-14/*.xml"
"junit-test-summary-15/*.xml"
- if: always()
name: Report
uses: dorny/test-reporter@v1
with:
name: Python Tests
path: junit-test-summary.xml
reporter: java-junit
fail-on-error: 'false'
build:
name: Build
needs: test-results
runs-on: ubuntu-latest
# group: ubuntu-large-runner
# labels: 4core-ubuntu-runner
steps:
- uses: actions/checkout@v4
with:
fetch-depth: '0'
- name: Set up Python
uses: actions/setup-python@v5
with:
# Need 3.11 because of https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean
python-version: 3.11
- name: Install GitVersion
uses: gittools/actions/gitversion/[email protected]
with:
versionSpec: 5.x
- id: determine_version
name: Determine Version
uses: gittools/actions/gitversion/[email protected]
with:
overrideConfig: mode=Mainline
- name: List Dependencies
run: pip install pipdeptree; pipdeptree > dependencies.txt
shell: bash
- name: Collect Dependencies
uses: actions/upload-artifact@v4
with:
name: Dependencies
path: dependencies.txt
- name: List Dependency Updates
run: python3 -m pip list --outdated --format=json | jq -r '.[] | .name+"="+.latest_version' > dependencyUpdates.txt || true
shell: bash
- name: Collect Dependency Updates
uses: actions/upload-artifact@v4
with:
name: Dependencies Updates
path: dependencyUpdates.txt
- name: Generate SBOM
if: ${{ github.event_name != 'schedule' }}
run: |
python -m pip install cyclonedx-bom
cyclonedx-py environment > bom.json
shell: bash
- name: Package
if: ${{ github.event_name != 'schedule' }}
run: |-
zip -R OctopusCopilot.${{ steps.determine_version.outputs.semVer }}.zip \
'*.py' \
'*.pyc' \
'*.html' \
'*.htm' \
'*.css' \
'*.js' \
'*.min' \
'*.map' \
'*.sql' \
'*.png' \
'*.jpg' \
'*.jpeg' \
'*.gif' \
'*.json' \
'*.env' \
'*.txt' \
'*.Procfile' \
'*.bestType' \
'*.conf' \
'*.encodings' \
'*.graphql' \
-x "octoterra/*" \
-x "**/__pycache__/*" \
-x ".vscode/*" \
-x "tests/*" \
shell: bash
- name: Tag Release
if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }}
uses: mathieudutour/[email protected]
with:
custom_tag: ${{ steps.determine_version.outputs.semVer }}
github_token: ${{ secrets.GITHUB_TOKEN }}
- id: create_release
name: Create Release
if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }}
uses: softprops/action-gh-release@v1
with:
tag_name: ${{ steps.determine_version.outputs.semVer }}+run${{ github.run_number }}-attempt${{ github.run_attempt }}
release_name: Release ${{ steps.determine_version.outputs.semVer }} Run ${{ github.run_number }} Attempt ${{ github.run_attempt }}
draft: ${{ github.ref == 'refs/heads/main' && 'false' || 'true' }}
name: Release ${{ steps.determine_version.outputs.semVer }} Run ${{ github.run_number }} Attempt ${{ github.run_attempt }}
- name: Upload Release Asset
if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }}
uses: softprops/action-gh-release@v1
with:
tag_name: ${{ steps.determine_version.outputs.semVer }}+run${{ github.run_number }}-attempt${{ github.run_attempt }}
files: OctopusCopilot.${{ steps.determine_version.outputs.semVer }}.zip
- name: Push packages to Octopus Deploy
if: ${{ github.event_name != 'schedule' }}
uses: OctopusDeploy/push-package-action@v3
env:
OCTOPUS_API_KEY: ${{ secrets.COPILOT_OCTOPUS_API }}
OCTOPUS_URL: ${{ secrets.COPILOT_OCTOPUS_URL }}
OCTOPUS_SPACE: ${{ secrets.COPILOT_OCTOPUS_SPACE }}
with:
packages: OctopusCopilot.${{ steps.determine_version.outputs.semVer }}.zip
overwrite_mode: OverwriteExisting
- name: Push build information to Octopus Deploy
if: ${{ github.event_name != 'schedule' }}
uses: OctopusDeploy/push-build-information-action@v3
env:
OCTOPUS_API_KEY: ${{ secrets.COPILOT_OCTOPUS_API }}
OCTOPUS_URL: ${{ secrets.COPILOT_OCTOPUS_URL }}
OCTOPUS_SPACE: ${{ secrets.COPILOT_OCTOPUS_SPACE }}
with:
packages: OctopusCopilot
version: ${{ steps.determine_version.outputs.semVer }}
overwrite_mode: OverwriteExisting
- name: Create Octopus Release
if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }}
uses: OctopusDeploy/create-release-action@v3
env:
OCTOPUS_API_KEY: ${{ secrets.COPILOT_OCTOPUS_API }}
OCTOPUS_URL: ${{ secrets.COPILOT_OCTOPUS_URL }}
OCTOPUS_SPACE: ${{ secrets.COPILOT_OCTOPUS_SPACE }}
with:
project: Octopus Copilot Function
packages: OctopusCopilot:${{ steps.determine_version.outputs.semVer }}
release_number: ${{ steps.determine_version.outputs.semVer }}+${{ steps.determine_version.outputs.ShortSha }}.${{ github.run_number }}.${{ github.run_attempt }}
git_ref: main
release_notes: |
* GitHub Owner: OctopusSolutionsEngineering
* GitHub Repo: OctopusCopilot
* GitHub Workflow: build.yaml
* GitHub Sha: ${{ github.sha }}
* GitHub Run: ${{ github.run_number }}
* GitHub Attempt: ${{ github.run_attempt }}
* GitHub Run ID: ${{ github.run_id }}
Here are the notes for the packages
#{each package in Octopus.Release.Package}
- #{package.PackageId} #{package.Version}
#{each commit in package.Commits}
- [#{commit.CommitId}](#{commit.LinkUrl}) - #{commit.Comment}
#{/each}
#{each workItem in package.WorkItems}
- [#{workItem.Id}](#{workItem.LinkUrl}) - #{workItem.Description}
#{/each}
#{/each}
permissions:
id-token: write
checks: write
contents: write