Added pipeline along with auto fetch of build ids and publishing the result to Dashboard!

.pipelines/templates/stages/common_tasks/db/push-test-results-to-db.yml

@@ -0,0 +1,46 @@
+# .pipelines/templates/stages/common_tasks/db/push-test-results-to-db.yml
+parameters:
+  - name: marinertridentPipelinesSourceDirectory
+    type: string
+  - name: junitFilesDirectory
+    type: string
+  - name: isStaging
+    type: string           # pass "true"/"false" as string for simplicity
+  - name: buildNumber
+    type: string
+  - name: test_suite_name
+    type: string
+    default: ''
+  - name: architecture
+    type: string
+  - name: test_type_prefix
+    type: string
+    default: ''
+
+steps:
+  - task: Bash@3
+    displayName: 'Install Python dependencies for pushing results to DB'
+    inputs:
+      targetType: 'inline'
+      script: |
+        set -eux
+        sudo tdnf -y --rpmverbosity=debug install python3 python3-pip
+        python3 -m pip install --user -r ${{ parameters.marinertridentPipelinesSourceDirectory }}/scripts/tests/requirements.txt
+
+  - task: AzureCLI@2
+    displayName: "Push test results to DB"
+    condition: succeeded()
+    inputs:
+      azureSubscription: "bmp-azl-dashboard-service-connection"
+      scriptType: bash
+      scriptLocation: inlineScript
+      inlineScript: |
+        set -eux
+        python3 ${{ parameters.marinertridentPipelinesSourceDirectory }}/scripts/tests/push-test-results-to-db.py \
+          --junits_dir "${{ parameters.junitFilesDirectory }}" \
+          --arch "${{ parameters.architecture }}" \
+          --build_number "${{ parameters.buildNumber }}" \
+          --run_pipeline-id "$(Build.BuildId)" \
+          --is_staging "${{ parameters.isStaging }}" \
+          --test_suite "${{ parameters.test_suite_name }}" \
+          --test_type_prefix "${{ parameters.test_type_prefix }}"

.pipelines/templates/trident-pr-e2e-auto-template.yml

@@ -0,0 +1,201 @@
+# .pipelines/templates/trident-pr-e2e-auto-template.yml
+parameters:
+  # cross-org/project for base-image builds
+  - name: baseImageOrg
+    type: string
+  - name: baseImageProject
+    type: string
+  - name: amd64DefinitionId
+    type: number
+  - name: arm64DefinitionId
+    type: number
+  - name: baseImgTag
+    type: string
+    default: "3.0-preview"
+
+  # e2e controls
+  - name: buildType
+    type: string
+    default: preview
+    values: [preview, dev, release]
+  - name: includeAzure
+    type: boolean
+    default: false
+  - name: runBaremetalTests
+    type: boolean
+    default: false
+  - name: numberOfUpdateIterations
+    type: number
+    default: 3
+
+  # DB push controls
+  - name: isStaging
+    type: boolean
+    default: true
+  - name: testSuiteName
+    type: string
+    default: "trident-pr-e2e"
+  - name: testTypePrefix
+    type: string
+    default: "pr"
+  - name: architecture
+    type: string
+    default: "amd64"
+
+# ---------- Stage 1: Resolve base-image run IDs ----------
+- stage: ResolveBaseImages
+  displayName: "Resolve latest base-image run IDs"
+  jobs:
+    - job: ResolveIDs
+      displayName: "Fetch latest succeeded (tag-filtered)"
+      variables:
+        - key: ob_outputDirectory
+          value: $(Build.ArtifactStagingDirectory)/resolve-baseimg
+      steps:
+        - bash: |
+            set -euo pipefail
+
+            ORG="${{ parameters.baseImageOrg }}"
+            PROJ="${{ parameters.baseImageProject }}"
+            AMD=${{ parameters.amd64DefinitionId }}
+            ARM=${{ parameters.arm64DefinitionId }}
+            TAG="${{ parameters.baseImgTag }}"
+            API="https://dev.azure.com/${ORG}/${PROJ}/_apis/build/builds"
+            AUTH="Authorization: Bearer $(System.AccessToken)"
+            ACCEPT="Accept: application/json; api-version=7.1-preview.7"
+
+            fetch_latest_with_tag () {
+              local defId="$1" tag="$2"
+              curl -sS -H "$AUTH" -H "$ACCEPT" \
+                "${API}?definitions=${defId}&resultFilter=succeeded&statusFilter=completed&queryOrder=finishTimeDescending&`echo '$'`top=50" \
+              | python3 - "$tag" << 'PY'
+import sys, json
+tag = sys.argv[1]
+data = json.load(sys.stdin)
+for b in data.get("value", []):
+    if tag in (b.get("tags") or []):
+        print(b["id"]); sys.exit(0)
+# fallback: first succeeded
+if data.get("value"): print(data["value"][0]["id"])
+PY
+            }
+
+            AMD_RUNID="$(fetch_latest_with_tag "$AMD" "$TAG")"
+            ARM_RUNID="$(fetch_latest_with_tag "$ARM" "$TAG")"
+
+            echo "AMD64(${AMD}) runId: ${AMD_RUNID}"
+            echo "ARM64(${ARM}) runId: ${ARM_RUNID}"
+
+            echo "##vso[task.setvariable variable=ResolvedBaseImagePipelineBuildId;isOutput=true]${AMD_RUNID}"
+            echo "##vso[task.setvariable variable=ResolvedBaseImageArm64PipelineBuildId;isOutput=true]${ARM_RUNID}"
+          displayName: "Resolve run IDs (tag: ${{ parameters.baseImgTag }})"
+          env:
+            SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+
+# ---------- Stage 2: Run the standard PR-E2E flow ----------
+- stage: PR_E2E
+  displayName: "PR E2E (auto base-image)"
+  dependsOn: ResolveBaseImages
+  variables:
+    - name: AMD_RUNID
+      value: $[ stageDependencies.ResolveBaseImages.ResolveIDs.outputs['ResolveIDs.ResolvedBaseImagePipelineBuildId'] ]
+    - name: ARM_RUNID
+      value: $[ stageDependencies.ResolveBaseImages.ResolveIDs.outputs['ResolveIDs.ResolvedBaseImageArm64PipelineBuildId'] ]
+  jobs:
+    - job: KickE2E
+      displayName: "Execute e2e-template (pr-e2e)"
+      variables:
+        - key: ob_outputDirectory
+          value: $(Build.ArtifactStagingDirectory)/e2e
+      steps:
+        - template: e2e-template.yml
+          parameters:
+            stageType: pr-e2e
+            blockPublishing: true
+            includeAzure: ${{ parameters.includeAzure }}
+            baseimgBuildType: ${{ parameters.buildType }}
+            baseImagePipelineBuildId: "$(AMD_RUNID)"
+            baseImageArm64PipelineBuildId: "$(ARM_RUNID)"
+            runBaremetalTests: ${{ parameters.runBaremetalTests }}
+            numberOfUpdateIterations: ${{ parameters.numberOfUpdateIterations }}
+
+# ---------- Stage 3: Collect only the expected JUnit XMLs & Push ----------
+- stage: CollectAndPushResults
+  displayName: "Collect JUnit XMLs and push to DB"
+  dependsOn: PR_E2E
+  condition: succeededOrFailed()
+  jobs:
+    - job: CollectAndPush
+      displayName: "Aggregate JUnit & call DB API"
+      variables:
+        - key: ob_outputDirectory
+          value: $(Build.ArtifactStagingDirectory)/junit
+        - name: JUNIT_DIR
+          value: $(Pipeline.Workspace)/junit-xmls
+      steps:
+        - bash: |
+            set -eux
+            mkdir -p "$(JUNIT_DIR)"
+          displayName: "Create JUnit directory"
+
+        # Download ONLY the known JUnit artifacts produced by the 3 test families
+        - download: current
+          artifact: junit_for_trident_functionaltests
+          path: "$(JUNIT_DIR)"
+          displayName: "Grab: functional tests"
+          continueOnError: true
+          condition: succeededOrFailed()
+
+        - download: current
+          artifact: junit_for_trident_clean_install
+          path: "$(JUNIT_DIR)"
+          displayName: "Grab: e2e clean install"
+          continueOnError: true
+          condition: succeededOrFailed()
+
+        - download: current
+          artifact: junit_for_trident_ab_update_A
+          path: "$(JUNIT_DIR)"
+          displayName: "Grab: e2e AB update A"
+          continueOnError: true
+          condition: succeededOrFailed()
+
+        - download: current
+          artifact: junit_for_trident_ab_update_B
+          path: "$(JUNIT_DIR)"
+          displayName: "Grab: e2e AB update B"
+          continueOnError: true
+          condition: succeededOrFailed()
+
+        - download: current
+          artifact: junit_for_trident_ab_update_stage
+          path: "$(JUNIT_DIR)"
+          displayName: "Grab: e2e AB stage+finalize"
+          continueOnError: true
+          condition: succeededOrFailed()
+
+        # Only push if we actually found XMLs
+        - bash: |
+            set -eux
+            shopt -s nullglob
+            files=($(ls -1 "$(JUNIT_DIR)"/*.xml 2>/dev/null || true))
+            if [ "${#files[@]}" -eq 0 ]; then
+              echo "No JUnit XMLs found. Skipping DB push."
+              echo "##vso[task.setvariable variable=HAS_JUNIT;isOutput=true]false"
+            else
+              echo "JUnit XMLs found: ${#files[@]}"
+              echo "##vso[task.setvariable variable=HAS_JUNIT;isOutput=true]true"
+            fi
+          name: DetectJUnit
+          displayName: "Detect any JUnit files"
+
+        - ${{ if eq(variables['DetectJUnit.HAS_JUNIT'], 'true') }}:
+          - template: stages/common_tasks/db/push-test-results-to-db.yml
+            parameters:
+              marinertridentPipelinesSourceDirectory: "$(Build.SourcesDirectory)"
+              junitFilesDirectory: "$(JUNIT_DIR)"
+              isStaging: '${{ parameters.isStaging }}'
+              buildNumber: "$(Build.BuildNumber)"
+              test_suite_name: "${{ parameters.testSuiteName }}"
+              architecture: "${{ parameters.architecture }}"
+              test_type_prefix: "${{ parameters.testTypePrefix }}"

.pipelines/trident-pr-e2e-auto.yml

@@ -0,0 +1,51 @@
+# .pipelines/trident-pr-e2e-auto.yml
+trigger: none
+pr: none
+
+resources:
+  repositories:
+    - repository: argus-toolkit
+      type: git
+      name: argus-toolkit
+      ref: refs/heads/main
+    - repository: platform-tests
+      type: git
+      name: platform-tests
+      ref: refs/heads/main
+    - repository: test-images
+      type: git
+      name: test-images
+      ref: refs/heads/main
+    - repository: platform-pipelines
+      type: git
+      name: platform-pipelines
+      ref: refs/heads/main
+    - repository: platform-telemetry
+      type: git
+      name: platform-telemetry
+      ref: refs/heads/main
+
+extends:
+  template: templates/MockOB.yml
+  parameters:
+    stages:
+      - template: templates/trident-pr-e2e-auto-template.yml
+        parameters:
+          # base-image builds live in mariner-org/mariner
+          baseImageOrg: "mariner-org"
+          baseImageProject: "mariner"
+          amd64DefinitionId: 2116
+          arm64DefinitionId: 2117
+          baseImgTag: "3.0-preview"
+
+          # E2E controls (keeps existing trident-pr-e2e behavior)
+          buildType: preview            # preview | dev | release
+          includeAzure: false
+          runBaremetalTests: false
+          numberOfUpdateIterations: 3
+
+          # DB push controls
+          isStaging: true
+          testSuiteName: "trident-pr-e2e"
+          testTypePrefix: "pr"
+          architecture: "amd64"         # change if you want to label results as arm64

scripts/tests/push-test-results-to-db.py

@@ -0,0 +1,94 @@
+import time
+import requests
+import os
+import argparse
+import xml.etree.ElementTree as ET
+from azure.identity import DefaultAzureCredential
+
+APP_REG_ID = "13a078bd-0bae-4c0a-b691-df710bc234c1"
+TOKEN_SCOPE = f"api://{APP_REG_ID}/.default"
+
+def get_token():
+    credential = DefaultAzureCredential()
+    token = None
+    for i in range(5):
+        try:
+            token = credential.get_token(TOKEN_SCOPE)
+            break
+        except Exception as e:
+            print(f"Token retrieval failed: {e}")
+            print("Retrying...")
+            time.sleep(2**i)
+    if token is None:
+        raise Exception("Failed to retrieve token after multiple attempts")
+    return token.token
+
+def extract_test_type_name(junit_file_path, test_type_prefix=""):
+    try:
+        tree = ET.parse(junit_file_path)
+        root = tree.getroot()
+        testsuite = root.find('.//testsuite')
+        if testsuite is not None and 'name' in testsuite.attrib:
+            base_name = testsuite.attrib['name']
+        else:
+            base_name = os.path.splitext(os.path.basename(junit_file_path))[0]
+    except Exception as e:
+        print(f"Warning: Could not parse test suite name from {junit_file_path}: {e}")
+        base_name = os.path.splitext(os.path.basename(junit_file_path))[0]
+    return f"{test_type_prefix}-{base_name}" if test_type_prefix else base_name
+
+def push_test_result_junit(test_suite, test_type, arch, build_number, run_pipeline_id, junit_file_path):
+    url = "https://azlinux-api-management.azure-api.net/di/staging/image_release/push_test_result_junit"
+    headers = {"Authorization": f"Bearer {get_token()}"}
+    data = {
+        "test_suite": test_suite,
+        "arch": arch,
+        "build_number": build_number,
+        "run_pipeline_id": run_pipeline_id,
+        "test_type_override": test_type,
+    }
+    with open(junit_file_path, "rb") as junit_file:
+        files = {"junit_xml": junit_file}
+        resp = requests.post(url, headers=headers, data=data, files=files)
+        print(resp.text)
+        resp.raise_for_status()
+        return resp.json()
+
+def parse_args():
+    p = argparse.ArgumentParser(description="Push test results to database using API")
+    p.add_argument("--arch", required=True)
+    p.add_argument("--build_number", required=True)
+    p.add_argument("--run_pipeline-id", required=True)
+    p.add_argument("--junits_dir", required=True)
+    p.add_argument("--is_staging", required=True)
+    p.add_argument("--test_suite", required=True)
+    p.add_argument("--test_type_prefix", default="")
+    return p.parse_args()
+
+def main():
+    args = parse_args()
+    print("Architecture:", args.arch)
+    print("Build number:", args.build_number)
+    print("Run pipeline ID:", args.run_pipeline_id)
+    print("Junits directory:", args.junits_dir)
+    print("Test suite:", args.test_suite)
+    print("Test type prefix:", args.test_type_prefix)
+    is_staging = args.is_staging.lower() == 'true'
+    print("Is staging:", is_staging)
+
+    for name in os.listdir(args.junits_dir):
+        if name.endswith(".xml"):
+            path = os.path.join(args.junits_dir, name)
+            print("Processing:", path)
+            test_type_name = extract_test_type_name(path, args.test_type_prefix)
+            try:
+                result = push_test_result_junit(
+                    args.test_suite, test_type_name, args.arch,
+                    args.build_number, args.run_pipeline_id, path
+                )
+                print(f"Successfully pushed {name}: {result}")
+            except Exception as e:
+                print(f"Failed to push {name}: {e}")
+
+if __name__ == "__main__":
+    main()