From 8bb8fc8b27e75100e50cc98eeab7369c00678b67 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 18 Sep 2025 13:31:05 -0400 Subject: [PATCH 01/10] ENH: allow to test against bids-spec PR schema. --- .github/workflows/validate_datasets.yml | 42 +++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_datasets.yml b/.github/workflows/validate_datasets.yml index d92b67f0c..4653a1af7 100644 --- a/.github/workflows/validate_datasets.yml +++ b/.github/workflows/validate_datasets.yml @@ -21,16 +21,47 @@ defaults: shell: bash jobs: + # Prepare the matrix dynamically based on whether a bids-specification PR is referenced + prepare-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + bids_pr: ${{ steps.find-pr.outputs.pr_number }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 # Get full history for commit message scanning + + - name: Check for bids-specification PR reference + id: find-pr + run: | + # Also check PR description if this is a PR + if [ "${{ github.event_name }}" == "pull_request" ]; then + PR_NUM=$(echo "${{ github.event.pull_request.body }}" | grep -oP 'bids-specification-pr:s*(https://github\.com/bids-standard/bids-specification/pulls/)*K[0-9]+/*' || true) + [ -n "$PR_BODY_NUM" ] && echo "pr_number=$PR_NUM" >> $GITHUB_OUTPUT + fi + + - name: Set matrix + id: set-matrix + run: | + EXTRA_ITEM='' + if [ -n "${{ steps.find-pr.outputs.pr_number }}" ]; then + EXTRA_ITEM=', "bids-pr"' + fi + echo 'matrix=["stable", "main", "dev", "legacy"${EXTRA_ITEM}' >> $GITHUB_OUTPUT + build: + needs: prepare-matrix strategy: fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - bids-validator: [stable, main, dev, legacy] + bids-validator: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} runs-on: ${{ matrix.platform }} env: + BIDS_PR: ${{ needs.prepare-matrix.outputs.bids_pr }} TZ: Europe/Berlin FORCE_COLOR: 1 @@ -66,7 +97,7 @@ jobs: deno install -Agf -n bids-validator jsr:@bids/validator - name: Install BIDS validator (main) - if: matrix.bids-validator == 'main' + if: matrix.bids-validator == 'main' or matrix.bids-validator == 'bids-pr' run: | # If unmerged validator PRs are needed for testing, you can use # https://github.com//bids-validator/raw//bids-validator/src/bids-validator.ts @@ -131,6 +162,13 @@ jobs: # release of https://jsr.io/@bids/schema run: echo BIDS_SCHEMA=https://bids-specification.readthedocs.io/en/latest/schema.json >> $GITHUB_ENV + - name: Set BIDS_SCHEMA variable for bids-pr version + if: matrix.bids-validator == 'bids-pr' + # Use the readthedocs PR preview build for the schema + run: | + echo "Using schema from bids-specification PR #${{ env.BIDS_PR }}" + echo BIDS_SCHEMA=https://bids-specification--${{ env.BIDS_PR }}.org.readthedocs.build/en/${{ env.BIDS_PR }}/schema.json >> $GITHUB_ENV + - name: Validate all BIDS datasets using bids-validator run: | cat ./run_tests.sh From 8fa6f5d6c5ae3175a0aa381c203807d369c7bdc5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 18 Sep 2025 13:39:00 -0400 Subject: [PATCH 02/10] Add rudimentary pull_requests_template.md with syntax to list bids-specification-pr --- .github/pull_request_template.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..31758c080 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,12 @@ + + +- Needs testing against a bids-specification-pr: NUMBER-or-URL +- TODOs: + - [ ] ... + + + +See the [CONTRIBUTING](https://github.com/bids-standard/bids-examples/blob/master/CONTRIBUTING.md) guide. Specifically: + +- Please keep the title of your Pull Request (PR) short but informative - it will appear in the changelog. + From 4d07be29f3015d2637574cc5c57a905ccc269ca3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 18 Sep 2025 13:43:27 -0400 Subject: [PATCH 03/10] CI: trigger workflow on PR description edits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow validate_datasets workflow to re-run when PR description is edited, enabling dynamic detection of bids-specification-pr references added after PR creation. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/validate_datasets.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_datasets.yml b/.github/workflows/validate_datasets.yml index 4653a1af7..26cd465fc 100644 --- a/.github/workflows/validate_datasets.yml +++ b/.github/workflows/validate_datasets.yml @@ -4,6 +4,7 @@ on: push: branches: ['**'] pull_request: + types: [opened, synchronize, reopened, edited] branches: ['**'] create: branches: [master] From e0d76c712db2b0bcea99d902cb3ec366ca678a6f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 18 Sep 2025 14:02:12 -0400 Subject: [PATCH 04/10] CI: fix workflow syntax errors for bids-pr matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix missing closing bracket in matrix JSON output - Fix variable name mismatch (PR_BODY_NUM vs PR_NUM) - Fix regex pattern for PR detection (escape sequences) - Fix bash conditional syntax (use || instead of 'or') 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/validate_datasets.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate_datasets.yml b/.github/workflows/validate_datasets.yml index 26cd465fc..6048ec7ef 100644 --- a/.github/workflows/validate_datasets.yml +++ b/.github/workflows/validate_datasets.yml @@ -38,8 +38,8 @@ jobs: run: | # Also check PR description if this is a PR if [ "${{ github.event_name }}" == "pull_request" ]; then - PR_NUM=$(echo "${{ github.event.pull_request.body }}" | grep -oP 'bids-specification-pr:s*(https://github\.com/bids-standard/bids-specification/pulls/)*K[0-9]+/*' || true) - [ -n "$PR_BODY_NUM" ] && echo "pr_number=$PR_NUM" >> $GITHUB_OUTPUT + PR_NUM=$(echo "${{ github.event.pull_request.body }}" | grep -oP 'bids-specification-pr:\s*(https://github\.com/bids-standard/bids-specification/pulls?/)*\K[0-9]+' | head -1 || true) + [ -n "$PR_NUM" ] && echo "pr_number=$PR_NUM" >> $GITHUB_OUTPUT fi - name: Set matrix @@ -49,7 +49,7 @@ jobs: if [ -n "${{ steps.find-pr.outputs.pr_number }}" ]; then EXTRA_ITEM=', "bids-pr"' fi - echo 'matrix=["stable", "main", "dev", "legacy"${EXTRA_ITEM}' >> $GITHUB_OUTPUT + echo "matrix=[\"stable\", \"main\", \"dev\", \"legacy\"${EXTRA_ITEM}]" >> $GITHUB_OUTPUT build: needs: prepare-matrix @@ -98,7 +98,7 @@ jobs: deno install -Agf -n bids-validator jsr:@bids/validator - name: Install BIDS validator (main) - if: matrix.bids-validator == 'main' or matrix.bids-validator == 'bids-pr' + if: matrix.bids-validator == 'main' || matrix.bids-validator == 'bids-pr' run: | # If unmerged validator PRs are needed for testing, you can use # https://github.com//bids-validator/raw//bids-validator/src/bids-validator.ts From 08b9d61092e6c40a2c98ea2e9e9ac14c6a8d6b12 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 18 Sep 2025 14:06:38 -0400 Subject: [PATCH 05/10] CI: fix PR body parsing to handle special characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use heredoc to safely handle PR body content that may contain: - Quotes (single and double) - Newlines and multiline text - Special bash characters 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/validate_datasets.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_datasets.yml b/.github/workflows/validate_datasets.yml index 6048ec7ef..5e7af2387 100644 --- a/.github/workflows/validate_datasets.yml +++ b/.github/workflows/validate_datasets.yml @@ -36,9 +36,12 @@ jobs: - name: Check for bids-specification PR reference id: find-pr run: | - # Also check PR description if this is a PR + # Check PR description if this is a PR if [ "${{ github.event_name }}" == "pull_request" ]; then - PR_NUM=$(echo "${{ github.event.pull_request.body }}" | grep -oP 'bids-specification-pr:\s*(https://github\.com/bids-standard/bids-specification/pulls?/)*\K[0-9]+' | head -1 || true) + cat << 'EOF' > /tmp/pr_body.txt + ${{ github.event.pull_request.body }} + EOF + PR_NUM=$(grep -oP 'bids-specification-pr:\s*(https://github\.com/bids-standard/bids-specification/pulls?/)*\K[0-9]+' /tmp/pr_body.txt | head -1 || true) [ -n "$PR_NUM" ] && echo "pr_number=$PR_NUM" >> $GITHUB_OUTPUT fi From 539efa14cda40a0080cb8eb6982d1a56860a87a4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 18 Sep 2025 14:07:44 -0400 Subject: [PATCH 06/10] noop if no match --- .github/workflows/validate_datasets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_datasets.yml b/.github/workflows/validate_datasets.yml index 5e7af2387..daa4738e3 100644 --- a/.github/workflows/validate_datasets.yml +++ b/.github/workflows/validate_datasets.yml @@ -42,7 +42,7 @@ jobs: ${{ github.event.pull_request.body }} EOF PR_NUM=$(grep -oP 'bids-specification-pr:\s*(https://github\.com/bids-standard/bids-specification/pulls?/)*\K[0-9]+' /tmp/pr_body.txt | head -1 || true) - [ -n "$PR_NUM" ] && echo "pr_number=$PR_NUM" >> $GITHUB_OUTPUT + [ -n "$PR_NUM" ] && echo "pr_number=$PR_NUM" >> $GITHUB_OUTPUT || : fi - name: Set matrix From 077717244a3658cd53348963ab2995e03dd9d2f7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 15 Dec 2025 13:07:00 -0500 Subject: [PATCH 07/10] default to NONE for PRs and add validator PR Co-authored-by: Chris Markiewicz --- .github/pull_request_template.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 31758c080..d6d514ec6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,7 @@ - + -- Needs testing against a bids-specification-pr: NUMBER-or-URL +- BIDS Specification PR: NONE +- BIDS Validator PR: NONE - TODOs: - [ ] ... From a76de45a8fe017d46ee0fc16467f24d4bbd519b0 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 15 Dec 2025 13:14:44 -0500 Subject: [PATCH 08/10] Adjust for the rename of the tag in prior PR + rename run to bids-prs --- .github/workflows/validate_datasets.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate_datasets.yml b/.github/workflows/validate_datasets.yml index daa4738e3..a175823c0 100644 --- a/.github/workflows/validate_datasets.yml +++ b/.github/workflows/validate_datasets.yml @@ -41,16 +41,16 @@ jobs: cat << 'EOF' > /tmp/pr_body.txt ${{ github.event.pull_request.body }} EOF - PR_NUM=$(grep -oP 'bids-specification-pr:\s*(https://github\.com/bids-standard/bids-specification/pulls?/)*\K[0-9]+' /tmp/pr_body.txt | head -1 || true) - [ -n "$PR_NUM" ] && echo "pr_number=$PR_NUM" >> $GITHUB_OUTPUT || : + SPEC_PR_NUM=$(grep -oP 'BIDS Specification PR:\s*(https://github\.com/bids-standard/bids-specification/pulls?/|bids-standard/bids-specification#)*\K[0-9]+' /tmp/pr_body.txt | head -1 || true) + [ -n "$SPEC_PR_NUM" ] && echo "spec_pr_number=$SPEC_PR_NUM" >> $GITHUB_OUTPUT || : fi - name: Set matrix id: set-matrix run: | EXTRA_ITEM='' - if [ -n "${{ steps.find-pr.outputs.pr_number }}" ]; then - EXTRA_ITEM=', "bids-pr"' + if [ -n "${{ steps.find-pr.outputs.spec_pr_number }}" ]; then + EXTRA_ITEM=', "bids-prs"' fi echo "matrix=[\"stable\", \"main\", \"dev\", \"legacy\"${EXTRA_ITEM}]" >> $GITHUB_OUTPUT @@ -101,7 +101,7 @@ jobs: deno install -Agf -n bids-validator jsr:@bids/validator - name: Install BIDS validator (main) - if: matrix.bids-validator == 'main' || matrix.bids-validator == 'bids-pr' + if: matrix.bids-validator == 'main' || matrix.bids-validator == 'bids-prs' run: | # If unmerged validator PRs are needed for testing, you can use # https://github.com//bids-validator/raw//bids-validator/src/bids-validator.ts @@ -166,8 +166,8 @@ jobs: # release of https://jsr.io/@bids/schema run: echo BIDS_SCHEMA=https://bids-specification.readthedocs.io/en/latest/schema.json >> $GITHUB_ENV - - name: Set BIDS_SCHEMA variable for bids-pr version - if: matrix.bids-validator == 'bids-pr' + - name: Set BIDS_SCHEMA variable for bids-prs version + if: matrix.bids-validator == 'bids-prs' # Use the readthedocs PR preview build for the schema run: | echo "Using schema from bids-specification PR #${{ env.BIDS_PR }}" From de7e98522ed0100c1581958a0701f33d15573065 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 15 Dec 2025 13:50:09 -0500 Subject: [PATCH 09/10] Install if needed custom bids-validator and also adjust installation to use SPEC_PR instead of BIDS_PR var Also renamed into `dev-prs` to ensure reflecting the fact that we are installing a development version of the validator even if PR for it not given. --- .github/workflows/validate_datasets.yml | 42 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/.github/workflows/validate_datasets.yml b/.github/workflows/validate_datasets.yml index a175823c0..5d650f209 100644 --- a/.github/workflows/validate_datasets.yml +++ b/.github/workflows/validate_datasets.yml @@ -27,7 +27,8 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} - bids_pr: ${{ steps.find-pr.outputs.pr_number }} + spec_pr: ${{ steps.find-pr.outputs.spec_pr }} + val_pr: ${{ steps.find-pr.outputs.val_pr }} steps: - uses: actions/checkout@v5 with: @@ -41,16 +42,18 @@ jobs: cat << 'EOF' > /tmp/pr_body.txt ${{ github.event.pull_request.body }} EOF - SPEC_PR_NUM=$(grep -oP 'BIDS Specification PR:\s*(https://github\.com/bids-standard/bids-specification/pulls?/|bids-standard/bids-specification#)*\K[0-9]+' /tmp/pr_body.txt | head -1 || true) - [ -n "$SPEC_PR_NUM" ] && echo "spec_pr_number=$SPEC_PR_NUM" >> $GITHUB_OUTPUT || : + SPEC_PR=$(grep -oP 'BIDS Specification PR:\s*(https://github\.com/bids-standard/bids-specification/pulls?/|bids-standard/bids-specification#)*\K[0-9]+' /tmp/pr_body.txt | head -1 || true) + [ -n "$SPEC_PR" ] && echo "spec_pr=$SPEC_PR" >> $GITHUB_OUTPUT || : + VAL_PR=$(grep -oP 'BIDS Validator PR:\s*(https://github\.com/bids-standard/bids-validator/pulls?/|bids-standard/bids-validator#)*\K[0-9]+' /tmp/pr_body.txt | head -1 || true) + [ -n "$VAL_PR" ] && echo "val_pr=$VAL_PR" >> $GITHUB_OUTPUT || : fi - name: Set matrix id: set-matrix run: | EXTRA_ITEM='' - if [ -n "${{ steps.find-pr.outputs.spec_pr_number }}" ]; then - EXTRA_ITEM=', "bids-prs"' + if [ -n "${{ steps.find-pr.outputs.spec_pr }}${{ steps.find-pr.outputs.val_pr }}" ]; then + EXTRA_ITEM=', "dev-prs"' fi echo "matrix=[\"stable\", \"main\", \"dev\", \"legacy\"${EXTRA_ITEM}]" >> $GITHUB_OUTPUT @@ -65,7 +68,8 @@ jobs: runs-on: ${{ matrix.platform }} env: - BIDS_PR: ${{ needs.prepare-matrix.outputs.bids_pr }} + SPEC_PR: ${{ needs.prepare-matrix.outputs.spec_pr }} + VAL_PR: ${{ needs.prepare-matrix.outputs.val_pr }} TZ: Europe/Berlin FORCE_COLOR: 1 @@ -101,14 +105,14 @@ jobs: deno install -Agf -n bids-validator jsr:@bids/validator - name: Install BIDS validator (main) - if: matrix.bids-validator == 'main' || matrix.bids-validator == 'bids-prs' + if: matrix.bids-validator == 'main' run: | # If unmerged validator PRs are needed for testing, you can use # https://github.com//bids-validator/raw//bids-validator/src/bids-validator.ts deno install -Agf https://github.com/bids-standard/bids-validator/raw/deno-build/bids-validator.js - name: Install BIDS validator (dev) - if: matrix.bids-validator == 'dev' + if: ( matrix.bids-validator == 'dev' ) || ( matrix.bids-validator == 'dev-prs' && env.VAL_PR == '' ) run: | git clone -b dev https://github.com/bids-standard/bids-validator/ ../bids-validator cd ../bids-validator @@ -119,6 +123,20 @@ jobs: run: | npm install -g bids-validator + - name: Install BIDS validator (dev-prs with VAL_PR) + if: matrix.bids-validator == 'dev-prs' && env.VAL_PR != '' + shell: bash + run: | + # Fetch PR info (fork URL and branch) - avoid mapfile for macOS compatibility + pr_json=$(curl -s https://api.github.com/repos/bids-standard/bids-validator/pulls/${{ env.VAL_PR }}) + fork_url=$(echo "$pr_json" | jq -r '.head.repo.html_url') + branch=$(echo "$pr_json" | jq -r '.head.ref') + echo "I: Received fork_url=$fork_url branch=$branch" + # Perform the same installation as in "dev" + git clone -b "$branch" "$fork_url" ../bids-validator + cd ../bids-validator + deno compile -A -o $HOME/.deno/bin/bids-validator src/bids-validator.ts + - name: Display versions and environment information run: | echo $TZ @@ -166,12 +184,12 @@ jobs: # release of https://jsr.io/@bids/schema run: echo BIDS_SCHEMA=https://bids-specification.readthedocs.io/en/latest/schema.json >> $GITHUB_ENV - - name: Set BIDS_SCHEMA variable for bids-prs version - if: matrix.bids-validator == 'bids-prs' + - name: Set BIDS_SCHEMA variable for dev-prs version + if: matrix.bids-validator == 'dev-prs' && env.SPEC_PR != '' # Use the readthedocs PR preview build for the schema run: | - echo "Using schema from bids-specification PR #${{ env.BIDS_PR }}" - echo BIDS_SCHEMA=https://bids-specification--${{ env.BIDS_PR }}.org.readthedocs.build/en/${{ env.BIDS_PR }}/schema.json >> $GITHUB_ENV + echo "Using schema from bids-specification PR #${{ env.SPEC_PR }}" + echo BIDS_SCHEMA=https://bids-specification--${{ env.SPEC_PR }}.org.readthedocs.build/en/${{ env.SPEC_PR }}/schema.json >> $GITHUB_ENV - name: Validate all BIDS datasets using bids-validator run: | From 03622a868f492c9ab45e7b3ebe52cf0a7b064f82 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 16 Dec 2025 16:23:42 -0500 Subject: [PATCH 10/10] Use gh cli for API calls to avoid rate limiting on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch from curl to gh api for fetching PR info. The gh cli handles authentication automatically and avoids GitHub API rate limits that were causing failures on macOS runners (60 req/hr unauthenticated vs 1000 req/hr authenticated). Also add error checking for failed API responses. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/validate_datasets.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate_datasets.yml b/.github/workflows/validate_datasets.yml index 5d650f209..12f45900a 100644 --- a/.github/workflows/validate_datasets.yml +++ b/.github/workflows/validate_datasets.yml @@ -126,12 +126,17 @@ jobs: - name: Install BIDS validator (dev-prs with VAL_PR) if: matrix.bids-validator == 'dev-prs' && env.VAL_PR != '' shell: bash + env: + GH_TOKEN: ${{ github.token }} run: | - # Fetch PR info (fork URL and branch) - avoid mapfile for macOS compatibility - pr_json=$(curl -s https://api.github.com/repos/bids-standard/bids-validator/pulls/${{ env.VAL_PR }}) - fork_url=$(echo "$pr_json" | jq -r '.head.repo.html_url') - branch=$(echo "$pr_json" | jq -r '.head.ref') + # Fetch PR info using gh cli (handles auth automatically) + fork_url=$(gh api repos/bids-standard/bids-validator/pulls/${{ env.VAL_PR }} --jq '.head.repo.html_url') + branch=$(gh api repos/bids-standard/bids-validator/pulls/${{ env.VAL_PR }} --jq '.head.ref') echo "I: Received fork_url=$fork_url branch=$branch" + if [ "$fork_url" = "null" ] || [ -z "$fork_url" ]; then + echo "E: Failed to get PR info for VAL_PR=${{ env.VAL_PR }}" + exit 1 + fi # Perform the same installation as in "dev" git clone -b "$branch" "$fork_url" ../bids-validator cd ../bids-validator