Improved tokenizer

.github/workflows/quality-check.yml

@@ -12,18 +12,21 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install minimal nightly (for fmt)
+        uses: dtolnay/rust-toolchain@nightly
+        with:
+          components: rustfmt
+
       - name: Set up Rust
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@1.90.0
         with:
-          toolchain: stable
-          override: true
           components: rustfmt, clippy
 
       - name: Cargo check
         run: cargo clippy --all-targets --all-features -- -D warnings
 
       - name: Format check
-        run: cargo fmt --all -- --check
+        run: cargo +nightly fmt --all -- --check
 
   test:
     needs:
@@ -37,10 +40,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+        uses: dtolnay/[email protected]
 
       - name: Install nextest
         uses: taiki-e/install-action@nextest
@@ -60,17 +60,13 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Build script
-        uses: docker://ghcr.io/amber-lang/amber:alpine-0.4.0-alpha
-        with:
-          args: |
-            build ./run_coverage.ab ./run_coverage.sh
-
       - name: Test coverage
-        run: ./run_coverage.sh xml
+        run: cargo tarpaulin --verbose --all-features --workspace --timeout 120 --out xml
 
       - name: Upload to codecov.io
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v5
         with:
           token: ${{secrets.CODECOV_TOKEN}}
           fail_ci_if_error: true
+          files: ./coverage_output/cobertura.xml
+          verbose: true

.github/workflows/release.yml

@@ -113,6 +113,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
+      - name: Set up Rust
+        uses: dtolnay/[email protected]
       - uses: swatinem/rust-cache@v2
         with:
           key: ${{ join(matrix.targets, '-') }}
@@ -167,6 +169,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
+      - name: Set up Rust
+        uses: dtolnay/[email protected]
       - name: Install cargo-dist
         shell: bash
         run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.20.0/cargo-dist-installer.sh | sh"
@@ -341,6 +345,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
+      - name: Set up Rust
+        uses: dtolnay/[email protected]
       - name: Install cargo-dist
         run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.20.0/cargo-dist-installer.sh | sh"
       # Fetch artifacts from scratch-storage

.gitignore

@@ -39,3 +39,8 @@ coverage_output
 
 # MacOS system files
 .DS_Store
+
+# Openspec
+AGENTS.md
+openspec/
+.github/prompts

.pre-commit-config.yaml

@@ -4,17 +4,16 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
-      - id: check-added-large-files
       - id: check-yaml
       - id: check-toml
 
   - repo: local
     hooks:
       - id: find-fixme-phrase
         name: Find FIXME phrase
-        entry: bash -c 'git diff --cached --name-only --diff-filter=ACM | grep -E "\.(txt|js|ts|py|rs|go|java|c|cpp|sh)$" | xargs -r grep -l "FIXME" && exit 1 || exit 0'
+        entry: bash -c 'git diff --cached --name-only --diff-filter=ACM | grep -E "\.(txt|js|ts|py|rs|go|java|c|cpp|sh|ab)$" | xargs -r grep -l "FIXME" && exit 1 || exit 0'
         language: system
-        files: '\.(txt|js|ts|py|rs|go|java|c|cpp|sh)$'
+        files: '\.(txt|js|ts|py|rs|go|java|c|cpp|sh|ab)$'
         description: Checks for the "FIXME" phrase in staged text files.
 
   - repo: local
@@ -30,7 +29,7 @@ repos:
     hooks:
       - id: cargo-fmt
         name: cargo fmt --check
-        entry: cargo fmt -- --check
+        entry: cargo +nightly fmt --check
         language: system
         types: [rust]
         pass_filenames: false