Improved tokenizer

.github/workflows/quality-check.yml

@@ -12,6 +12,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install minimal nightly (for fmt)
+        uses: dtolnay/rust-toolchain@nightly
+        with:
+          components: rustfmt
+
       - name: Set up Rust
         uses: dtolnay/[email protected]
         with:
@@ -21,7 +26,7 @@ jobs:
         run: cargo clippy --all-targets --all-features -- -D warnings
 
       - name: Format check
-        run: cargo fmt --all -- --check
+        run: cargo +nightly fmt --all -- --check
 
   test:
     needs:
@@ -55,17 +60,13 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Build script
-        uses: docker://ghcr.io/amber-lang/amber:alpine-0.4.0-alpha
-        with:
-          args: |
-            build ./run_coverage.ab ./run_coverage.sh
-
       - name: Test coverage
-        run: ./run_coverage.sh xml
+        run: cargo tarpaulin --verbose --all-features --workspace --timeout 120 --out xml
 
       - name: Upload to codecov.io
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v5
         with:
           token: ${{secrets.CODECOV_TOKEN}}
           fail_ci_if_error: true
+          files: ./coverage_output/cobertura.xml
+          verbose: true

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -11,7 +11,7 @@ repos:
     hooks:
       - id: find-fixme-phrase
         name: Find FIXME phrase
-        entry: bash -c 'git diff --cached --name-only --diff-filter=ACM | grep -E "\.(txt|js|ts|py|rs|go|java|c|cpp|sh|ab)$" | xargs -r grep -l "FIXME" && exit 1 || exit 0'
+        entry: bash -c 'git diff --cached --name-only --diff-filter=ACM | grep -E "\.(txt|js|ts|py|rs|go|java|c|cpp|sh|ab)$" | xargs -r grep -l "FIXME" | grep -E ".+" && exit 1 || exit 0'
         language: system
         files: '\.(txt|js|ts|py|rs|go|java|c|cpp|sh|ab)$'
         description: Checks for the "FIXME" phrase in staged text files.
@@ -29,7 +29,7 @@ repos:
     hooks:
       - id: cargo-fmt
         name: cargo fmt --check
-        entry: cargo fmt -- --check
+        entry: cargo +nightly fmt --check
         language: system
         types: [rust]
         pass_filenames: false

Cargo.toml

@@ -16,6 +16,7 @@ chumsky = { git = "https://github.com/KrosFire/chumsky", rev = "406ea80", featur
   "label",
 ] }
 heraclitus-compiler = "1.8.2"
+logos = "0.14"
 serde_json = "1.0.128"
 rangemap = "1.5.1"
 indexmap = "2.6.0"
@@ -47,7 +48,13 @@ ci = "github"
 # The installers to generate for each app
 installers = []
 # Target platforms to build apps for (Rust target-triple syntax)
-targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "x86_64-pc-windows-msvc"]
+targets = [
+  "aarch64-apple-darwin",
+  "x86_64-apple-darwin",
+  "x86_64-unknown-linux-gnu",
+  "x86_64-unknown-linux-musl",
+  "x86_64-pc-windows-msvc",
+]
 # Publish jobs to run in CI
 pr-run-mode = "plan"
 # Skip checking whether the specified configuration files are up to date

codecov.yml

@@ -11,6 +11,10 @@ flags:
     paths:
       - src/grammar/alpha040/
       - src/analysis/alpha040/
+  0.5.0-alpha:
+    paths:
+      - src/grammar/alpha050/
+      - src/analysis/alpha050/
 
 coverage:
   status:
@@ -34,9 +38,12 @@ coverage:
       0.4.0-alpha:
         flags:
           - 0.4.0-alpha
+      0.5.0-alpha:
+        flags:
+          - 0.5.0-alpha
     patch:
       default:
-        target: 100%
+        target: 80%
 
 comment:
   layout: " diff, flags, files"

rustfmt.toml

@@ -0,0 +1,2 @@
+imports_layout="Vertical"
+imports_granularity="Module"

src/analysis/alpha034/exp.rs

@@ -2,20 +2,34 @@ use std::vec;
 
 use chumsky::span::SimpleSpan;
 
-use crate::{
-    analysis::{
-        get_symbol_definition_info, insert_symbol_reference,
-        types::{make_union_type, matches_type, DataType, GenericsMap},
-        Context, FunctionArgument, FunctionSymbol, SymbolInfo, SymbolLocation, SymbolType,
-        VariableSymbol,
-    },
-    files::{FileVersion, Files},
-    grammar::{
-        alpha034::{Expression, InterpolatedCommand, InterpolatedText},
-        Spanned,
-    },
-    paths::FileId,
+use crate::analysis::types::{
+    make_union_type,
+    matches_type,
+    DataType,
+    GenericsMap,
 };
+use crate::analysis::{
+    get_symbol_definition_info,
+    insert_symbol_reference,
+    Context,
+    FunctionArgument,
+    FunctionSymbol,
+    SymbolInfo,
+    SymbolLocation,
+    SymbolType,
+    VariableSymbol,
+};
+use crate::files::{
+    FileVersion,
+    Files,
+};
+use crate::grammar::alpha034::{
+    Expression,
+    InterpolatedCommand,
+    InterpolatedText,
+};
+use crate::grammar::Spanned;
+use crate::paths::FileId;
 
 use super::stmnts::analyze_failure_handler;
 

src/analysis/alpha034/global.rs

@@ -1,18 +1,33 @@
-use crate::{
-    analysis::{
-        self, import_symbol, insert_symbol_definition, map_import_path,
-        types::{make_union_type, matches_type, DataType},
-        Context, FunctionContext, FunctionSymbol, ImportContext, SymbolInfo, SymbolType,
-        VariableSymbol,
-    },
-    backend::Backend,
-    files::FileVersion,
-    grammar::{
-        alpha034::{FunctionArgument, GlobalStatement, ImportContent},
-        Span, Spanned,
-    },
-    paths::FileId,
+use crate::analysis::types::{
+    make_union_type,
+    matches_type,
+    DataType,
 };
+use crate::analysis::{
+    self,
+    import_symbol,
+    insert_symbol_definition,
+    map_import_path,
+    Context,
+    FunctionContext,
+    FunctionSymbol,
+    ImportContext,
+    SymbolInfo,
+    SymbolType,
+    VariableSymbol,
+};
+use crate::backend::Backend;
+use crate::files::FileVersion;
+use crate::grammar::alpha034::{
+    FunctionArgument,
+    GlobalStatement,
+    ImportContent,
+};
+use crate::grammar::{
+    Span,
+    Spanned,
+};
+use crate::paths::FileId;
 
 use super::stmnts::analyze_stmnt;
 

src/analysis/alpha034/stmnts.rs

@@ -1,13 +1,27 @@
-use crate::{
-    analysis::{
-        get_symbol_definition_info, insert_symbol_definition, insert_symbol_reference,
-        types::{make_union_type, matches_type, GenericsMap},
-        BlockContext, Context, DataType, SymbolInfo, SymbolLocation, SymbolType, VariableSymbol,
-    },
-    files::{FileVersion, Files},
-    grammar::{alpha034::*, Spanned},
-    paths::FileId,
+use crate::analysis::types::{
+    make_union_type,
+    matches_type,
+    GenericsMap,
 };
+use crate::analysis::{
+    get_symbol_definition_info,
+    insert_symbol_definition,
+    insert_symbol_reference,
+    BlockContext,
+    Context,
+    DataType,
+    SymbolInfo,
+    SymbolLocation,
+    SymbolType,
+    VariableSymbol,
+};
+use crate::files::{
+    FileVersion,
+    Files,
+};
+use crate::grammar::alpha034::*;
+use crate::grammar::Spanned;
+use crate::paths::FileId;
 
 use super::exp::analyze_exp;