Add CI for regression tests

.github/workflows/run_tests.yml

@@ -0,0 +1,125 @@
+name: Run Tests
+
+on:
+  push:
+    branches:
+       - main
+  pull_request:
+    branches:
+       - main
+
+jobs:
+  unit-tests:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up MATLAB
+        uses: matlab-actions/setup-matlab@v2
+        with:
+          products: Signal_Processing_Toolbox
+          cache: true
+
+      - name: Run unit tests
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath("${{ github.workspace }}/k-Wave");   
+            cd("${{ github.workspace }}/k-Wave/testing/unit");
+            test_struct = runUnitTests("",false);
+            save('test_struct.mat', 'test_struct');
+          startup-options: -nojvm -logfile output.log
+
+      - name: Create artifact
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath("${{ github.workspace }}/k-Wave/testing");
+            cd("${{ github.workspace }}/k-Wave/testing/unit");
+            load('test_struct.mat', 'test_struct');
+            save_artifact(test_struct);
+          startup-options: -nojvm 
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit_test_results
+          path: ${{ github.workspace }}/k-Wave/testing/unit/test_results.json
+
+      - name: Test results 
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath("${{ github.workspace }}/k-Wave/testing");  
+            cd("${{ github.workspace }}/k-Wave/testing/unit");
+            load('test_struct.mat', 'test_struct');
+            show_test_results(test_struct);
+            disp('  ');
+            disp('NOTE:');
+            disp('Test output details are in the "Run unit tests" section of the workflow.');
+            disp('You can also download a JSON summary from the "Upload Artifact" section in your CI logs or dashboard.');
+          startup-options: -nojvm 
+
+  regression-tests:
+    name: Regression tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up MATLAB
+        uses: matlab-actions/setup-matlab@v2
+        with:
+          products: Signal_Processing_Toolbox
+          cache: true
+
+      - name: Generate data
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath("${{ github.workspace }}/k-Wave");  
+            cd("${{ github.workspace }}/k-Wave/testing/regression");
+            generateRegressionData
+          startup-options: -nojvm 
+
+      - name: Run regression tests 
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath("${{ github.workspace }}/k-Wave");   
+            cd("${{ github.workspace }}/k-Wave/testing/regression");
+            test_struct = runRegressionTests("${{ github.workspace }}/k-Wave/testing/regression",false);
+            save('test_struct.mat', 'test_struct');
+          startup-options: -nojvm 
+
+      - name: Create artifact
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath("${{ github.workspace }}/k-Wave/testing");  
+            cd("${{ github.workspace }}/k-Wave/testing/regression");
+            load('test_struct.mat', 'test_struct');
+            save_artifact(test_struct);
+          startup-options: -nojvm 
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: regression_test_results
+          path: ${{ github.workspace }}/k-Wave/testing/regression/test_results.json
+
+      - name: Test results 
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath("${{ github.workspace }}/k-Wave/testing");  
+            cd("${{ github.workspace }}/k-Wave/testing/regression");
+            load('test_struct.mat', 'test_struct');
+            show_test_results(test_struct);
+            disp('  ');
+            disp('NOTE:');
+            disp('Test output details are in the "Run regression tests" section of the workflow.');
+            disp('You can also download a JSON summary from the "Upload Artifact" section in your CI logs or dashboard.');
+          startup-options: -nojvm 

k-Wave/testing/regression/generateRegressionData.m

@@ -410,6 +410,7 @@
     load([tempdir 'generate_regression_data_TEMP_VARS']);
 
     % move to the testing directory
+    if ~exist(testing_dir, 'dir'), mkdir(testing_dir); end
     cd(testing_dir);
 
     % generate computer info and add precision

k-Wave/testing/regression/runRegressionTests.m

@@ -1,4 +1,4 @@
-function runRegressionTests(data_folder)
+function test_struct = runRegressionTests(data_folder, show_results)
 %RUNREGRESSIONTESTS Run MATLAB regression tests.
 %
 % DESCRIPTION:
@@ -18,6 +18,7 @@ function runRegressionTests(data_folder)
 %
 % INPUTS:
 %     data_folder   - string of folder containing regression data
+%     show_results  - logical flag to display results (default: true)
 %
 % ABOUT:
 %     author        - Bradley Treeby
@@ -44,6 +45,11 @@ function runRegressionTests(data_folder)
 %#ok<*NASGU>
 %#ok<*IDISVAR>
 
+% Set defaults for optional arguments
+if nargin < 2 || isempty(show_results)
+    show_results = true;
+end
+
 % start the timer
 regression_start_time = datetime('now');
 
@@ -77,6 +83,9 @@ function runRegressionTests(data_folder)
 % keep a list of whether the test passed or failed
 test_result = false(num_files, 1);
 
+% preallocate cell array for test_info with empty strings by default
+test_info = repmat({''}, num_files, 1);
+
 % =========================================================================
 % RUN TESTS
 % =========================================================================
@@ -93,7 +102,7 @@ function runRegressionTests(data_folder)
     save(temp_var_filename, ...
         'DOUBLE_COMPARISON_THRESHOLD', 'SINGLE_COMPARISON_THRESHOLD', ...
         'regression_test_folder', 'temp_var_filename', 'regression_start_time', ...
-        'filename_index', 'filenames', 'fn', 'num_files', 'test_result');
+        'filename_index', 'filenames', 'fn', 'num_files', 'test_result', 'show_results');
 
     % display the filename
     disp(' ');
@@ -238,10 +247,14 @@ function runRegressionTests(data_folder)
                         test_pass = true;
                         fprintf('passed');
                     end
+                    % save the error string
+                    error_str = sprintf(' (L_inf = %e)\n', L_inf);
 
                     % display the error
-                    fprintf(' (L_inf = %e)\n', L_inf);
-
+                    fprintf('%s', error_str);
+
+                    % save L_inf as test_info
+                    test_info{filename_index} = error_str;
                 end
 
                 % clear the variables just in case
@@ -256,69 +269,27 @@ function runRegressionTests(data_folder)
 
     % store test result
     test_result(filename_index) = test_pass_overall;
-
 end
 
 % =========================================================================
-% DISPLAY SUMMARY
+% CREATE OUTPUT
 % =========================================================================
 
-% get information about PC
+completion_time = scaleTime(seconds(datetime('now') - regression_start_time));
 comp_info = getComputerInfo;
+info = comp_info;
+info.completion_time = completion_time;
 
-% get k-Wave version
-eval('cur_dir = pwd; cd(getkWavePath(''private'')); kwave_ver = getkWaveVersion; cd(cur_dir);');
-
-% display test header
-disp('   ');
-disp('-------------------------------------------------------------------------------------');
-disp('            _      __        __                _____         _            ');
-disp('           | | __  \ \      / /_ ___   _____  |_   _|__  ___| |_ ___ _ __ ');
-disp('           | |/ /___\ \ /\ / / _` \ \ / / _ \   | |/ _ \/ __| __/ _ \ ''__|');
-disp('           |   <_____\ V  V / (_| |\ V /  __/   | |  __/\__ \ ||  __/ |   ');
-disp('           |_|\_\     \_/\_/ \__,_| \_/ \___|   |_|\___||___/\__\___|_|   ');
-disp('  ');                                                                
-disp('-------------------------------------------------------------------------------------');
-disp('  ');
-disp(['DATE:                     ' comp_info.date]);
-disp(['HOST NAME:                ' comp_info.computer_name]);
-disp(['USER NAME:                ' comp_info.user_name]);
-disp(['O/S TYPE:                 ' comp_info.operating_system_type]);
-disp(['O/S:                      ' comp_info.operating_system]);
-disp(['MATLAB VERSION:           ' comp_info.matlab_version]);
-disp(['TESTED K-WAVE VERSION:    ' kwave_ver]);
-disp(['TESTS COMPLETED IN:       ' scaleTime(seconds(datetime('now') - regression_start_time))]);
-disp('  ');
-
-% display individual test results
-disp('REGRESSION TEST RESULTS:');
-for filename_index = 1:length(filenames)
-
-    % trim the filename and add number
-    fn = filenames{filename_index};
-    fn = [num2str(filename_index, '%02.f') ' ' fn(1:end - 4), ':'];
-
-    % add some spaces to align results
-    fn = sprintf('%-50s', fn);
-
-    % append the test result
-    if test_result(filename_index)
-        disp(['  ' fn 'passed']);
-    else
-        disp(['  ' fn 'failed']);
-    end
-end
-
-% display test summary
-disp('  ');
-if all(test_result)
-    disp('ALL REGRESSION TESTS PASSED!');
-else
-    disp('REGRESSION TEST FAILED...');
-end
+% create results struct
+test_struct = struct( ...
+    'info', info, ...
+    'results', struct('test', filenames(:), 'pass', num2cell(test_result(:)), 'test_info', test_info(:)) ...
+);
 
-disp('  ');
-disp('-------------------------------------------------------------------------------------');
+% =========================================================================
+% SHOW RESULTS
+% =========================================================================
 
-% remove temp data
-delete(temp_var_filename);
+if show_results
+    show_test_results(test_struct);
+end

k-Wave/testing/unit/runUnitTests_artifact.m → k-Wave/testing/save_artifact.m

@@ -1,8 +1,8 @@
-function runUnitTests_artifact(test_struct)
-%RUNUNITTESTS_ARTIFACT 
+function save_artifact(test_struct)
+%SAVE_ARTIFACT 
 %
 % DESCRIPTION:
-%     runUnitTests_actions processes the provided test_struct, saves the results
+%     save_artifact processes the provided test_struct, saves the results
 %     as a test_results.json artifact.
 %