Fix amalgamation and distribution test setup

.github/workflows/build_embedDB_distribution_desktop.yml

@@ -5,25 +5,13 @@ on:
         branches: ["update-distribution"]
 
 jobs:
-    build_platformIO:
-        name: Build and Run EmbedDB Distribution Using PlatformIO
-        runs-on: ${{ matrix.os }}
-        strategy:
-            matrix:
-                os: [ubuntu-latest, windows-latest]
+    build_platformIO_ubuntu:
+        name: Build and Run EmbedDB Distribution Using PlatformIO (Ubuntu)
+        runs-on: ubuntu-latest
 
         steps:
             - uses: actions/checkout@v3
 
-            - name: Set environment variable
-              run: |
-                if [[ "$RUNNER_OS" == "Windows" ]]; then
-                    echo "EXPORT_COMMAND=set" >> $GITHUB_ENV
-                else
-                    echo "EXPORT_COMMAND=export" >> $GITHUB_ENV
-                fi
-              shell: bash
-
             - name: Set up Python
               uses: actions/setup-python@v4
 
@@ -32,33 +20,72 @@ jobs:
                 python -m pip install --upgrade pip
                 python -m pip install platformio
 
-            - name: Run EmbedDB Distribution Example
+            - name: Run EmbedDB Example (Ubuntu)
+              shell: bash
+              run: |
+                pio run -e desktop-dist -t fullclean
+                PLATFORMIO_BUILD_FLAGS="-DWHICH_PROGRAM=0 -lm" pio run -e desktop-dist -t exec
+
+            - name: Run EmbedDB Sequential Data Benchmark (Ubuntu)
+              shell: bash
+              run: |
+                pio run -e desktop-dist -t fullclean
+                PLATFORMIO_BUILD_FLAGS="-DWHICH_PROGRAM=1 -lm" pio run -e desktop-dist -t exec
+
+            - name: Run EmbedDB Variable Data Benchmark (Ubuntu)
+              shell: bash
+              run: |
+                pio run -e desktop-dist -t fullclean
+                PLATFORMIO_BUILD_FLAGS="-DWHICH_PROGRAM=2 -lm" pio run -e desktop-dist -t exec
+
+            - name: Run EmbedDB Query Interface Benchmark (Ubuntu)
+              shell: bash
               run: |
-                    pio run -e desktop-dist -t fullclean
-                    ${{ env.EXPORT_COMMAND }} PLATFORMIO_BUILD_FLAGS="-DWHICH_PROGRAM=0 -lm"
-                    pio run -e desktop-dist -t exec
+                pio run -e desktop-dist -t fullclean
+                PLATFORMIO_BUILD_FLAGS="-DWHICH_PROGRAM=3 -lm" pio run -e desktop-dist -t exec
 
-            - name: Run EmbedDB Distribution Sequential Data Benchmark
+    build_platformIO_windows:
+        name: Build and Run EmbedDB Distribution Using PlatformIO (Windows)
+        runs-on: windows-latest
+
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Set up Python
+              uses: actions/setup-python@v4
+
+            - name: Install PlatformIO
               run: |
-                    pio run -e desktop-dist -t fullclean
-                    ${{ env.EXPORT_COMMAND }} PLATFORMIO_BUILD_FLAGS="-DWHICH_PROGRAM=1 -lm"
-                    pio run -e desktop-dist -t exec
+                python -m pip install --upgrade pip
+                python -m pip install platformio         
 
-            - name: Run EmbedDB Distribution Variable Data Benchmark
+            - name: Run EmbedDB Example (Windows)
+              shell: pwsh
               run: |
-                    pio run -e desktop-dist -t fullclean
-                    ${{ env.EXPORT_COMMAND }} PLATFORMIO_BUILD_FLAGS="-DWHICH_PROGRAM=2 -lm"
-                    pio run -e desktop-dist -t exec
-
-            - name: Run EmbedDB Distribution Query Interface Benchmark
+                pio run -e desktop-dist -t fullclean
+                $env:PLATFORMIO_BUILD_FLAGS='-DWHICH_PROGRAM=0 -lm'; pio run -e desktop-dist -t exec
+
+            - name: Run EmbedDB Sequential Data Benchmark (Windows)
+              shell: pwsh
+              run: |
+                pio run -e desktop-dist -t fullclean
+                $env:PLATFORMIO_BUILD_FLAGS='-DWHICH_PROGRAM=1 -lm'; pio run -e desktop-dist -t exec
+
+            - name: Run EmbedDB Variable Data Benchmark (Windows)
+              shell: pwsh
+              run: |
+                pio run -e desktop-dist -t fullclean
+                $env:PLATFORMIO_BUILD_FLAGS='-DWHICH_PROGRAM=2 -lm'; pio run -e desktop-dist -t exec
+
+            - name: Run EmbedDB Query Interface Benchmark (Windows)
+              shell: pwsh
               run: |
-                    pio run -e desktop-dist -t fullclean
-                    ${{ env.EXPORT_COMMAND }} PLATFORMIO_BUILD_FLAGS="-DWHICH_PROGRAM=3 -lm"
-                    pio run -e desktop-dist -t exec
+                pio run -e desktop-dist -t fullclean
+                $env:PLATFORMIO_BUILD_FLAGS='-DWHICH_PROGRAM=3 -lm'; pio run -e desktop-dist -t exec
 
     test_platformIO:
         name: Test EmbedDB Distribution using PlatformIO
-        needs: build_platformIO
+        needs: [build_platformIO_ubuntu, build_platformIO_windows]
         runs-on: ${{ matrix.os }}
         strategy:
             matrix:

.github/workflows/build_run_embedDB_desktop.yml

@@ -22,7 +22,7 @@ jobs:
             - name: Run EmbedDB Example (Ubuntu)
               shell: bash
               run: |
-                pio run -e desktop-dist -t fullclean
+                pio run -e desktop -t fullclean
                 PLATFORMIO_BUILD_FLAGS="-DWHICH_PROGRAM=0 -lm" pio run -e desktop -t exec
 
             - name: Run EmbedDB Sequential Data Benchmark (Ubuntu)

amalgamation/amalgamation.py

@@ -677,7 +677,12 @@ def amalgamate(
         sorted_h, isCpp
     )
 
-    sorted_graph_c_files = [item.replace(".h", ".c") for item in sorted_graph]
+    sorted_graph_c_files = [
+        item.replace(".h", ".c")
+        for item in sorted_graph
+        if any(fn.file_name == item.replace(".h", ".c") for fn in source_file_nodes)
+    ]
+
     sorted_c = order_file_nodes_by_sorted_filenames(
         source_file_nodes, sorted_graph_c_files
     )

makefile

@@ -55,7 +55,7 @@ DEV_TEST_OBJECTS = $(EMBEDDB_OBJECTS) $(QUERY_OBJECTS) $(EMBEDDB_FILE_INTERFACE)
 
 TEST_FLAGS = -I. -I$(PATHU) -I $(PATHS) -I$(PATH_UTILITY) -I$(PATH_FILE_INTERFACE) -D TEST
 EXAMPLE_FLAGS = -I. -I$(PATHS) -I$(PATH_UTILITY) -I$(PATH_FILE_INTERFACE) -I$(PATH_DISTRIBUTION) -DPRINT_ERRORS
-TEST_DIST_FLAGS = -I. -I$(PATHU) -I$(PATH_FILE_INTERFACE) -I$(PATH_DISTRIBUTION) -DDIST -D TEST
+TEST_DIST_FLAGS = -I. -I$(PATHS) -I$(PATHU) -I$(PATH_FILE_INTERFACE) -I$(PATH_DISTRIBUTION) -I$(PATH_UTILITY) -DDIST -D TEST
 
 override CFLAGS += $(if $(filter test-dist,$(MAKECMDGOALS)), $(TEST_DIST_FLAGS), $(if $(filter test,$(MAKECMDGOALS)),$(TEST_FLAGS),$(EXAMPLE_FLAGS)) )
 

test/test_in_memort_quick_sort/test_in_memory_quick_sort.cpp

@@ -1,5 +1,5 @@
 #ifdef DIST
-#include "in_memory_sort.h"
+#include "embedDB.h"
 #else
 #include "query-interface/sort/in_memory_sort.h"
 #endif

test/test_sort/test_sort_query_interface.cpp

@@ -1,6 +1,10 @@
-#include "embedDB/embedDB.h"
+#ifdef DIST
+#include "embedDB.h"
+#else
 #include "embedDBUtility.h"
-#include "unity.h"
+#include "query-interface/advancedQueries.h"
+
+#endif
 
 #ifdef ARDUINO
 // For Arduino, setupFile is not used since we use pure memory sort
@@ -48,23 +52,71 @@ void* setupFile(const char* filename) {
 #define DATA_FILE_PATH_SEA "dataFileSEA.bin"
 #define INDEX_FILE_PATH_SEA "indexFileSEA.bin"
 
-#include "query-interface/advancedQueries.h"
-
 #else
 
-#include "desktopFileInterface.h"
-#include "query-interface/advancedQueries.h"
 #define FILE_TYPE FILE
+#include "desktopFileInterface.h"
 #define DATA_FILE_PATH_UWA "build/artifacts/dataFileUWA.bin"
 #define INDEX_FILE_PATH_UWA "build/artifacts/indexFileUWA.bin"
 #define DATA_FILE_PATH_SEA "build/artifacts/dataFileSEA.bin"
 #define INDEX_FILE_PATH_SEA "build/artifacts/indexFileSEA.bin"
 
 #endif
 
-void setUp() {}
+#include "unity.h"
+
+embedDBState* stateUWA;
+embedDBSchema* baseSchema;
+
+void setUp() {
+    if (STORAGE_TYPE == 1) {
+        TEST_FAIL_MESSAGE("Dataflash is not currently supported. Defaulting to SD card interface.");
+    }
+    stateUWA = (embedDBState*)malloc(sizeof(embedDBState));
+    stateUWA->keySize = 4;
+    stateUWA->dataSize = 12;
+    stateUWA->compareKey = int32Comparator;
+    stateUWA->compareData = int32Comparator;
+    stateUWA->pageSize = 512;
+    stateUWA->eraseSizeInPages = 4;
+    stateUWA->numDataPages = 20000;
+    stateUWA->numIndexPages = 1000;
+    stateUWA->numSplinePoints = 30;
+    /* Setup files */
+    char dataPath[] = DATA_FILE_PATH_UWA, indexPath[] = INDEX_FILE_PATH_UWA;
+    stateUWA->fileInterface = getFileInterface();
+    stateUWA->dataFile = setupFile(dataPath);
+    stateUWA->indexFile = setupFile(indexPath);
+
+    stateUWA->bufferSizeInBlocks = 4;
+    stateUWA->buffer = malloc(stateUWA->bufferSizeInBlocks * stateUWA->pageSize);
+    stateUWA->parameters = EMBEDDB_USE_BMAP | EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA;
+    stateUWA->bitmapSize = 2;
+    stateUWA->inBitmap = inBitmapInt16;
+    stateUWA->updateBitmap = updateBitmapInt16;
+    stateUWA->buildBitmapFromRange = buildBitmapInt16FromRange;
+    int8_t initResult = embedDBInit(stateUWA, 1);
+    if (initResult != 0) {
+        TEST_FAIL_MESSAGE("There was an error setting up the state of the UWA dataset.");
+    }
+    stateUWA->rules = NULL;
+    stateUWA->numRules = 0;
+
+    int8_t colSizes[] = {4, 12};
+    int8_t colSignedness[] = {embedDB_COLUMN_UNSIGNED, embedDB_COLUMN_UNSIGNED};
+    ColumnType colTypes[] = {embedDB_COLUMN_UINT32, embedDB_COLUMN_UINT32};
+    baseSchema = embedDBCreateSchema(2, colSizes, colSignedness, colTypes);
+}
 
-void tearDown() {}
+void tearDown() {
+    embedDBClose(stateUWA);
+    tearDownFile(stateUWA->dataFile);
+    tearDownFile(stateUWA->indexFile);
+    free(stateUWA->fileInterface);
+    free(stateUWA->buffer);
+    free(stateUWA);
+    embedDBFreeSchema(&baseSchema);
+}
 
 void insertData(embedDBState* state, const char* filename) {
     FILE_TYPE* fp = fopen(filename, "rb");
@@ -107,45 +159,7 @@ void insertNValues(embedDBState* state, int n, int mode) {
 }
 
 void runTestSequentialValues() {
-    if (STORAGE_TYPE == 1) {
-        TEST_FAIL_MESSAGE("Dataflash is not currently supported. Defaulting to SD card interface.");
-    }
-    embedDBState* stateUWA = (embedDBState*)malloc(sizeof(embedDBState));
-    stateUWA->keySize = 4;
-    stateUWA->dataSize = 12;
-    stateUWA->compareKey = int32Comparator;
-    stateUWA->compareData = int32Comparator;
-    stateUWA->pageSize = 512;
-    stateUWA->eraseSizeInPages = 4;
-    stateUWA->numDataPages = 20000;
-    stateUWA->numIndexPages = 1000;
-    stateUWA->numSplinePoints = 30;
-    /* Setup files */
-    char dataPath[] = DATA_FILE_PATH_UWA, indexPath[] = INDEX_FILE_PATH_UWA;
-    stateUWA->fileInterface = getFileInterface();
-    stateUWA->dataFile = setupFile(dataPath);
-    stateUWA->indexFile = setupFile(indexPath);
-
-    stateUWA->bufferSizeInBlocks = 4;
-    stateUWA->buffer = malloc(stateUWA->bufferSizeInBlocks * stateUWA->pageSize);
-    stateUWA->parameters = EMBEDDB_USE_BMAP | EMBEDDB_USE_INDEX | EMBEDDB_RESET_DATA;
-    stateUWA->bitmapSize = 2;
-    stateUWA->inBitmap = inBitmapInt16;
-    stateUWA->updateBitmap = updateBitmapInt16;
-    stateUWA->buildBitmapFromRange = buildBitmapInt16FromRange;
-    int8_t initResult = embedDBInit(stateUWA, 1);
-    if (initResult != 0) {
-        TEST_FAIL_MESSAGE("There was an error setting up the state of the UWA dataset.");
-    }
-    stateUWA->rules = NULL;
-    stateUWA->numRules = 0;
-
-    int8_t colSizes[] = {4, 12};
-    int8_t colSignedness[] = {embedDB_COLUMN_UNSIGNED, embedDB_COLUMN_UNSIGNED};
-    ColumnType colTypes[] = {embedDB_COLUMN_UINT32, embedDB_COLUMN_UINT32};
-    embedDBSchema* baseSchema = embedDBCreateSchema(2, colSizes, colSignedness, colTypes);
-
-// Insert test data
+    // Insert test data
 #ifdef ARDUINO
     insertNValues(stateUWA, 1, 0);
 #else
@@ -183,15 +197,6 @@ void runTestSequentialValues() {
 
     orderByOp->close(orderByOp);
     embedDBFreeOperatorRecursive(&orderByOp);
-
-    // Close embedDB
-    embedDBClose(stateUWA);
-    tearDownFile(stateUWA->dataFile);
-    tearDownFile(stateUWA->indexFile);
-    free(stateUWA->fileInterface);
-    free(stateUWA->buffer);
-    free(stateUWA);
-    embedDBFreeSchema(&baseSchema);
 }
 
 void runTestUsingUWA500k() {
@@ -261,7 +266,6 @@ void runTestUsingUWA500k() {
 
     orderByOp->close(orderByOp);
     embedDBFreeOperatorRecursive(&orderByOp);
-
     // Close embedDB
     embedDBClose(stateUWA);
     tearDownFile(stateUWA->dataFile);