Raw Datalogger

.github/sample-roots.yaml

@@ -46,6 +46,9 @@ app/samples/horus:
 app/samples/compression:
   - paths:
       - "app/samples/compression/**"
+app/samples/raw_datalogger:
+    - paths:
+        - "app/samples/raw_datalogger/**"
 ALL:
   - paths:
       - "include/f_core/**"

app/backplane/power_module/include/c_power_module.h

@@ -69,7 +69,7 @@ class CPowerModule : public CProjectConfiguration {
         "Broadcast Tenant", ipAddrStr, downlinkBroadcastPort, downlinkBroadcastPort, sensorDataDownlinkMessagePort
     };
 
-    CDataLoggerTenant<NTypes::TimestampedSensorData> dataLoggerTenant{
+    CFsDataLoggerTenant<NTypes::TimestampedSensorData> dataLoggerTenant{
         "Data Logger Tenant", "/lfs/sensor_data.bin", LogMode::Growing, 0, sensorDataLogMessagePort, K_SECONDS(60), 5
     };
     CUdpAlertTenant alertTenant{"Alert Tenant", ipAddrStr, NNetworkDefs::ALERT_PORT};

app/backplane/radio_module/include/c_radio_module.h

@@ -75,7 +75,7 @@ class CRadioModule : public CProjectConfiguration {
     CLoraTransmitTenant loraTransmitTenant{"LoRa Transmit Tenant", lora, &loraBroadcastMessagePort};
     CLoraReceiveTenant loraReceiveTenant{"LoRa Receive Tenant", loraTransmitTenant, ipAddrStr, radioModuleSourcePort};
 #endif
-    CDataLoggerTenant<NTypes::GnssData> dataLoggerTenant{"Data Logger Tenant", "/lfs/gps_data.bin", LogMode::Growing, 0, gnssDataLogMessagePort, K_SECONDS(15), 5};
+    CFsDataLoggerTenant<NTypes::GnssData> dataLoggerTenant{"Data Logger Tenant", "/lfs/gps_data.bin", LogMode::Growing, 0, gnssDataLogMessagePort, K_SECONDS(15), 5};
     CStateMachineUpdater stateMachineUpdater;
 
     // Tasks

app/backplane/sensor_module/include/c_sensor_module.h

@@ -71,7 +71,7 @@ class CSensorModule : public CProjectConfiguration {
     CUdpBroadcastTenant<NTypes::SensorData> broadcastTenant{"Broadcast Tenant", ipAddrStr.c_str(), telemetryBroadcastPort, telemetryBroadcastPort, sensorDataBroadcastMessagePort};
     CUdpBroadcastTenant<NTypes::LoRaBroadcastSensorData> downlinkTelemTenant{"Telemetry Downlink Tenant", ipAddrStr.c_str(), telemetryDownlinkPort, telemetryDownlinkPort, downlinkMessagePort};
     CUdpBroadcastTenant<NAlerts::AlertPacket> udpAlertTenant{"UDP Alert Tenant", ipAddrStr.c_str(), alertPort, alertPort, alertMessagePort};
-    CDataLoggerTenant<NTypes::TimestampedSensorData> dataLoggerTenant{"Data Logger Tenant", "/lfs/sensor_module_data.bin", LogMode::Growing, 0, sensorDataLogMessagePort, K_SECONDS(3), 64};
+    CFsDataLoggerTenant<NTypes::TimestampedSensorData> dataLoggerTenant{"Data Logger Tenant", "/lfs/sensor_module_data.bin", LogMode::Growing, 0, sensorDataLogMessagePort, K_SECONDS(3), 64};
 
     // Tasks
     CTask networkTask{"Networking Task", 15, 3072, 0};

app/backplane/sensor_module/src/c_sensor_module.cpp

@@ -4,6 +4,7 @@
 #include <f_core/messaging/c_msgq_message_port.h>
 #include <f_core/os/n_rtos.h>
 #include <f_core/utils/n_time_utils.h>
+#include <f_core/os/c_raw_datalogger.h>
 #include <zephyr/logging/log.h>
 LOG_MODULE_REGISTER(sensor_module);
 
@@ -13,7 +14,7 @@ static auto broadcastMsgQueue = CMsgqMessagePort<NTypes::SensorData>(broadcastQu
 K_MSGQ_DEFINE(downlinkQueue, sizeof(NTypes::LoRaBroadcastSensorData), 10, 4);
 static auto downlinkMsgQueue = CMsgqMessagePort<NTypes::LoRaBroadcastSensorData>(downlinkQueue);
 
-K_MSGQ_DEFINE(dataLogQueue, sizeof(NTypes::TimestampedSensorData), 512, 4);
+K_MSGQ_DEFINE(dataLogQueue, sizeof(NTypes::TimestampedSensorData), 10, 4);
 static auto dataLogMsgQueue = CMsgqMessagePort<NTypes::TimestampedSensorData>(dataLogQueue);
 
 K_MSGQ_DEFINE(alertQueue, sizeof(NAlerts::AlertPacket), 4, 4);

app/samples/benchmark_littlefs_datalogger/src/main.cpp

@@ -73,7 +73,7 @@ template <typename T>
 void benchmarkDataloggerMode(const char *testName, const char *filePath, LogMode mode, size_t maxPackets = 1000, size_t syncFrequency = 10) {
     LOG_INF("\n\n=== %s ===", testName);
 
-    CDataLogger<T> logger(filePath, mode, maxPackets);
+    CFsDataLogger<T> logger(filePath, mode, maxPackets);
 
     uint64_t totalWriteCycles = 0;
     uint64_t totalSyncCycles = 0;

app/samples/datalogger/native_sim.overlay

@@ -1,9 +1,14 @@
 / {
 	chosen {
+		flashcontroller = &flash0;
 		storage = &flash0;
 		logfs = &lfs1;
 	};
 
+	aliases {
+		storage = &flash0;
+	};
+
 
 	fstab {
 		compatible = "zephyr,fstab";

app/samples/datalogger/src/main.cpp

@@ -1,14 +1,14 @@
-#include "f_core/os/c_datalogger.h"
+#include "f_core/os/c_fs_datalogger.h"
 
 #include <zephyr/kernel.h>
 
 struct Packet {
     uint8_t a;
     uint8_t b;
 };
-CDataLogger<Packet> expand_logger{"/lfs/expand.bin"};
-CDataLogger<Packet> fill_logger{"/lfs/fill.bin", LogMode::FixedSize, 10};
-CDataLogger<Packet> wrap_logger{"/lfs/wrap.bin", LogMode::Circular, 10};
+CFsDataLogger<Packet> expand_logger{"/lfs/expand.bin"};
+CFsDataLogger<Packet> fill_logger{"/lfs/fill.bin", LogMode::FixedSize, 10};
+CFsDataLogger<Packet> wrap_logger{"/lfs/wrap.bin", LogMode::Circular, 10};
 
 int main() {
     for (uint8_t i = 0; i < 100; i++) {

app/samples/raw_datalogger/CMakeLists.txt

@@ -0,0 +1,10 @@
+cmake_minimum_required(VERSION 3.20.0)
+
+set(FSW_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
+
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(raw_datalogger LANGUAGES CXX)
+
+target_compile_options(app PRIVATE -Wall -Wextra -Wno-unused-parameter -Wno-unused-function -Wno-ignored-qualifiers)
+FILE(GLOB app_sources src/*.cpp)
+target_sources(app PRIVATE ${app_sources})

app/samples/raw_datalogger/Kconfig

@@ -0,0 +1 @@
+source "Kconfig.zephyr"

app/samples/raw_datalogger/native_sim.overlay

@@ -0,0 +1,27 @@
+	/ {
+	chosen {
+		storage = &flash0;
+	};
+
+	aliases {
+		storage = &flash0;
+	};
+};
+
+&flashcontroller0 {
+    status = "okay";
+    compatible = "zephyr,sim-flash";
+    reg = <0x00000000 0x00100000>;
+
+    #address-cells = <1>;
+    #size-cells = <1>;
+    erase-value = <0xff>;
+
+    flash0: flash@0 {
+        status = "okay";
+        compatible = "soc-nv-flash";
+        erase-block-size = <4096>;
+        write-block-size = <1>;
+        reg = <0x00000000 0x00100000>;
+    };
+};

app/samples/raw_datalogger/prj.conf

@@ -0,0 +1,24 @@
+CONFIG_CPP=y
+CONFIG_REQUIRES_FULL_LIBCPP=y
+CONFIG_STD_CPP20=y
+
+CONFIG_F_CORE=y
+CONFIG_F_CORE_OS=y
+
+ # outputs
+CONFIG_SERIAL=y
+CONFIG_HEAP_MEM_POOL_SIZE=8192
+
+CONFIG_DEBUG=y
+CONFIG_CBPRINTF_FP_SUPPORT=y
+
+CONFIG_MAIN_STACK_SIZE=8192
+CONFIG_FLASH=y
+CONFIG_SHELL=y
+CONFIG_FLASH_SHELL=y
+CONFIG_FLASH_MAP=y
+CONFIG_FLASH_PAGE_LAYOUT=y
+CONFIG_FLASH_SIMULATOR=y
+CONFIG_STREAM_FLASH=y
+
+CONFIG_LOG=y

app/samples/raw_datalogger/sample.yaml

@@ -0,0 +1,10 @@
+sample:
+  description:
+  name: raw_datalogger
+common:
+  build_only: true
+  platform_allow:
+    - native_sim
+    - nucleo_f446re
+tests:
+  samples.template.default: {}

app/samples/raw_datalogger/src/main.cpp

@@ -0,0 +1,139 @@
+#include "zephyr/logging/log.h"
+
+#include <f_core/os/c_raw_datalogger.h>
+#include <zephyr/kernel.h>
+
+LOG_MODULE_REGISTER(main);
+
+struct TestData {
+    const char name[16];
+    int a;
+    int b;
+    int c;
+    int d;
+};
+
+int main() {
+    const device* flash = DEVICE_DT_GET(DT_CHOSEN(zephyr_flash_controller));
+    if (!device_is_ready(flash)) {
+        LOG_ERR("Flash device not ready!");
+        return -1;
+    } else {
+        // Print flash size
+        uint64_t flashSize = 0;
+        int ret = flash_get_size(flash, &flashSize);
+        if (ret < 0) {
+            LOG_ERR("Failed to get flash size: %d", ret);
+            return -1;
+        }
+
+        LOG_INF("Flash Size: %llu bytes", flashSize);
+    }
+    off_t start_addr = 0x00000000;
+    size_t erase_size = 0x00100000; // 1 MiB
+
+    int err = flash_erase(flash, start_addr, erase_size);
+    if (err < 0) {
+        LOG_ERR("Failed to erase flash: %d", err);
+        return -1;
+    }
+
+    // Fixed
+    off_t nextAddr = 0x00000000;
+    const int packetsInFixedFile = 5;
+    const size_t fixedFileSize = sizeof(TestData) * packetsInFixedFile + sizeof(DataloggerMetadata);
+    CRawDataLogger<TestData, 3> fixedLogger(flash, nextAddr, fixedFileSize, "test_fixed", DataloggerMode::Fixed);
+    for (int i = 0; i < packetsInFixedFile; ++i) {
+        char iter = '0' + i;
+        TestData data = { "fixed", iter, iter, iter, iter };
+        int ret = fixedLogger.Write(data);
+        if (ret < 0) {
+            LOG_ERR("Error writing data for fixed: %d", ret);
+        } else {
+            LOG_INF("Successfully wrote %d entries to fixed logger", i + 1);
+        }
+    }
+    LOG_INF("Finished writing data for fixed logger.");
+    if (fixedLogger.Write(TestData{ "fixed_overflow", 'B', 'A', 'D', '!' }) != -ENOSPC) {
+        LOG_ERR("Error: Should not have been able to write more data to fixed logger");
+    }
+
+    nextAddr += fixedFileSize;
+
+    // LinkedFixed
+    const size_t linkedFixedFileSize = (sizeof(TestData) * 5) + sizeof(DataloggerMetadata);
+    CRawDataLogger<TestData, 3> linkedFixedLogger(flash, nextAddr, linkedFixedFileSize, "test_link_fixed", DataloggerMode::LinkedFixed);
+    for (int i = 0; i < 5; ++i) {
+        char iter = '0' + i;
+        TestData data = { "linked_fixed", iter, iter, iter, iter };
+        int ret = linkedFixedLogger.Write(data);
+        if (ret < 0) {
+            LOG_ERR("Error writing data for linked fixed: %d", ret);
+        } else {
+            LOG_INF("Successfully wrote %d entries to linked fix logger", i + 1);
+        }
+    }
+
+    LOG_INF("Finished writing data for linked fixed logger.");
+
+
+    nextAddr += linkedFixedFileSize;
+    // Add a file in between to test finding next linked space using a fixed logger
+    const size_t intermediateFileSize = sizeof(TestData) + sizeof(DataloggerMetadata);
+    CRawDataLogger<TestData, 1> intermediateLogger(flash, nextAddr, intermediateFileSize, "intermediate", DataloggerMode::Fixed);
+    TestData intermediateData = { "intermediate", 'I', 'N', 'T', 'R' };
+    intermediateLogger.Write(intermediateData);
+    if (intermediateLogger.GetLastError() != 0) {
+        LOG_ERR("Error writing intermediate data: %d", intermediateLogger.GetLastError());
+    }
+
+    LOG_INF("Finished writing data for intermediate fixed logger");
+
+    // LinkedFixed should skip the intermediate file and create 2 new files consecutively
+    for (int i = 0; i < 10; ++i) {
+        char iter = '0' + i;
+
+        TestData data = { "linked_fixed2", iter, iter, iter, iter };
+        int ret = linkedFixedLogger.Write(data);
+        if (ret < 0) {
+            LOG_ERR("Error writing data for linked fixed 2: %d", ret);
+        } else {
+            LOG_INF("Successfully wrote %d entries to linked fixed logger", i + 1);
+        }
+    }
+
+    LOG_INF("Finished writing more data for linked fixed logger.");
+
+    nextAddr += intermediateFileSize + (2 * linkedFixedFileSize); // Move past the intermediate and 2 linked fixed files
+
+    // Test LinkedTruncate
+    const size_t linkedTruncateFileSize = (sizeof(TestData) * 5) + sizeof(DataloggerMetadata);
+    CRawDataLogger<TestData, 3> linkedTruncateLogger(flash, nextAddr, linkedTruncateFileSize, "test_link_trunc", DataloggerMode::LinkedTruncate);
+
+    nextAddr += ((linkedTruncateFileSize * 2) + sizeof(DataloggerMetadata)) - (sizeof(TestData) * 2); // Leave space for 2 files minus 2 entries to test truncation
+    CRawDataLogger<TestData, 1> anotherIntermediateLogger(flash, nextAddr, intermediateFileSize, "intermediate2", DataloggerMode::Fixed);
+    anotherIntermediateLogger.Write(intermediateData);
+
+    if (anotherIntermediateLogger.GetLastError() != 0) {
+        LOG_ERR("Error writing another intermediate data: %d", anotherIntermediateLogger.GetLastError());
+    }
+    LOG_INF("Finished writing data for another intermediate fixed logger");
+
+
+    for (int i = 0; i < 15; ++i) {
+        char iter = '0' + (i % 10);
+
+        TestData data = { "linked_truncate", iter, iter, iter, iter  };
+        int ret = linkedTruncateLogger.Write(data);
+
+        if (ret < 0) {
+            LOG_ERR("Error writing data for linked truncate: %d", ret);
+        } else {
+            LOG_INF("Successfully wrote %d entries to linked truncate logger", i + 1);
+        }
+    }
+
+    LOG_INF("Finished writing data for linked truncate logger");
+
+    return 0;
+}

include/f_core/os/c_datalogger.h → include/f_core/os/c_fs_datalogger.h

@@ -7,7 +7,7 @@
 
 enum class LogMode { Growing, Circular, FixedSize };
 
-/// @brief Internal, type-unsafe datalogger. Don't use this directly. Use CDataLogger instead
+/// @brief Internal, type-unsafe datalogger. Don't use this directly. Use CFsDataLogger instead
 namespace detail {
 class datalogger {
   public:
@@ -32,7 +32,7 @@ class datalogger {
  * This class is implemented as a type safe wrapper to detail::datalogger.
  */
 template <typename T>
-class CDataLogger {
+class CFsDataLogger {
   public:
     using PacketType = T;
     static_assert(std::is_trivially_copyable<PacketType>::value,
@@ -46,14 +46,14 @@ class CDataLogger {
      * The logger will use the "Growing" mode and will expand as you write more data until your filesystem runs out of space.
      * @param filename the name of the file to write to
      */
-    CDataLogger(const char *filename) : internal(filename, LogMode::Growing, 0) {}
+    CFsDataLogger(const char *filename) : internal(filename, LogMode::Growing, 0) {}
     /**
      * Construct a Datalogger for the specified filename, grow mode, and size
      * @param filename the name of the file to write to
      * @param mode the logging mode to use
      * @param the number of packets to log (only used if mode is Circular or FixedSize)
      */
-    CDataLogger(const char *filename, LogMode mode, std::size_t num_packets) : internal(filename, mode, num_packets) {}
+    CFsDataLogger(const char *filename, LogMode mode, std::size_t num_packets) : internal(filename, mode, num_packets) {}
 
    /**
      * Write a packet to the file