Reusable server class, rotation fixes, scene generator, ... (#83)

* Added generator command line tool for Serum PUP Scenes
* increased frame queue to handle PUP scenes in parallel to ROM animations
* Improved frame buffer handling
* Added HD support for RGB24DMDs
* Refactored dmdServer into a reusable library functionality (used by PPUC backbox)
* fixed Serum color rotation issues
* fixed some memory leaks
* fixed dumps overwrite existing dumps
* updated libs

Author: mkalkbrenner

.github/workflows/libdmdutil.yml

@@ -104,6 +104,7 @@ jobs:
              cp build/Release/dmdutil_static.lib tmp
              cp build/Release/dmdutil_test_s.exe tmp
              cp build/Release/dmdutil_test.exe tmp
+             cp build/Release/dmdutil-generate-scenes.exe tmp
              cp -r test tmp/
           else
             ARTIFACT_PATH="libdmdutil-${{ needs.version.outputs.tag }}-${{ matrix.platform }}-${{ matrix.arch }}.tar.gz"
@@ -114,13 +115,15 @@ jobs:
                cp build/dmdserver_test tmp
                cp build/dmdutil_test_s tmp
                cp build/dmdutil_test tmp
+               cp build/dmdutil-generate-scenes tmp
             elif [[ "${{ matrix.platform }}" == "linux" ]]; then
                cp build/libdmdutil.a tmp
                cp -a build/*.{so,so.*} tmp
                cp build/dmdserver tmp
                cp build/dmdserver_test tmp
                cp build/dmdutil_test_s tmp
                cp build/dmdutil_test tmp
+               cp build/dmdutil-generate-scenes tmp
             elif [[ "${{ matrix.platform }}" == "ios" || "${{ matrix.platform }}" == "ios-simulator" || "${{ matrix.platform }}" == "tvos" ]]; then
                cp build/libdmdutil.a tmp
                cp -a build/*.dylib tmp

.gitignore

@@ -13,7 +13,10 @@ third-party/include/cargs.h
 third-party/include/*.hpp
 third-party/include/libserialport.h
 third-party/include/pupdmd.h
+third-party/include/SceneGenerator.h
 third-party/include/serum.h
 third-party/include/serum-decode.h
 third-party/include/ZeDMD.h
+third-party/include/LZ4Stream.h
+third-party/include/lz4
 .DS_Store

CMakeLists.txt

@@ -6,13 +6,15 @@ set(ARCH "x64" CACHE STRING "Arch")
 option(BUILD_SHARED "Option to build shared library" ON)
 option(BUILD_STATIC "Option to build static library" ON)
 option(POST_BUILD_COPY_EXT_LIBS "Option to copy external libraries to build directory" ON)
+option(ENABLE_SANITIZERS "Enable AddressSanitizer and UBSan for Debug builds" OFF)
 
 message(STATUS "PLATFORM: ${PLATFORM}")
 message(STATUS "ARCH: ${ARCH}")
 
 message(STATUS "BUILD_SHARED: ${BUILD_SHARED}")
 message(STATUS "BUILD_STATIC: ${BUILD_STATIC}")
 message(STATUS "POST_BUILD_COPY_EXT_LIBS: ${POST_BUILD_COPY_EXT_LIBS}")
+message(STATUS "ENABLE_SANITIZERS: ${ENABLE_SANITIZERS}")
 
 if(PLATFORM STREQUAL "macos")
    set(CMAKE_OSX_DEPLOYMENT_TARGET 14.0)
@@ -67,6 +69,18 @@ set(CMAKE_C_STANDARD 99)
 set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 set(CMAKE_C_VISIBILITY_PRESET hidden)
 
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+   if(ENABLE_SANITIZERS AND (PLATFORM STREQUAL "macos" OR PLATFORM STREQUAL "linux"))
+      add_compile_options(-fsanitize=address)
+      add_link_options(-fsanitize=address)
+
+      add_compile_options(-fsanitize=undefined)
+      add_link_options(-fsanitize=undefined)
+
+      add_compile_options(-fno-omit-frame-pointer)
+   endif()
+endif()
+
 set(DMDUTIL_SOURCES
    src/Config.cpp
    src/DMD.cpp
@@ -75,6 +89,7 @@ set(DMDUTIL_SOURCES
    src/ConsoleDMD.cpp
    src/Logger.cpp
    src/AlphaNumeric.cpp
+   src/DMDServer.cpp
 )
 
 if(PLATFORM STREQUAL "win" OR PLATFORM STREQUAL "macos" OR PLATFORM STREQUAL "linux")
@@ -143,7 +158,7 @@ if(BUILD_SHARED)
 
    if(PLATFORM STREQUAL "win" OR PLATFORM STREQUAL "macos" OR PLATFORM STREQUAL "linux")
       add_executable(dmdserver
-         src/dmdServer.cpp
+         src/server.cpp
          src/Logger.cpp
       )
       target_link_libraries(dmdserver PUBLIC dmdutil_shared)
@@ -158,6 +173,11 @@ if(BUILD_SHARED)
       )
       target_link_libraries(dmdutil_test PUBLIC dmdutil_shared)
 
+      add_executable(dmdutil-generate-scenes
+         src/generateScenesDump.cpp
+      )
+      target_link_libraries(dmdutil-generate-scenes PUBLIC dmdutil_shared)
+
       if(POST_BUILD_COPY_EXT_LIBS)
          if(PLATFORM STREQUAL "win")
             if(ARCH STREQUAL "x64")

README.md

@@ -47,35 +47,17 @@ void test()
 Per default it listens on port 6789 on localhost and accepts "raw" TCP connections.
 
 `dmdserver` accepts these command line options:
-* -c --config
-    * Config file
-    * optional
-    * default is no config file
-* -o --alt-color-path
-    * "Fixed alt color path, overwriting paths transmitted by DMDUpdates
-    * optional
-* -a --addr
-    * IP address or host name
-    * optional
-    * default is `localhost`
-* -p --port
-    * Port
-    * optional
-    * default is `6789`
-* -w --wait-for-displays
-    * Don't terminate if no displays are connected
-    * optional
-    * default is to terminate the server process if no displays could be found
-* -l --logging
-    * Enable logging to stderr
-    * optional
-    * default is no logging
-* -v --verbose
-    * Enables verbose logging, includes normal logging
-    * optional
-    * default is no logging
-* -h --help
-    * Show help
+```
+  -c, --config=VALUE              Config file (optional, default is no config file)
+  -o, --alt-color-path=VALUE      Fixed alt color path, overwriting paths transmitted by DMDUpdates (optional)
+  -u, --pup-videos-path=VALUE     Fixed PupVideos path, overwriting paths transmitted by DMDUpdates (optional)
+  -a, --addr=VALUE                IP address or host name (optional, default is 'localhost')
+  -p, --port=VALUE                Port (optional, default is '6789')
+  -w, --wait-for-displays         Don't terminate if no displays are connected (optional, default is to terminate the server process if no displays could be found)
+  -l, --logging                   Enable logging to stderr (optional, default is no logging)
+  -v, --verbose-logging           Enables verbose logging, includes normal logging (optional, default is no logging)
+  -h, --help                      Show help
+```
 
 `dmdserver` expects two packages to render a DMD frame. The first one is a DmdStream header followed by the "data".
 
@@ -209,7 +191,7 @@ SaveSettings = 0
 # Set to 1 if ZeDMD-WiFi is available.
 Enabled = 0
 # Enter your ZeDMD WiFi IP address here
-WiFiAddr = 
+WiFiAddr =
 
 [Pixelcade]
 # Set to 1 if Pixelcade is attached
@@ -218,6 +200,24 @@ Enabled = 1
 Device =
 ```
 
+## Serum PUP Scenes Generator
+
+Serum PUP Scenes are a new feature of libserum_concentrate.
+Such scenes base on a defined number of fake frames at a given frame rate.
+These frames can be added to the ROM's original frames or overlay some original frames.
+The purpose is to add new animations or to replace existing frames with smoother aniations.
+Therefore, a colorization other needs a set of scene frames that could be colorized in the colorization editor.
+The `dmdutil-generate-scenes` tool generates a dump of such scene frames according to the PUP Scenes specification CSV file.
+
+`dmdutil-generate-scenes` accepts these command line options:
+```
+  -c, --csv=VALUE        PUP scenes CSV file to parse
+  -o, --output=VALUE     Output dump file to generate
+  -d, --depth=VALUE      Bit depth of the DMD frames (2 or 4) (optional, default is 2)
+  -i, --id=VALUE         PUP scene trigger ID to generate (optional, default is all scenes)
+  -h, --help             Show help
+```
+
 ## Building:
 
 #### Windows (x64)

dmdserver.ini

@@ -39,10 +39,10 @@ SaveSettings = 0
 # Set to 1 if ZeDMD-WiFi is available.
 Enabled = 0
 # Enter your ZeDMD WiFi IP address here
-WiFiAddr = 
+WiFiAddr =
 
 [Pixelcade]
 # Set to 1 if Pixelcade is attached
-Enabled = 1
+Enabled = 0
 # Disable auto-detection and provide a fixed serial port
 Device =

include/DMDUtil/DMD.h

@@ -8,9 +8,9 @@
 #define DMDUTILCALLBACK
 #endif
 
-#define DMDUTIL_FRAME_BUFFER_SIZE 32
+#define DMDUTIL_FRAME_BUFFER_SIZE 128  // Must be a divider of 65535! Must not exceed 256!
 #define DMDUTIL_MIN_FRAMES_BEHIND 4
-#define DMDUTIL_MAX_FRAMES_BEHIND 16
+#define DMDUTIL_MAX_FRAMES_BEHIND 32
 #define DMDUTIL_MAX_NAME_SIZE 16
 #define DMDUTIL_MAX_PATH_SIZE 256
 #define DMDUTIL_MAX_TRANSITIONAL_FRAME_DURATION 25
@@ -68,7 +68,6 @@ class PixelcadeDMD;
 class LevelDMD;
 class RGB24DMD;
 class ConsoleDMD;
-
 class DMDServerConnector;
 
 class DMDUTILAPI DMD
@@ -77,7 +76,7 @@ class DMDUTILAPI DMD
   DMD();
   ~DMD();
 
-  enum class Mode
+  enum class Mode : int
   {
     Unknown = 0,
     Data = 1,
@@ -108,20 +107,20 @@ class DMDUTILAPI DMD
 #pragma pack(push, 1)  // Align to 1-byte boundaries, important for sending over socket.
   struct Update
   {
-    Mode mode;
-    AlphaNumericLayout layout;
-    int depth;
-    uint8_t data[256 * 64 * 3];
-    uint16_t segData[256 * 64];  // RGB16 or segment data or SerumV1 palette
+    Mode mode = Mode::Data;                                    // int
+    AlphaNumericLayout layout = AlphaNumericLayout::NoLayout;  // int
+    int depth = 2;
+    uint8_t data[256 * 64 * 3] = {0};
+    uint16_t segData[256 * 64] = {0};  // RGB16 or segment data or SerumV1 palette
     uint16_t segData2[128];
-    bool hasData;
-    bool hasSegData;
-    bool hasSegData2;
-    uint8_t r;
-    uint8_t g;
-    uint8_t b;
-    uint16_t width;
-    uint16_t height;
+    bool hasData = false;
+    bool hasSegData = false;
+    bool hasSegData2 = false;
+    uint8_t r = 255;
+    uint8_t g = 255;
+    uint8_t b = 255;
+    uint16_t width = 128;
+    uint16_t height = 32;
 
     DMDUTILAPI void convertToHostByteOrder();
     DMDUTILAPI Update toNetworkByteOrder() const;
@@ -161,6 +160,7 @@ class DMDUTILAPI DMD
   void SetRomName(const char* name);
   void SetAltColorPath(const char* path);
   void SetPUPVideosPath(const char* path);
+  void SetPUPTrigger(const char source, const uint16_t id, const uint8_t value = 1);
   void DumpDMDTxt();
   void DumpDMDRaw();
   LevelDMD* CreateLevelDMD(uint16_t width, uint16_t height, bool sam);
@@ -185,7 +185,7 @@ class DMDUTILAPI DMD
   Update* m_pUpdateBufferQueue[DMDUTIL_FRAME_BUFFER_SIZE];
   std::shared_ptr<Update> m_updateBuffered;
 
-  uint8_t GetNextBufferQueuePosition(uint8_t bufferPosition, const uint8_t updateBufferQueuePosition);
+  uint16_t GetNextBufferQueuePosition(uint16_t bufferPosition, const uint16_t updateBufferQueuePosition);
   bool ConnectDMDServer();
   bool UpdatePalette(uint8_t* pPalette, uint8_t depth, uint8_t r, uint8_t g, uint8_t b);
   void UpdateData(const uint8_t* pData, int depth, uint16_t width, uint16_t height, uint8_t r, uint8_t g, uint8_t b,
@@ -232,7 +232,8 @@ class DMDUTILAPI DMD
   std::condition_variable_any m_dmdCV;
   std::atomic<bool> m_dmdFrameReady;
   std::atomic<bool> m_stopFlag;
-  std::atomic<uint8_t> m_updateBufferQueuePosition;
+  std::atomic<uint16_t> m_updateBufferQueuePosition;
+  std::atomic<uint16_t> m_pupSceneId;
 
   bool m_hasUpdateBuffered = false;
   static bool m_finding;

include/DMDUtil/DMDServer.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <atomic>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+#include "DMDUtil/DMD.h"
+#include "sockpp/tcp_acceptor.h"
+
+#define DMDSERVER_MAX_WIDTH 256
+#define DMDSERVER_MAX_HEIGHT 64
+
+namespace DMDUtil
+{
+
+class DMDUTILAPI DMDServer
+{
+ public:
+  DMDServer(DMD* dmd, bool fixedAltColorPath = false, bool fixedPupPath = false);
+  ~DMDServer();
+
+  bool Start(const char* addr, in_port_t port);
+  void Stop();
+  bool IsRunning() const { return m_running.load(std::memory_order_acquire); }
+
+ private:
+  void AcceptLoop();
+  void ClientThread(sockpp::tcp_socket sock, uint32_t threadId);
+
+  DMD* m_dmd;
+  bool m_fixedAltColorPath;
+  bool m_fixedPupPath;
+  std::atomic<bool> m_running{false};
+  sockpp::tcp_acceptor m_acceptor;
+
+  uint32_t m_currentThreadId{0};
+  std::mutex m_threadMutex;
+  uint32_t m_disconnectOtherClients{0};
+  std::vector<uint32_t> m_threads;
+  std::thread* m_acceptThread{nullptr};
+};
+
+}  // namespace DMDUtil

include/DMDUtil/DMDUtil.h

@@ -1,8 +1,8 @@
 #pragma once
 
-#define DMDUTIL_VERSION_MAJOR 0  // X Digits
-#define DMDUTIL_VERSION_MINOR 9  // Max 2 Digits
-#define DMDUTIL_VERSION_PATCH 2  // Max 2 Digits
+#define DMDUTIL_VERSION_MAJOR 0   // X Digits
+#define DMDUTIL_VERSION_MINOR 10  // Max 2 Digits
+#define DMDUTIL_VERSION_PATCH 0   // Max 2 Digits
 
 #define _DMDUTIL_STR(x) #x
 #define DMDUTIL_STR(x) _DMDUTIL_STR(x)
@@ -14,5 +14,6 @@
 #include "Config.h"
 #include "ConsoleDMD.h"
 #include "DMD.h"
+#include "DMDServer.h"
 #include "LevelDMD.h"
 #include "RGB24DMD.h"

include/DMDUtil/RGB24DMD.h

@@ -19,7 +19,7 @@ class DMDUTILAPI RGB24DMD
   RGB24DMD(uint16_t width, uint16_t height);
   ~RGB24DMD();
 
-  virtual void Update(uint8_t* pRGB24Data);
+  virtual void Update(uint8_t* pRGB24Data, uint16_t width = 0, uint16_t height = 0);
   int GetWidth() const { return m_width; }
   int GetHeight() const { return m_height; }
   int GetLength() const { return m_length; }

platforms/android/arm64-v8a/external.sh

@@ -64,6 +64,9 @@ cmake \
    -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
    -B build
 cmake --build build -- -j${NUM_PROCS}
+cp -r third-party/include/lz4 ../../third-party/include/
+cp src/LZ4Stream.h ../../third-party/include/
+cp src/SceneGenerator.h ../../third-party/include/
 cp src/serum.h ../../third-party/include/
 cp src/serum-decode.h ../../third-party/include/
 cp build/libserum.so ../../third-party/runtime-libs/android/arm64-v8a/