diff --git a/.github/workflows/build_native.yml b/.github/workflows/build_native.yml index d052ddb1..6e7e119e 100644 --- a/.github/workflows/build_native.yml +++ b/.github/workflows/build_native.yml @@ -89,10 +89,10 @@ jobs: ARCHES="$ARCH_INPUT" fi - # On develop branch, limit to amd64 for faster feedback - if [ "${{ github.ref }}" = "refs/heads/develop" ]; then + # On develop branch, limit to amd64 for faster feedback (only for automatic pushes, not manual workflow_dispatch) + if [ "${{ github.ref }}" = "refs/heads/develop" ] && [ "${{ github.event_name }}" != "workflow_dispatch" ]; then ARCHES="amd64" - echo "Building only amd64 for develop branch" + echo "Building only amd64 for develop branch (automatic push)" fi # Expand distributions list @@ -208,6 +208,15 @@ jobs: run: | echo "Building OpenAuto for ${{ matrix.arch }} architecture" + # Determine build type based on branch + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + BUILD_TYPE="release" + echo "Building release packages for main branch" + else + BUILD_TYPE="debug" + echo "Building debug packages for ${{ github.ref_name }} branch" + fi + # Create output directory mkdir -p ./build-output @@ -216,6 +225,7 @@ jobs: --platform ${{ matrix.platform }} \ --build-arg TARGET_ARCH=${{ matrix.arch }} \ --build-arg DEBIAN_VERSION=${{ matrix.distro }} \ + --build-arg BUILD_TYPE=${BUILD_TYPE} \ --tag openauto-build:${{ matrix.arch }} \ --load \ . diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f968db1..c99a9702 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,7 +264,14 @@ else() endif() set(CPACK_GENERATOR "DEB") -set(CPACK_PACKAGE_NAME "openauto") + +# Add debug suffix to package name for debug builds +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CPACK_PACKAGE_NAME "openauto-dbg") +else() + set(CPACK_PACKAGE_NAME "openauto") +endif() + set(CPACK_PACKAGE_VENDOR "OpenCarDev") set(CPACK_PACKAGE_CONTACT "OpenCarDev Team ") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "OpenAuto head unit application using libaasdk") diff --git a/Dockerfile b/Dockerfile index b42c96de..8e36b1e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ FROM debian:${DEBIAN_VERSION}-slim # Build arguments ARG TARGET_ARCH=amd64 +ARG BUILD_TYPE=release ARG DEBIAN_FRONTEND=noninteractive # Set locale to avoid encoding issues @@ -105,40 +106,11 @@ RUN echo "Contents of /src:" && ls -la # Create output directory for packages RUN mkdir -p /output -# Build OpenAuto -RUN export TARGET_ARCH=$(dpkg-architecture -qDEB_HOST_ARCH) && \ - echo "Building OpenAuto for architecture: $TARGET_ARCH (native compilation)" && \ - # Determine if this is a non-Pi architecture - CMAKE_NOPI_FLAG="" && \ - case "$TARGET_ARCH" in \ - amd64|arm64) CMAKE_NOPI_FLAG="-DNOPI=ON" ;; \ - armhf) CMAKE_NOPI_FLAG="-DNOPI=ON" ;; \ - *) CMAKE_NOPI_FLAG="-DNOPI=ON" ;; \ - esac && \ - echo "Using CMAKE_NOPI_FLAG: $CMAKE_NOPI_FLAG" && \ - # Compute distro-specific release suffix to avoid cross-suite overwrite - DISTRO_DEB_RELEASE=$(bash /src/scripts/distro_release.sh) && \ - CPACK_DEB_RELEASE="$DISTRO_DEB_RELEASE" && \ - echo "Using CPACK_DEBIAN_PACKAGE_RELEASE: $CPACK_DEB_RELEASE" && \ - # Configure - env DISTRO_DEB_RELEASE="$CPACK_DEB_RELEASE" \ - cmake -S . -B build -DCMAKE_BUILD_TYPE=Release ${CMAKE_NOPI_FLAG} -DCPACK_DEBIAN_PACKAGE_RELEASE="$CPACK_DEB_RELEASE" -DCPACK_PROJECT_CONFIG_FILE=/src/cmake_modules/CPackProjectConfig.cmake && \ - # Build - cmake --build build -j$(nproc) && \ - # Package - cd build && \ - cpack -G DEB && \ - cd .. && \ - # Copy packages to output - if [ -d "build" ]; then \ - find build -name "*.deb" -exec cp {} /output/ \; 2>/dev/null || true && \ - echo "Packages built:" && \ - ls -la /output/; \ - else \ - echo "No build directory found"; \ - exit 1; \ - fi && \ - echo "Build completed" +# Make build script executable +RUN chmod +x /src/build.sh + +# Build OpenAuto using unified build script +RUN /src/build.sh ${BUILD_TYPE} --package --output-dir /output # Default command CMD ["bash", "-c", "echo 'OpenAuto build container ready. Packages are in /output/'"] diff --git a/README.md b/README.md index d73f20f8..95f83c0d 100644 --- a/README.md +++ b/README.md @@ -50,23 +50,76 @@ Copyrights (c) 2018 f1x.studio (Michal Szwaj) - OpenMAX IL API ### Building -#### Amd64 -Install the packages specified in the [prebuilts](https://github.com/opencardev/prebuilts) repository. Qt5 is required, versions packaged in modern Ubuntu and Debian -seem to work fine. - -You will also likely need to install the udev rules from `prebuilts` - -You need to point some CMAKE variables at your `aasdk` files. -```text --DAASDK_INCLUDE_DIRS=/include --DAASDK_LIBRARIES=/lib/libaasdk.so - DAASDK_PROTO_INCLUDE_DIRS= --DAASDK_PROTO_LIBRARIES=/lib/libaasdk_proto.so + +OpenAuto provides a unified build script (`build.sh`) that works consistently across local and Docker environments. + +#### Quick Start + +```bash +# Build release version (recommended for production) +# Note: main/master branches default to release, other branches default to debug +./build.sh release --package + +# Build debug version with symbols (for development/debugging) +# Debug builds create packages with -dbg suffix (openauto-dbg) +./build.sh debug --package + +# Auto-detect build type based on git branch +./build.sh --package + +# Clean build +./build.sh release --clean --package ``` -#### Raspberry Pi -Just run the scripts in the `prebuilts` repository for `aasdk` and `openauto`. It is possible to cross compile if your raspberry pi is too slow to compile the code itself. -However, its easiest to just develop on a more capable `amd64` device. +#### Build Script Options + +```bash +Usage: ./build.sh [release|debug] [OPTIONS] + +Build types: + release Build release version (default) + debug Build debug version with symbols + +Options: + --clean Clean build directory before building + --package Create DEB packages after building + --output-dir Directory to copy packages (default: /output) + --help Show help message + +Note: Builds are always done with NOPI=ON (no Pi-specific hardware) +``` + +#### Manual Building (Legacy) + +If you need to build manually: + +**AMD64/x86_64:** +1. Install dependencies from [prebuilts](https://github.com/opencardev/prebuilts) repository +2. Install Qt5 and development packages +3. Build with CMake: + ```bash + mkdir -p build-release + cd build-release + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j$(nproc) + ``` + + Note: The build script automatically enables NOPI=ON + +**Raspberry Pi:** +1. Use the scripts in the `prebuilts` repository for `aasdk` and `openauto` +2. Or use the unified `build.sh` script: + ```bash + ./build.sh release --package + ``` + +#### Docker Building + +The Dockerfile uses the same `build.sh` script: + +```bash +docker build -t openauto --build-arg DEBIAN_VERSION=trixie . +``` ### Remarks **This software is not certified by Google Inc. It is created for R&D purposes and may not work as expected by the original authors. Do not use while driving. You use this software at your own risk.** diff --git a/build.sh b/build.sh new file mode 100644 index 00000000..784b9e99 --- /dev/null +++ b/build.sh @@ -0,0 +1,203 @@ +#!/bin/bash +# * Project: OpenAuto +# * This file is part of openauto project. +# * Copyright (C) 2025 OpenCarDev Team +# * +# * openauto is free software: you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * openauto is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with openauto. If not, see . + +set -e + +# Script to build OpenAuto consistently across Docker and local environments +# Usage: ./build.sh [release|debug] [--clean] [--package] [--output-dir DIR] + +# Default values +NOPI_FLAG="-DNOPI=ON" +CLEAN_BUILD=false +PACKAGE=false +OUTPUT_DIR="/output" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_DIR="${SCRIPT_DIR}" + +# Auto-detect build type based on git branch +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") +if [ "$CURRENT_BRANCH" = "main" ] || [ "$CURRENT_BRANCH" = "master" ]; then + BUILD_TYPE="release" +else + BUILD_TYPE="debug" +fi + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + release|Release|RELEASE) + BUILD_TYPE="release" + shift + ;; + debug|Debug|DEBUG) + BUILD_TYPE="debug" + shift + ;; + --clean) + CLEAN_BUILD=true + shift + ;; + --package) + PACKAGE=true + shift + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [release|debug] [OPTIONS]" + echo "" + echo "Build types:" + echo " release Build release version (default)" + echo " debug Build debug version with symbols" + echo "" + echo "Options:" + echo " --clean Clean build directory before building" + echo " --package Create DEB packages after building" + echo " --output-dir Directory to copy packages (default: /output)" + echo " --help Show this help message" + echo "" + echo "Examples:" + echo " $0 release --package" + echo " $0 debug --clean" + echo " $0 release --package --output-dir ./packages" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Determine build directory and CMake build type +if [ "$BUILD_TYPE" = "debug" ]; then + BUILD_DIR="${SOURCE_DIR}/build-debug" + CMAKE_BUILD_TYPE="Debug" + CMAKE_CXX_FLAGS="-g3 -O0" + echo "=== Building OpenAuto (Debug) ===" +else + BUILD_DIR="${SOURCE_DIR}/build-release" + CMAKE_BUILD_TYPE="Release" + CMAKE_CXX_FLAGS="" + echo "=== Building OpenAuto (Release) ===" +fi + +echo "Source directory: ${SOURCE_DIR}" +echo "Build directory: ${BUILD_DIR}" +echo "Build type: ${CMAKE_BUILD_TYPE}" +echo "NOPI: ON (no Pi hardware dependencies)" +echo "Package: ${PACKAGE}" + +# Clean build directory if requested +if [ "$CLEAN_BUILD" = true ]; then + echo "" + echo "Cleaning build directory..." + rm -rf "${BUILD_DIR}" +fi + +# Create build directory +mkdir -p "${BUILD_DIR}" + +# Detect architecture +TARGET_ARCH=$(dpkg-architecture -qDEB_HOST_ARCH 2>/dev/null || echo "amd64") +echo "Target architecture: ${TARGET_ARCH}" + +# Compute distro-specific release suffix +if [ -f "${SOURCE_DIR}/scripts/distro_release.sh" ]; then + DISTRO_DEB_RELEASE=$(bash "${SOURCE_DIR}/scripts/distro_release.sh") + echo "Distro release suffix: ${DISTRO_DEB_RELEASE}" +else + DISTRO_DEB_RELEASE="" + echo "Warning: distro_release.sh not found, using default release suffix" +fi + +# Configure CMake +echo "" +echo "Configuring with CMake..." +CMAKE_ARGS=( + -S "${SOURCE_DIR}" + -B "${BUILD_DIR}" + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" +) + +if [ -n "$CMAKE_CXX_FLAGS" ]; then + CMAKE_ARGS+=(-DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}") +fi + +if [ -n "$NOPI_FLAG" ]; then + CMAKE_ARGS+=("${NOPI_FLAG}") +fi + +if [ -n "$DISTRO_DEB_RELEASE" ]; then + CMAKE_ARGS+=(-DCPACK_DEBIAN_PACKAGE_RELEASE="${DISTRO_DEB_RELEASE}") +fi + +CMAKE_ARGS+=(-DCPACK_PROJECT_CONFIG_FILE="${SOURCE_DIR}/cmake_modules/CPackProjectConfig.cmake") + +# Run CMake configuration +env DISTRO_DEB_RELEASE="${DISTRO_DEB_RELEASE}" cmake "${CMAKE_ARGS[@]}" + +# Build +echo "" +echo "Building..." +NUM_CORES=$(nproc 2>/dev/null || echo 4) +cmake --build "${BUILD_DIR}" -j"${NUM_CORES}" + +echo "" +echo "✓ Build completed successfully" + +# Package if requested +if [ "$PACKAGE" = true ]; then + echo "" + echo "Creating packages..." + cd "${BUILD_DIR}" + cpack -G DEB + cd "${SOURCE_DIR}" + + # Copy packages to output directory + if [ -n "$OUTPUT_DIR" ] && [ "$OUTPUT_DIR" != "${BUILD_DIR}" ]; then + echo "" + echo "Copying packages to ${OUTPUT_DIR}..." + mkdir -p "${OUTPUT_DIR}" + find "${BUILD_DIR}" -name "*.deb" -exec cp -v {} "${OUTPUT_DIR}/" \; + echo "" + echo "Packages in ${OUTPUT_DIR}:" + ls -lh "${OUTPUT_DIR}"/*.deb 2>/dev/null || echo "No packages found" + else + echo "" + echo "Packages in ${BUILD_DIR}:" + find "${BUILD_DIR}" -name "*.deb" -ls + fi +fi + +echo "" +echo "=== Build Summary ===" +echo "Build type: ${CMAKE_BUILD_TYPE}" +echo "Build directory: ${BUILD_DIR}" +if [ -f "${BUILD_DIR}/autoapp" ]; then + echo "Binary: ${BUILD_DIR}/autoapp" +fi +if [ -f "${BUILD_DIR}/btservice" ]; then + echo "Binary: ${BUILD_DIR}/btservice" +fi + +echo "" +echo "Done!" diff --git a/docs/troubleshooting/DEBUG_QUICKREF.md b/docs/troubleshooting/DEBUG_QUICKREF.md new file mode 100644 index 00000000..86c12293 --- /dev/null +++ b/docs/troubleshooting/DEBUG_QUICKREF.md @@ -0,0 +1,195 @@ +# Quick Reference: Remote Debugging OpenAuto + +## Initial Setup (One-time) + +### On Windows: +```powershell +cd C:\Users\matth\install\repos\opencardev\openauto\scripts +.\setup-remote-debug.ps1 +``` + +### On Raspberry Pi: +```bash +chmod +x /tmp/setup-remote-debug.sh +/tmp/setup-remote-debug.sh +``` + +Choose option 1: "Start autoapp under gdbserver" (recommended) + +## Starting a Debug Session + +### 1. In VS Code: +- Press `F5` +- Select: **"Remote Debug on rpi5 (Launch)"** + +### 2. Set Key Breakpoints: + +**SensorService.cpp:** +``` +Line ~59: stopPolling.store(true, ...) // In stop() +Line ~226: if (!stopPolling.load(...)) // In sensorPolling() +Line ~275: void SensorService::onChannelError // Error handler +``` + +**AndroidAutoEntity.cpp:** +``` +Line ~64: void AndroidAutoEntity::stop() // Main stop +Line ~320: void AndroidAutoEntity::onChannelError // Error handler +``` + +### 3. Reproduce the Issue: +1. Let debugger connect and app start +2. Connect Android Auto normally +3. Click "Exit" in Android Auto +4. **Watch the breakpoints** + +## When It Hangs + +### Pause and Inspect: +1. Click **Pause** button (or `Ctrl+Shift+F5`) +2. Open **Call Stack** panel (View > Call Stack) +3. Look at all threads - which are active? + +### Key Things to Check: + +**In Variables Panel:** +- `this->stopPolling` - what's its value? +- `this->gpsEnabled_` - is it true? +- `timer_` state + +**In Call Stack:** +- Is `sensorPolling()` still in the stack? +- Multiple threads showing sensor code? +- Any threads blocked on mutexes? + +### Debug Console Commands: + +```gdb +# List all threads +info threads + +# Show all thread backtraces +thread apply all bt + +# Check stopPolling value +p this->stopPolling + +# Check if timer is active +p timer_ + +# See what thread 2 is doing +thread 2 +bt + +# Continue execution +continue +``` + +## Common Issues Found + +### Issue 1: stopPolling Not Being Checked +**Symptom:** `sensorPolling()` continues after `stop()` called +**Check:** Is the atomic load being used? Are there queued callbacks? + +### Issue 2: Timer Not Cancelled +**Symptom:** New timer callbacks scheduled after `stop()` +**Check:** Was `timer_.cancel()` called? Check error code. + +### Issue 3: Channel Error Not Stopping Service +**Symptom:** Polling continues after channel error +**Check:** Does `onChannelError()` call `stop()`? + +### Issue 4: Race Condition +**Symptom:** Sometimes works, sometimes hangs +**Check:** Thread timing between `stop()` and `sensorPolling()` + +## Manual Debugging (SSH) + +### Connect to Pi: +```bash +ssh pi@rpi5.home.lan +``` + +### Check if process is hung: +```bash +# Find process +ps aux | grep autoapp + +# Check threads +ps -eLf | grep autoapp + +# See what it's doing +sudo strace -p PID -f + +# Get thread dump +sudo gdb -p PID -batch -ex "thread apply all bt" +``` + +### Stop hung process: +```bash +sudo systemctl stop openauto +# or +sudo killall -9 autoapp +``` + +### Start fresh debug session: +```bash +sudo gdbserver :2345 /path/to/autoapp +``` + +## Testing the Fix + +After making code changes: + +1. **Rebuild on Pi:** + ```bash + cd ~/src/openauto + # Quick rebuild (if CMake already configured) + cmake --build build-debug -j$(nproc) + + # Or full rebuild with build script + ./build.sh debug + ``` + +2. **Start new debug session:** + ```bash + sudo gdbserver :2345 ./autoapp + ``` + +3. **In VS Code:** Press F5 again + +4. **Test exit multiple times:** + - Start Android Auto + - Click Exit + - Check logs for clean shutdown + - Verify no timeout/hang + - Repeat 3-5 times + +## Expected Behavior (Fixed) + +When you click "Exit" in Android Auto: + +1. Channel errors logged (expected) +2. `onChannelError()` called +3. `stop()` called +4. `stopPolling` set to true +5. Timer cancelled +6. No more `sensorPolling()` logs +7. Clean shutdown in ~1 second +8. Service stops without SIGKILL + +## Quick Tips + +- **Lost connection?** Check Pi is still on: `ping rpi5.home.lan` +- **Can't set breakpoints?** Check source file paths in launch.json +- **Symbols not loading?** Verify debug build: `file ~/src/openauto/build-debug/autoapp` +- **Need to restart?** Kill gdbserver on Pi: `sudo pkill gdbserver` +- **Want clean slate?** Stop service first: `sudo systemctl stop openauto` + +## File Locations + +- **Windows Scripts:** `C:\Users\matth\install\repos\opencardev\openauto\scripts\` +- **VS Code Config:** `C:\Users\matth\install\repos\opencardev\openauto\.vscode\launch.json` +- **Pi Source:** `~/src/openauto/` +- **Pi Binary (debug):** `~/src/openauto/build-debug/autoapp` +- **Pi Binary (installed):** `/opt/crankshaft/autoapp` diff --git a/docs/troubleshooting/remote-debugging.md b/docs/troubleshooting/remote-debugging.md new file mode 100644 index 00000000..03943d71 --- /dev/null +++ b/docs/troubleshooting/remote-debugging.md @@ -0,0 +1,472 @@ +# Remote Debugging OpenAuto on Raspberry Pi + +This guide explains how to debug the OpenAuto application running on a remote Raspberry Pi from your development machine using GDB and VS Code. + +## Prerequisites + +- SSH access to the Raspberry Pi +- GDB installed on both development machine and Raspberry Pi +- OpenAuto built with debug symbols (`-DCMAKE_BUILD_TYPE=Debug`) +- VS Code with C/C++ extension installed + +## Setup on Raspberry Pi + +### 1. Install Required Packages + +```bash +sudo apt-get update +sudo apt-get install -y \ + gdbserver \ + git \ + cmake \ + build-essential \ + pkg-config +``` + +### 2. Clone OpenAuto Repository (if not already present) + +```bash +# Create source directory +mkdir -p ~/src +cd ~/src + +# Clone the repository +git clone https://github.com/opencardev/openauto.git +cd openauto + +# Switch to develop branch +git checkout develop +``` + +### 3. Install Build Dependencies + +```bash +# Install aasdk dependency +sudo apt-get install -y libaasdk-dev + +# Or if building aasdk from source: +cd ~/src +git clone https://github.com/opencardev/aasdk.git +cd aasdk +mkdir -p build-release +cd build-release +cmake -DCMAKE_BUILD_TYPE=Release .. +make -j$(nproc) +sudo make install + +# Install other dependencies +sudo apt-get install -y \ + libboost-all-dev \ + libusb-1.0-0-dev \ + libssl-dev \ + libprotobuf-dev \ + protobuf-compiler \ + libqt5multimedia5 \ + libqt5multimedia5-plugins \ + libqt5multimediawidgets5 \ + qtmultimedia5-dev \ + libqt5bluetooth5 \ + libqt5bluetooth5-bin \ + qtconnectivity5-dev \ + pulseaudio \ + librtaudio-dev \ + libgstreamer1.0-dev \ + libgstreamer-plugins-base1.0-dev \ + gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-ugly \ + gstreamer1.0-libav \ + gstreamer1.0-alsa \ + libgps-dev \ + gpsd \ + libblkid-dev \ + libtag1-dev +``` + +### 4. Build OpenAuto with Debug Symbols + +```bash +cd ~/src/openauto + +# Build with debug symbols using the unified build script +./build.sh debug + +# The binary will be in build-debug/autoapp +# Note: Builds are done with NOPI=ON by default +# Optionally install +cd build-debug +sudo make install +``` + +### 3. Stop the Running Service + +```bash +sudo systemctl stop openauto +``` + +### 4. Start Application with GDB Server + +**Option A: Attach to Running Process** + +```bash +# Start the application normally +sudo /path/to/autoapp & + +# Find the process ID +ps aux | grep autoapp + +# Attach gdbserver (replace PID with actual process ID) +sudo gdbserver --attach :2345 PID +``` + +**Option B: Start Application Under GDB Server** + +```bash +sudo gdbserver :2345 /opt/crankshaft/autoapp +``` + +The application will wait for the debugger to connect before starting. + +## Setup on Development Machine (Windows) + +### 1. Install Required Tools + +**Install GDB for Windows:** + +Download from: https://www.msys2.org/ + +```powershell +winget install -e --id Codeblocks.Codeblocks +``` + +Or use WSL: + +```bash +sudo apt-get install gdb-multiarch +``` + +### 2. Configure VS Code + +Create or update `.vscode/launch.json` in your openauto workspace: + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Remote Debug OpenAuto", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/bin/autoapp", + "args": [], + "stopAtEntry": false, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ], + "miDebuggerServerAddress": "rpi5.home.lan:2345", + "sourceFileMap": { + "/opt/openauto": "${workspaceFolder}", + "/usr/src/openauto": "${workspaceFolder}" + } + }, + { + "name": "Attach to Remote Process", + "type": "cppdbg", + "request": "attach", + "program": "${workspaceFolder}/bin/autoapp", + "processId": "${command:pickRemoteProcess}", + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "miDebuggerServerAddress": "rpi5.home.lan:2345", + "sourceFileMap": { + "/opt/openauto": "${workspaceFolder}", + "/usr/src/openauto": "${workspaceFolder}" + } + } + ] +} +``` + +### 3. Set Up SSH Port Forwarding (Optional) + +If direct connection doesn't work, use SSH tunneling: + +```powershell +ssh -L 2345:localhost:2345 pi@rpi5.home.lan +``` + +Keep this terminal open while debugging. + +## Debugging the Shutdown Hang + +### 1. Set Breakpoints + +In VS Code, set breakpoints in key locations: + +- `src/autoapp/Service/Sensor/SensorService.cpp`: + - Line in `stop()` method + - Line in `sensorPolling()` method + - Line in `onChannelError()` method + +- `src/autoapp/Service/AndroidAutoEntity.cpp`: + - Line in `stop()` method + - Line in `onChannelError()` method + +- `src/autoapp/App.cpp`: + - Line in `onAndroidAutoQuit()` method + +### 2. Start Debugging Session + +1. Press `F5` or select "Run > Start Debugging" +2. Choose "Remote Debug OpenAuto" configuration +3. Wait for connection to establish + +### 3. Reproduce the Hang + +1. Use Android Auto normally +2. Click "Exit" in Android Auto +3. Observe which breakpoints are hit +4. Check the call stack and variable states + +### 4. Inspect Thread State During Hang + +If the application hangs: + +1. Press the pause button in VS Code +2. Open the "Call Stack" panel to see all threads +3. Look for threads that are blocked or in infinite loops +4. Inspect local variables and member variables + +**Key things to check:** + +- `SensorService::stopPolling` value +- Timer state (`timer_.expires_at()`) +- Thread IDs and what they're waiting on +- Strand queue contents (if visible) +- io_service state + +### 5. Get Thread Backtrace + +In the Debug Console: + +```gdb +info threads +thread apply all bt +``` + +This shows all threads and their call stacks. + +## Common Debugging Commands + +### GDB Commands in VS Code Debug Console + +```gdb +# List all threads +info threads + +# Switch to thread N +thread N + +# Show backtrace for current thread +bt + +# Show backtrace for all threads +thread apply all bt + +# Print variable value +p variableName + +# Print with more detail +p /x variableName + +# Continue execution +continue + +# Step over +next + +# Step into +step + +# Break on function +break SensorService::stop + +# List breakpoints +info breakpoints +``` + +### Analyzing Deadlocks + +```gdb +# Check if threads are waiting on mutexes +thread apply all bt + +# Look for patterns like: +# - pthread_mutex_lock +# - boost::asio::detail::scheduler::run +# - std::condition_variable::wait +``` + +## SSH Commands for Manual Inspection + +### Check Process State + +```bash +# Find process +ps aux | grep autoapp + +# Check thread count +ps -eLf | grep autoapp + +# Get detailed thread info +cat /proc/PID/status + +# Check open file descriptors +lsof -p PID +``` + +### Attach GDB Manually + +```bash +# Attach to running process +sudo gdb -p PID + +# Inside GDB +(gdb) info threads +(gdb) thread apply all bt +(gdb) continue +``` + +### Monitor System Calls + +```bash +# See what system calls the process is making +sudo strace -p PID -f + +# Focus on timing-related calls +sudo strace -p PID -f -e trace=futex,poll,select,epoll_wait +``` + +## Troubleshooting + +### Cannot Connect to Remote Debugger + +**Check firewall:** + +```bash +# On Raspberry Pi +sudo ufw allow 2345/tcp +``` + +**Check gdbserver is running:** + +```bash +ps aux | grep gdbserver +``` + +**Test connectivity:** + +```powershell +Test-NetConnection -ComputerName rpi5.home.lan -Port 2345 +``` + +### Source File Mapping Issues + +If breakpoints show "Breakpoint in file that does not exist": + +1. Check `sourceFileMap` in `launch.json` +2. Verify source paths on Pi: `readlink -f /proc/PID/cwd` +3. Update mapping to match actual build paths + +### Symbol Loading Issues + +```bash +# On Pi, check if debug symbols are present +file /path/to/autoapp +# Should show "not stripped" + +# If stripped, rebuild with debug symbols +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-g3" .. +``` + +## Example: Debugging the SensorService Hang + +### Scenario: Application hangs on exit + +**Steps:** + +1. Start gdbserver on Pi: + ```bash + sudo systemctl stop openauto + sudo gdbserver :2345 /opt/crankshaft/autoapp + ``` + +2. In VS Code: + - Set breakpoint in `SensorService::stop()` at line where `stopPolling` is set + - Set breakpoint in `SensorService::sensorPolling()` where it checks `stopPolling` + - Press F5 to start debugging + +3. Reproduce the issue: + - Use Android Auto + - Click Exit + - Watch breakpoints + +4. When hung: + - Pause execution + - Check "Call Stack" for all threads + - Look for `sensorPolling` in any thread + - Inspect `stopPolling` value in Variables panel + - Check if timer is cancelled + +5. Expected findings: + - If `sensorPolling` is still running after `stop()`, there's a race condition + - If multiple threads show `sensorPolling`, timer wasn't properly cancelled + - If thread is in `timer_.async_wait`, the timer callback is pending + +## Advanced: Core Dump Analysis + +If the process gets killed, analyze the core dump: + +```bash +# Enable core dumps +ulimit -c unlimited + +# Generate core dump when killed +sudo gdb /opt/crankshaft/autoapp core + +# Or manually generate +sudo gcore PID +sudo gdb /opt/crankshaft/autoapp core.PID +``` + +## Performance Profiling + +To identify slow operations during shutdown: + +```bash +# Use perf to profile +sudo perf record -p PID -g +# Trigger the hang +# Press Ctrl+C +sudo perf report +``` + +## References + +- [GDB Remote Debugging Documentation](https://sourceware.org/gdb/onlinedocs/gdb/Remote-Debugging.html) +- [VS Code C++ Debugging](https://code.visualstudio.com/docs/cpp/cpp-debug) +- [Linux Process Debugging](https://www.kernel.org/doc/html/latest/admin-guide/README.html) diff --git a/scripts/setup-remote-debug.ps1 b/scripts/setup-remote-debug.ps1 new file mode 100644 index 00000000..41b196dc --- /dev/null +++ b/scripts/setup-remote-debug.ps1 @@ -0,0 +1,135 @@ +# PowerShell script to set up remote debugging from Windows +# Run this on your Windows machine + +param( + [string]$RpiHost = "rpi5.home.lan", + [int]$DebugPort = 2345, + [switch]$SetupTunnel, + [switch]$TestConnection +) + +Write-Host "=== OpenAuto Remote Debugging Setup (Windows) ===" -ForegroundColor Cyan +Write-Host "" + +# Test connection to Pi +Write-Host "Testing connection to $RpiHost..." -ForegroundColor Yellow +$pingResult = Test-Connection -ComputerName $RpiHost -Count 1 -Quiet + +if (-not $pingResult) { + Write-Host "[X] Cannot reach $RpiHost" -ForegroundColor Red + Write-Host " Check that the Pi is on and network is configured" -ForegroundColor Red + exit 1 +} +Write-Host "[OK] Pi is reachable" -ForegroundColor Green + +# Test SSH connection +Write-Host "" +Write-Host "Testing SSH connection..." -ForegroundColor Yellow +$null = ssh -o ConnectTimeout=5 -o BatchMode=yes "pi@${RpiHost}" "echo ok" 2>&1 + +if ($LASTEXITCODE -eq 0) { + Write-Host "[OK] SSH connection successful" -ForegroundColor Green +} +else { + Write-Host "[X] SSH connection failed" -ForegroundColor Red + Write-Host " Make sure SSH keys are set up or use: ssh pi@$RpiHost" -ForegroundColor Yellow + Write-Host " You may need to accept the host key first" -ForegroundColor Yellow +} + +# Copy setup script to Pi +Write-Host "" +Write-Host "Copying setup script to Pi..." -ForegroundColor Yellow +$scriptPath = Join-Path $PSScriptRoot "setup-remote-debug.sh" + +if (Test-Path $scriptPath) { + scp $scriptPath "pi@${RpiHost}:/tmp/setup-remote-debug.sh" 2>&1 | Out-Null + if ($LASTEXITCODE -eq 0) { + Write-Host "[OK] Script copied successfully" -ForegroundColor Green + } + else { + Write-Host "[X] Failed to copy script" -ForegroundColor Red + } +} +else { + Write-Host "[!] Setup script not found at: $scriptPath" -ForegroundColor Yellow + Write-Host " Skipping script copy" -ForegroundColor Yellow +} + +if ($TestConnection) { + Write-Host "" + Write-Host "Testing debug port connection..." -ForegroundColor Yellow + $tcpTest = Test-NetConnection -ComputerName $RpiHost -Port $DebugPort -WarningAction SilentlyContinue + + if ($tcpTest.TcpTestSucceeded) { + Write-Host "[OK] Debug port $DebugPort is accessible" -ForegroundColor Green + } + else { + Write-Host "[X] Cannot connect to debug port $DebugPort" -ForegroundColor Red + Write-Host " Make sure gdbserver is running on the Pi" -ForegroundColor Yellow + } +} + +if ($SetupTunnel) { + Write-Host "" + Write-Host "Setting up SSH tunnel for debugging..." -ForegroundColor Yellow + Write-Host " Local port: $DebugPort -> ${RpiHost}:$DebugPort" -ForegroundColor Cyan + Write-Host "" + Write-Host "Keep this terminal open while debugging!" -ForegroundColor Yellow + Write-Host "Press Ctrl+C to close the tunnel" -ForegroundColor Yellow + Write-Host "" + + ssh -L "${DebugPort}:localhost:${DebugPort}" "pi@${RpiHost}" + exit +} + +# Display next steps +Write-Host "" +Write-Host "=== Next Steps ===" -ForegroundColor Cyan +Write-Host "" +Write-Host "1. On the Pi, run the setup script:" -ForegroundColor White +Write-Host " ssh pi@$RpiHost" -ForegroundColor Gray +Write-Host " chmod +x /tmp/setup-remote-debug.sh" -ForegroundColor Gray +Write-Host " sudo /tmp/setup-remote-debug.sh" -ForegroundColor Gray +Write-Host "" +Write-Host " The script will:" -ForegroundColor Gray +Write-Host " - Install gdbserver and build tools" -ForegroundColor Gray +Write-Host " - Clone OpenAuto repo (if needed)" -ForegroundColor Gray +Write-Host " - Install all build dependencies" -ForegroundColor Gray +Write-Host " - Build OpenAuto with debug symbols" -ForegroundColor Gray +Write-Host " - Start gdbserver with your choice of mode" -ForegroundColor Gray +Write-Host "" +Write-Host "2. In VS Code:" -ForegroundColor White +Write-Host " - Open the openauto workspace" -ForegroundColor Gray +Write-Host " - Press F5 or go to Run > Start Debugging" -ForegroundColor Gray +Write-Host " - Select 'Remote Debug on rpi5 (Launch)' or '(Attach)'" -ForegroundColor Gray +Write-Host "" +Write-Host "3. Set breakpoints to find the hang:" -ForegroundColor White +Write-Host " Files: src/autoapp/Service/Sensor/SensorService.cpp" -ForegroundColor Gray +Write-Host " - Line in stop() where stopPolling is set" -ForegroundColor Gray +Write-Host " - Line in sensorPolling() where it checks stopPolling" -ForegroundColor Gray +Write-Host " - Line in onChannelError() at the start" -ForegroundColor Gray +Write-Host "" +Write-Host " Also check: src/autoapp/Service/AndroidAutoEntity.cpp" -ForegroundColor Gray +Write-Host " - stop() method" -ForegroundColor Gray +Write-Host " - onChannelError() method" -ForegroundColor Gray +Write-Host "" +Write-Host "4. Reproduce the hang:" -ForegroundColor White +Write-Host " - Let Android Auto connect and run normally" -ForegroundColor Gray +Write-Host " - Click 'Exit' in Android Auto" -ForegroundColor Gray +Write-Host " - Watch which breakpoints are hit" -ForegroundColor Gray +Write-Host " - If it hangs, pause execution and check:" -ForegroundColor Gray +Write-Host " * Thread states (View > Call Stack)" -ForegroundColor Gray +Write-Host " * Value of stopPolling flag" -ForegroundColor Gray +Write-Host " * Timer state" -ForegroundColor Gray +Write-Host " * Which threads are still running" -ForegroundColor Gray +Write-Host "" + +# Offer to open SSH connection +Write-Host "Would you like to connect to the Pi now? [Y/n]: " -NoNewline -ForegroundColor Yellow +$response = Read-Host + +if ($response -eq "" -or $response -eq "Y" -or $response -eq "y") { + Write-Host "" + Write-Host "Connecting to $RpiHost..." -ForegroundColor Green + ssh "pi@${RpiHost}" +} diff --git a/scripts/setup-remote-debug.sh b/scripts/setup-remote-debug.sh new file mode 100644 index 00000000..7f2b829d --- /dev/null +++ b/scripts/setup-remote-debug.sh @@ -0,0 +1,219 @@ +#!/bin/bash +# Setup script for remote debugging on Raspberry Pi +# Run this on the Raspberry Pi to prepare for remote debugging + +set -e + +echo "=== OpenAuto Remote Debugging Setup ===" +echo "" + +# Detect if running as sudo and get the real user +if [ "$EUID" -eq 0 ] && [ -n "$SUDO_USER" ]; then + REAL_USER="$SUDO_USER" + REAL_HOME=$(eval echo "~$SUDO_USER") + echo "⚠️ Running as sudo. Using $REAL_USER's home directory: $REAL_HOME" +else + REAL_USER="$USER" + REAL_HOME="$HOME" +fi + +# Check if this is a fresh setup +FRESH_SETUP=false +if [ ! -d "$REAL_HOME/src/openauto" ]; then + FRESH_SETUP=true +fi + +# Check if gdbserver is installed +if ! command -v gdbserver &> /dev/null; then + echo "Installing gdbserver..." + sudo apt-get update + sudo apt-get install -y gdbserver git cmake build-essential pkg-config +else + echo "✓ gdbserver is already installed" +fi + +# If fresh setup, offer to clone and build openauto +if [ "$FRESH_SETUP" = true ]; then + echo "" + echo "OpenAuto source not found in $REAL_HOME/src/openauto" + read -p "Do you want to clone and build OpenAuto with debug symbols? [Y/n]: " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] || [[ -z $REPLY ]]; then + echo "" + echo "Cloning OpenAuto repository..." + mkdir -p "$REAL_HOME/src" + cd "$REAL_HOME/src" + git clone https://github.com/opencardev/openauto.git + cd openauto + git checkout develop + + echo "" + echo "Installing build dependencies..." + sudo apt-get install -y \ + libboost-all-dev \ + libusb-1.0-0-dev \ + libssl-dev \ + libprotobuf-dev \ + protobuf-compiler \ + libqt5multimedia5 \ + libqt5multimedia5-plugins \ + libqt5multimediawidgets5 \ + qtmultimedia5-dev \ + libqt5bluetooth5 \ + libqt5bluetooth5-bin \ + qtconnectivity5-dev \ + librtaudio-dev \ + libgstreamer1.0-dev \ + libgstreamer-plugins-base1.0-dev \ + gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-ugly \ + gstreamer1.0-libav \ + gstreamer1.0-alsa \ + libgps-dev \ + gpsd \ + libblkid-dev \ + libtag1-dev + + # Try to install aasdk from apt first + if sudo apt-cache show libaasdk-dev &> /dev/null; then + echo "Installing aasdk from apt..." + sudo apt-get install -y libaasdk-dev + else + echo "Building aasdk from source..." + cd "$REAL_HOME/src" + git clone https://github.com/opencardev/aasdk.git + cd aasdk + mkdir -p build-release + cd build-release + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j$(nproc) + sudo make install + sudo ldconfig + fi + + echo "" + echo "Building OpenAuto with debug symbols..." + cd "$REAL_HOME/src/openauto" + chmod +x build.sh + sudo -u "$REAL_USER" ./build.sh debug + + echo "" + echo "✓ OpenAuto built successfully with debug symbols" + echo " Binary location: $REAL_HOME/src/openauto/build-debug/autoapp" + fi +fi + +# Stop the running service +echo "" +echo "Stopping openauto service..." +sudo systemctl stop openauto || true + +# Find the autoapp binary +AUTOAPP_PATH="/opt/crankshaft/autoapp" +if [ ! -f "$AUTOAPP_PATH" ]; then + echo "autoapp not found at $AUTOAPP_PATH" + + # Try the build directory + if [ -f "$REAL_HOME/src/openauto/build-debug/autoapp" ]; then + AUTOAPP_PATH="$REAL_HOME/src/openauto/build-debug/autoapp" + echo "✓ Using debug build: $AUTOAPP_PATH" + else + echo "Searching for autoapp..." + AUTOAPP_PATH=$(which autoapp 2>/dev/null || echo "") + if [ -z "$AUTOAPP_PATH" ]; then + echo "Error: Could not find autoapp binary" + echo "" + echo "Please ensure OpenAuto is built with debug symbols:" + echo " cd $REAL_HOME/src/openauto" + echo " ./build.sh debug" + exit 1 + fi + echo "✓ Found autoapp at: $AUTOAPP_PATH" + fi +else + echo "✓ Found autoapp at: $AUTOAPP_PATH" +fi + +# Check if binary has debug symbols +echo "" +echo "Checking for debug symbols..." +if file "$AUTOAPP_PATH" | grep -q "not stripped"; then + echo "✓ Debug symbols present" +else + echo "⚠ Warning: Binary appears to be stripped (no debug symbols)" + echo " Debugging will be limited. Consider rebuilding with -DCMAKE_BUILD_TYPE=Debug" +fi + +# Kill any existing gdbserver +echo "" +echo "Cleaning up any existing gdbserver processes..." +sudo pkill -9 gdbserver 2>/dev/null || true + +# Display options +echo "" +echo "=== Debug Options ===" +echo "" +echo "Choose how to start debugging:" +echo "1) Start autoapp under gdbserver (recommended for reproducing hang)" +echo "2) Attach gdbserver to running autoapp" +echo "3) Just prepare (manual start)" +echo "" +read -p "Enter choice [1-3]: " choice + +case $choice in + 1) + echo "" + echo "Starting autoapp under gdbserver on port 2345..." + echo "The application will wait for debugger to connect." + echo "" + echo "In VS Code:" + echo " 1. Press F5" + echo " 2. Select 'Remote Debug on rpi5 (Launch)'" + echo " 3. The application will start and you can set breakpoints" + echo "" + echo "Starting gdbserver..." + sudo gdbserver :2345 "$AUTOAPP_PATH" + ;; + 2) + echo "" + echo "Starting autoapp normally..." + sudo "$AUTOAPP_PATH" & + AUTOAPP_PID=$! + sleep 2 + + if ps -p $AUTOAPP_PID > /dev/null; then + echo "✓ autoapp started with PID: $AUTOAPP_PID" + echo "" + echo "Attaching gdbserver to PID $AUTOAPP_PID on port 2345..." + echo "" + echo "In VS Code:" + echo " 1. Press F5" + echo " 2. Select 'Remote Debug on rpi5 (Attach)'" + echo " 3. You can now debug the running process" + echo "" + sudo gdbserver --attach :2345 $AUTOAPP_PID + else + echo "Error: Failed to start autoapp" + exit 1 + fi + ;; + 3) + echo "" + echo "=== Manual Start Instructions ===" + echo "" + echo "To start with gdbserver:" + echo " sudo gdbserver :2345 $AUTOAPP_PATH" + echo "" + echo "To attach to running process:" + echo " 1. Start autoapp: sudo $AUTOAPP_PATH &" + echo " 2. Get PID: ps aux | grep autoapp" + echo " 3. Attach: sudo gdbserver --attach :2345 PID" + echo "" + echo "Then in VS Code, press F5 and select the remote debug configuration" + ;; + *) + echo "Invalid choice" + exit 1 + ;; +esac