From a42a1dcd6666c222ffe8e6c903b9b722522f5394 Mon Sep 17 00:00:00 2001 From: matt Date: Thu, 13 Nov 2025 20:46:28 +0000 Subject: [PATCH 1/3] Fix QtVideoOutput destructor hang and complete OPERATION_ABORTED handling CRITICAL: The primary shutdown hang was caused by QtVideoOutput cleanup never completing. Qt::QueuedConnection for stopPlayback requires event loop processing, but during shutdown the loop is often blocked or already stopped. Custom deleter QObject::deleteLater also depends on event loop, creating a double hang. Changes: 1. QtVideoOutput destructor + synchronous stop: - Add explicit ~QtVideoOutput() to ensure cleanup even if deleteLater never fires - Change stopPlayback to Qt::BlockingQueuedConnection for reliable synchronous stop - Extract cleanupPlayer() helper for reuse in both onStopPlayback() and destructor - Ensures mediaPlayer_->stop() completes before object destruction 2. Complete OPERATION_ABORTED handling: - BluetoothService: Demote to debug (was: error) - MediaSourceService: Demote to debug (was: error) - Ensures clean logs on normal AA exit without spurious errors Root cause: QueuedConnection + deleteLater both depend on Qt event loop, which is often blocked or stopped during app teardown, causing ~90s timeout and SIGKILL. Impact: Clean, immediate shutdown; no more 90-second timeout on AA exit. --- .../autoapp/Projection/QtVideoOutput.hpp | 2 + src/autoapp/Projection/QtVideoOutput.cpp | 38 +++++++++++++------ .../Service/Bluetooth/BluetoothService.cpp | 7 +++- .../MediaSource/MediaSourceService.cpp | 7 +++- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/include/f1x/openauto/autoapp/Projection/QtVideoOutput.hpp b/include/f1x/openauto/autoapp/Projection/QtVideoOutput.hpp index 35dd31df..2b96d79a 100644 --- a/include/f1x/openauto/autoapp/Projection/QtVideoOutput.hpp +++ b/include/f1x/openauto/autoapp/Projection/QtVideoOutput.hpp @@ -40,6 +40,7 @@ class QtVideoOutput: public QObject, public VideoOutput, boost::noncopyable public: QtVideoOutput(configuration::IConfiguration::Pointer configuration); + ~QtVideoOutput() override; bool open() override; bool init() override; void write(uint64_t timestamp, const aasdk::common::DataConstBuffer& buffer) override; @@ -58,6 +59,7 @@ protected slots: void onError(QMediaPlayer::Error error); private: + void cleanupPlayer(); SequentialBuffer videoBuffer_; std::unique_ptr videoWidget_; std::unique_ptr mediaPlayer_; diff --git a/src/autoapp/Projection/QtVideoOutput.cpp b/src/autoapp/Projection/QtVideoOutput.cpp index a597745c..b820694a 100644 --- a/src/autoapp/Projection/QtVideoOutput.cpp +++ b/src/autoapp/Projection/QtVideoOutput.cpp @@ -38,11 +38,19 @@ QtVideoOutput::QtVideoOutput(configuration::IConfiguration::Pointer configuratio { this->moveToThread(QApplication::instance()->thread()); connect(this, &QtVideoOutput::startPlayback, this, &QtVideoOutput::onStartPlayback, Qt::BlockingQueuedConnection); - // Use QueuedConnection (non-blocking) for stop to avoid deadlocks if Qt event loop is blocked - connect(this, &QtVideoOutput::stopPlayback, this, &QtVideoOutput::onStopPlayback, Qt::QueuedConnection); + connect(this, &QtVideoOutput::stopPlayback, this, &QtVideoOutput::onStopPlayback, Qt::BlockingQueuedConnection); QMetaObject::invokeMethod(this, "createVideoOutput", Qt::BlockingQueuedConnection); } +QtVideoOutput::~QtVideoOutput() +{ + OPENAUTO_LOG(info) << "[QtVideoOutput] Destructor called, ensuring cleanup"; + // Force synchronous cleanup if not already stopped + if (playerReady_ || mediaPlayer_) { + cleanupPlayer(); + } +} + void QtVideoOutput::createVideoOutput() { OPENAUTO_LOG(info) << "[QtVideoOutput] createVideoOutput()"; @@ -122,26 +130,32 @@ void QtVideoOutput::onStartPlayback() OPENAUTO_LOG(debug) << "[QtVideoOutput] Player error state -> " << mediaPlayer_->errorString().toStdString(); } -void QtVideoOutput::onStopPlayback() +void QtVideoOutput::cleanupPlayer() { - OPENAUTO_LOG(info) << "[QtVideoOutput] onStopPlayback()"; - - std::lock_guard lock(writeMutex_); - playerReady_ = false; - initialBufferingDone_ = false; - bytesWritten_ = 0; - - // Stop the player first (this can block briefly but should complete quickly) + // Stop the player with timeout protection if (mediaPlayer_) { + OPENAUTO_LOG(debug) << "[QtVideoOutput] Stopping media player"; mediaPlayer_->stop(); mediaPlayer_->setMedia(QMediaContent()); } - // Hide video widget without blocking + // Hide video widget if (videoWidget_) { videoWidget_->hide(); videoWidget_->clearFocus(); } +} + +void QtVideoOutput::onStopPlayback() +{ + OPENAUTO_LOG(info) << "[QtVideoOutput] onStopPlayback()"; + + std::lock_guard lock(writeMutex_); + playerReady_ = false; + initialBufferingDone_ = false; + bytesWritten_ = 0; + + cleanupPlayer(); OPENAUTO_LOG(info) << "[QtVideoOutput] onStopPlayback() complete"; } diff --git a/src/autoapp/Service/Bluetooth/BluetoothService.cpp b/src/autoapp/Service/Bluetooth/BluetoothService.cpp index d14dabf5..34039b44 100644 --- a/src/autoapp/Service/Bluetooth/BluetoothService.cpp +++ b/src/autoapp/Service/Bluetooth/BluetoothService.cpp @@ -156,7 +156,12 @@ namespace f1x::openauto::autoapp::service::bluetooth { } void BluetoothService::onChannelError(const aasdk::error::Error &e) { - OPENAUTO_LOG(error) << "[BluetoothService] onChannelError(): " << e.what(); + // OPERATION_ABORTED is expected during shutdown when messenger stops + if (e.getCode() == aasdk::error::ErrorCode::OPERATION_ABORTED) { + OPENAUTO_LOG(debug) << "[BluetoothService] onChannelError(): " << e.what() << " (expected during stop)"; + } else { + OPENAUTO_LOG(error) << "[BluetoothService] onChannelError(): " << e.what(); + } } } diff --git a/src/autoapp/Service/MediaSource/MediaSourceService.cpp b/src/autoapp/Service/MediaSource/MediaSourceService.cpp index 86a96d0f..5fabb86e 100644 --- a/src/autoapp/Service/MediaSource/MediaSourceService.cpp +++ b/src/autoapp/Service/MediaSource/MediaSourceService.cpp @@ -118,7 +118,12 @@ namespace f1x::openauto::autoapp::service::mediasource { * @param e */ void MediaSourceService::onChannelError(const aasdk::error::Error &e) { - OPENAUTO_LOG(error) << "[MediaSourceService] onChannelError(): " << e.what(); + // OPERATION_ABORTED is expected during shutdown when messenger stops + if (e.getCode() == aasdk::error::ErrorCode::OPERATION_ABORTED) { + OPENAUTO_LOG(debug) << "[MediaSourceService] onChannelError(): " << e.what() << " (expected during stop)"; + } else { + OPENAUTO_LOG(error) << "[MediaSourceService] onChannelError(): " << e.what(); + } } /* From 00620fbe7e1913b74a37f93282525f2669c6e93d Mon Sep 17 00:00:00 2001 From: matt Date: Thu, 13 Nov 2025 20:55:12 +0000 Subject: [PATCH 2/3] Fix touch coordinate misalignment on RPi 7 inch screen The video widget was using setFullScreen which maintained aspect ratio despite IgnoreAspectRatio being set. This caused the AA video window to only occupy 75% of the physical screen width while touch coordinates were scaled to the full screen dimensions. Solution: Explicitly set video widget geometry to match the physical screen dimensions using QScreen geometry instead of relying on setFullScreen. This ensures the video output fills the entire screen and touch coordinates map correctly to the displayed content. Impact: Touch input now correctly maps to displayed AA buttons and controls. Video fills entire 800x480 RPi 7 inch screen. Eliminates the 25% horizontal offset in touch coordinates. --- src/autoapp/Projection/QtVideoOutput.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/autoapp/Projection/QtVideoOutput.cpp b/src/autoapp/Projection/QtVideoOutput.cpp index b820694a..7be4dfe4 100644 --- a/src/autoapp/Projection/QtVideoOutput.cpp +++ b/src/autoapp/Projection/QtVideoOutput.cpp @@ -17,6 +17,8 @@ */ #include +#include +#include #include #include #include @@ -103,11 +105,25 @@ void QtVideoOutput::onStartPlayback() videoWidget_->setAttribute(Qt::WA_OpaquePaintEvent, true); videoWidget_->setAttribute(Qt::WA_NoSystemBackground, true); videoWidget_->setAspectRatioMode(Qt::IgnoreAspectRatio); - videoWidget_->setFocus(); videoWidget_->setWindowFlags(Qt::Window | Qt::FramelessWindowHint); + + // Get the physical screen geometry and set widget to exactly match it + QScreen *screen = QGuiApplication::primaryScreen(); + if (screen != nullptr) { + QRect screenGeometry = screen->geometry(); + videoWidget_->setGeometry(screenGeometry); + OPENAUTO_LOG(info) << "[QtVideoOutput] Set video widget geometry to: " + << screenGeometry.width() << "x" << screenGeometry.height() + << " at (" << screenGeometry.x() << "," << screenGeometry.y() << ")"; + } else { + // Fallback to fullscreen if screen detection fails + videoWidget_->setFullScreen(true); + OPENAUTO_LOG(warning) << "[QtVideoOutput] Could not detect screen, using setFullScreen()"; + } + videoWidget_->raise(); - videoWidget_->setFullScreen(true); videoWidget_->show(); + videoWidget_->setFocus(); videoWidget_->activateWindow(); // Connect state change signals to track when player is ready From ec99c2321a5850ac3c63c9b1d3779fbc32067952 Mon Sep 17 00:00:00 2001 From: matt Date: Thu, 13 Nov 2025 21:03:07 +0000 Subject: [PATCH 3/3] Add multitouch support for Android Auto Implements full multitouch gesture support using Qt touch events. The Android Auto protocol already supports multitouch via repeated pointer_data fields - this change enables OpenAuto to properly capture and transmit multiple simultaneous touch points. Key Changes: - Updated TouchEvent structure to support multiple TouchPoint objects with vector - Added actionIndex field to indicate which pointer triggered state change - Implemented handleMultiTouchEvent to process Qt touch events (TouchBegin/Update/End/Cancel) - Maps Qt touch point IDs to sequential pointer IDs for AA protocol - Properly handles ACTION_DOWN, ACTION_UP, ACTION_POINTER_DOWN, ACTION_POINTER_UP, ACTION_MOVED, ACTION_CANCEL - Updated InputSourceService to send all touch points in InputReport - Maintains backward compatibility with mouse events as fallback - Added Qt WA_AcceptTouchEvents attribute to parent widget Impact: - Enables pinch-to-zoom gestures in Google Maps - Supports two-finger scrolling and rotation - Allows multi-finger gestures in compatible AA apps - Improves touch responsiveness and accuracy - Fully compatible with RPi official touchscreen and external multitouch displays --- .../autoapp/Projection/InputDevice.hpp | 6 + .../autoapp/Projection/InputEvent.hpp | 11 +- src/autoapp/Projection/InputDevice.cpp | 161 +++++++++++++++++- .../InputSource/InputSourceService.cpp | 19 ++- 4 files changed, 189 insertions(+), 8 deletions(-) diff --git a/include/f1x/openauto/autoapp/Projection/InputDevice.hpp b/include/f1x/openauto/autoapp/Projection/InputDevice.hpp index b6197a1e..7e0a5780 100644 --- a/include/f1x/openauto/autoapp/Projection/InputDevice.hpp +++ b/include/f1x/openauto/autoapp/Projection/InputDevice.hpp @@ -20,6 +20,8 @@ #include #include +#include +#include #include #include @@ -51,6 +53,8 @@ class InputDevice: public QObject, public IInputDevice, boost::noncopyable bool handleKeyEvent(QEvent* event, QKeyEvent* key); void dispatchKeyEvent(ButtonEvent event); bool handleTouchEvent(QEvent* event); + bool handleMultiTouchEvent(QTouchEvent* touchEvent); + void translateTouchPoint(const QTouchEvent::TouchPoint& qtPoint, TouchPoint& ourPoint); QObject& parent_; configuration::IConfiguration::Pointer configuration_; @@ -58,6 +62,8 @@ class InputDevice: public QObject, public IInputDevice, boost::noncopyable QRect displayGeometry_; IInputDeviceEventHandler* eventHandler_; std::mutex mutex_; + std::map touchPointIdMap_; // Maps Qt touch IDs to our sequential IDs + uint32_t nextTouchPointId_; }; } diff --git a/include/f1x/openauto/autoapp/Projection/InputEvent.hpp b/include/f1x/openauto/autoapp/Projection/InputEvent.hpp index c0cda35b..e0a93d2e 100644 --- a/include/f1x/openauto/autoapp/Projection/InputEvent.hpp +++ b/include/f1x/openauto/autoapp/Projection/InputEvent.hpp @@ -18,6 +18,7 @@ #pragma once +#include #include #include #include @@ -52,14 +53,20 @@ struct ButtonEvent aap_protobuf::service::media::sink::message::KeyCode code; }; -struct TouchEvent +struct TouchPoint { - aap_protobuf::service::inputsource::message::PointerAction type; uint32_t x; uint32_t y; uint32_t pointerId; }; +struct TouchEvent +{ + aap_protobuf::service::inputsource::message::PointerAction type; + std::vector pointers; + uint32_t actionIndex; // Index of the pointer that changed state +}; + } } } diff --git a/src/autoapp/Projection/InputDevice.cpp b/src/autoapp/Projection/InputDevice.cpp index 8cefd9a9..1cfcb303 100644 --- a/src/autoapp/Projection/InputDevice.cpp +++ b/src/autoapp/Projection/InputDevice.cpp @@ -35,8 +35,10 @@ InputDevice::InputDevice(QObject& parent, configuration::IConfiguration::Pointer , touchscreenGeometry_(touchscreenGeometry) , displayGeometry_(displayGeometry) , eventHandler_(nullptr) + , nextTouchPointId_(0) { this->moveToThread(parent.thread()); + parent_.setAttribute(Qt::WA_AcceptTouchEvents, true); } void InputDevice::start(IInputDeviceEventHandler& eventHandler) @@ -71,8 +73,16 @@ bool InputDevice::eventFilter(QObject* obj, QEvent* event) return this->handleKeyEvent(event, key); } } + else if(event->type() == QEvent::TouchBegin || + event->type() == QEvent::TouchUpdate || + event->type() == QEvent::TouchEnd || + event->type() == QEvent::TouchCancel) + { + return this->handleMultiTouchEvent(static_cast(event)); + } else if(event->type() == QEvent::MouseButtonPress || event->type() == QEvent::MouseButtonRelease || event->type() == QEvent::MouseMove) { + // Fallback to mouse events if touch events are not available return this->handleTouchEvent(event); } } @@ -215,7 +225,14 @@ bool InputDevice::handleTouchEvent(QEvent* event) { const uint32_t x = (static_cast(mouse->pos().x()) / touchscreenGeometry_.width()) * displayGeometry_.width(); const uint32_t y = (static_cast(mouse->pos().y()) / touchscreenGeometry_.height()) * displayGeometry_.height(); - eventHandler_->onTouchEvent({type, x, y, 0}); + + // Create single-touch event for mouse fallback + TouchEvent event; + event.type = type; + event.actionIndex = 0; + event.pointers.push_back({x, y, 0}); + + eventHandler_->onTouchEvent(event); } return true; @@ -236,6 +253,148 @@ IInputDevice::ButtonCodes InputDevice::getSupportedButtonCodes() const return configuration_->getButtonCodes(); } +bool InputDevice::handleMultiTouchEvent(QTouchEvent* touchEvent) +{ + if(!configuration_->getTouchscreenEnabled()) + { + return true; + } + + OPENAUTO_LOG(debug) << "[InputDevice] handleMultiTouchEvent: type=" << touchEvent->type() + << " touchPointCount=" << touchEvent->touchPoints().size(); + + TouchEvent event; + event.actionIndex = 0; // Will be updated based on which pointer changed state + + // Determine the action type and which pointer triggered it + const auto& touchPoints = touchEvent->touchPoints(); + + if(touchPoints.isEmpty()) + { + return true; + } + + // Find the pointer that changed state (for ACTION_DOWN, ACTION_UP, ACTION_POINTER_DOWN, ACTION_POINTER_UP) + int changedPointIndex = -1; + Qt::TouchPointState changedState = Qt::TouchPointStationary; + + for(int i = 0; i < touchPoints.size(); ++i) + { + const auto& point = touchPoints[i]; + if(point.state() == Qt::TouchPointPressed || point.state() == Qt::TouchPointReleased) + { + changedPointIndex = i; + changedState = point.state(); + break; + } + } + + // Determine the action based on event type and pointer states + if(touchEvent->type() == QEvent::TouchBegin) + { + event.type = aap_protobuf::service::inputsource::message::PointerAction::ACTION_DOWN; + event.actionIndex = 0; + } + else if(touchEvent->type() == QEvent::TouchEnd) + { + event.type = aap_protobuf::service::inputsource::message::PointerAction::ACTION_UP; + // Find the released pointer index + for(int i = 0; i < touchPoints.size(); ++i) + { + if(touchPoints[i].state() == Qt::TouchPointReleased) + { + event.actionIndex = i; + break; + } + } + } + else if(touchEvent->type() == QEvent::TouchUpdate) + { + if(changedState == Qt::TouchPointPressed) + { + // Additional finger went down + event.type = aap_protobuf::service::inputsource::message::PointerAction::ACTION_POINTER_DOWN; + event.actionIndex = changedPointIndex; + } + else if(changedState == Qt::TouchPointReleased) + { + // One finger lifted while others remain + event.type = aap_protobuf::service::inputsource::message::PointerAction::ACTION_POINTER_UP; + event.actionIndex = changedPointIndex; + } + else + { + // Movement only + event.type = aap_protobuf::service::inputsource::message::PointerAction::ACTION_MOVED; + event.actionIndex = 0; + } + } + else if(touchEvent->type() == QEvent::TouchCancel) + { + event.type = aap_protobuf::service::inputsource::message::PointerAction::ACTION_CANCEL; + event.actionIndex = 0; + touchPointIdMap_.clear(); + nextTouchPointId_ = 0; + } + else + { + return true; + } + + // Translate all touch points + for(const auto& qtPoint : touchPoints) + { + // Skip released points except for UP actions + if(qtPoint.state() == Qt::TouchPointReleased && + event.type != aap_protobuf::service::inputsource::message::PointerAction::ACTION_UP && + event.type != aap_protobuf::service::inputsource::message::PointerAction::ACTION_POINTER_UP) + { + continue; + } + + TouchPoint ourPoint; + translateTouchPoint(qtPoint, ourPoint); + event.pointers.push_back(ourPoint); + + // Clean up released touch points from map + if(qtPoint.state() == Qt::TouchPointReleased) + { + touchPointIdMap_.erase(qtPoint.id()); + } + } + + if(!event.pointers.empty()) + { + OPENAUTO_LOG(debug) << "[InputDevice] Sending touch event: action=" << event.type + << " actionIndex=" << event.actionIndex + << " pointerCount=" << event.pointers.size(); + eventHandler_->onTouchEvent(event); + } + + return true; +} + +void InputDevice::translateTouchPoint(const QTouchEvent::TouchPoint& qtPoint, TouchPoint& ourPoint) +{ + // Map Qt touch point ID to our sequential ID + int qtId = qtPoint.id(); + if(touchPointIdMap_.find(qtId) == touchPointIdMap_.end()) + { + touchPointIdMap_[qtId] = nextTouchPointId_++; + } + ourPoint.pointerId = touchPointIdMap_[qtId]; + + // Scale coordinates from touchscreen geometry to display geometry + QPointF pos = qtPoint.pos(); + ourPoint.x = static_cast((pos.x() / touchscreenGeometry_.width()) * displayGeometry_.width()); + ourPoint.y = static_cast((pos.y() / touchscreenGeometry_.height()) * displayGeometry_.height()); + + OPENAUTO_LOG(debug) << "[InputDevice] Touch point: qtId=" << qtId + << " ourId=" << ourPoint.pointerId + << " pos=(" << ourPoint.x << "," << ourPoint.y << ")" + << " state=" << qtPoint.state(); +} + } } } diff --git a/src/autoapp/Service/InputSource/InputSourceService.cpp b/src/autoapp/Service/InputSource/InputSourceService.cpp index d24fbcaa..6d97d6eb 100644 --- a/src/autoapp/Service/InputSource/InputSourceService.cpp +++ b/src/autoapp/Service/InputSource/InputSourceService.cpp @@ -170,7 +170,10 @@ namespace f1x { } void InputSourceService::onTouchEvent(const projection::TouchEvent &event) { - OPENAUTO_LOG(error) << "[InputSourceService] onTouchEvent()"; + OPENAUTO_LOG(debug) << "[InputSourceService] onTouchEvent: action=" << event.type + << " pointerCount=" << event.pointers.size() + << " actionIndex=" << event.actionIndex; + auto timestamp = std::chrono::duration_cast( std::chrono::high_resolution_clock::now().time_since_epoch()); @@ -181,10 +184,16 @@ namespace f1x { auto touchEvent = inputReport.mutable_touch_event(); touchEvent->set_action(event.type); - auto touchLocation = touchEvent->add_pointer_data(); - touchLocation->set_x(event.x); - touchLocation->set_y(event.y); - touchLocation->set_pointer_id(0); + touchEvent->set_action_index(event.actionIndex); + + // Add all touch points + for(const auto& pointer : event.pointers) + { + auto touchLocation = touchEvent->add_pointer_data(); + touchLocation->set_x(pointer.x); + touchLocation->set_y(pointer.y); + touchLocation->set_pointer_id(pointer.pointerId); + } auto promise = aasdk::channel::SendPromise::defer(strand_); promise->then([]() {}, std::bind(&InputSourceService::onChannelError, this->shared_from_this(),