diff --git a/.gitignore b/.gitignore index 985ddd5e949..4c3c6194eb2 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,5 @@ source/developer/localization.md *.key *.crt .aider* + +**/.claude/settings.local.json diff --git a/scripts/air-gap-docker-registry-setup.sh b/scripts/air-gap-docker-registry-setup.sh new file mode 100755 index 00000000000..52553102863 --- /dev/null +++ b/scripts/air-gap-docker-registry-setup.sh @@ -0,0 +1,366 @@ +#!/bin/bash + +# Air-Gap Docker Registry Setup for Mattermost Calls Offloader +# This script sets up a local Docker registry and pre-loads required images +# for air-gapped network deployments +# +# Usage: ./air-gap-docker-registry-setup.sh +# Example: ./air-gap-docker-registry-setup.sh v0.8.5 v0.6.3 + +set -e + +# Function to display usage +usage() { + echo "Usage: $0 " + echo "" + echo "Arguments:" + echo " recorder-version Version of mattermost/calls-recorder image (e.g., v0.8.5)" + echo " transcriber-version Version of mattermost/calls-transcriber image (e.g., v0.6.3)" + echo "" + echo "Examples:" + echo " $0 v0.8.5 v0.6.3" + echo " $0 v0.9.0 v0.7.0" + echo "" + echo "To find the correct versions for your Calls plugin:" + echo "1. Check your Calls plugin version in System Console > Plugins > Plugin Management" + echo "2. Visit: https://github.com/mattermost/mattermost-plugin-calls/blob/v/plugin.json" + echo "3. Look for 'RecorderImage' and 'TranscriberImage' entries (near the bottom)" + echo "" + echo "Environment variables (optional):" + echo " REGISTRY_HOST Docker registry host (default: localhost)" + echo " REGISTRY_PORT Docker registry port (default: 5000)" + echo " REGISTRY_DATA_DIR Registry data directory (default: /opt/docker-registry/data)" + exit 1 +} + +# Check if required arguments are provided +if [ $# -ne 2 ]; then + echo "ERROR: Missing required arguments" + echo "" + usage +fi + +# Validate version format (should start with 'v' followed by semantic version) +validate_version() { + local version=$1 + local image_name=$2 + + if [[ ! $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: Invalid version format for $image_name: $version" + echo "Expected format: vX.Y.Z (e.g., v0.8.5)" + exit 1 + fi +} + +# Parse and validate arguments +CALLS_RECORDER_VERSION="$1" +CALLS_TRANSCRIBER_VERSION="$2" + +validate_version "$CALLS_RECORDER_VERSION" "calls-recorder" +validate_version "$CALLS_TRANSCRIBER_VERSION" "calls-transcriber" + +# Configuration variables +REGISTRY_HOST="${REGISTRY_HOST:-localhost}" +REGISTRY_PORT="${REGISTRY_PORT:-5000}" +REGISTRY_DATA_DIR="${REGISTRY_DATA_DIR:-/opt/docker-registry/data}" +REGISTRY_CONFIG_DIR="${REGISTRY_CONFIG_DIR:-/opt/docker-registry/config}" + +LOG_FILE=/tmp/air-gap-registry-setup.log + +echo "Setting up Air-Gap Docker Registry for Mattermost Calls" | tee $LOG_FILE +echo "Recorder version: $CALLS_RECORDER_VERSION" | tee -a $LOG_FILE +echo "Transcriber version: $CALLS_TRANSCRIBER_VERSION" | tee -a $LOG_FILE + +# Function to check if running with internet access (for image pulling phase) +check_internet() { + if ! curl -s --connect-timeout 5 https://hub.docker.com > /dev/null 2>&1; then + echo "ERROR: This script requires internet access during the image preparation phase." | tee -a $LOG_FILE + echo "Please run this script on a machine with internet access first, then transfer the images." | tee -a $LOG_FILE + return 1 + fi +} + +# Function to setup local Docker registry +setup_registry() { + echo "Setting up local Docker registry..." | tee -a $LOG_FILE + + # Create directories + sudo mkdir -p $REGISTRY_DATA_DIR + sudo mkdir -p $REGISTRY_CONFIG_DIR + + # Create registry configuration + cat > /tmp/registry-config.yml << EOF +version: 0.1 +log: + level: info +storage: + filesystem: + rootdirectory: /var/lib/registry +http: + addr: 0.0.0.0:5000 + headers: + X-Content-Type-Options: [nosniff] +EOF + + sudo mv /tmp/registry-config.yml $REGISTRY_CONFIG_DIR/config.yml + + # Pull and run registry container + echo "Starting Docker registry container..." | tee -a $LOG_FILE + docker pull registry:2 >> $LOG_FILE 2>&1 + + # Stop existing registry if running + docker stop local-registry 2>/dev/null || true + docker rm local-registry 2>/dev/null || true + + # Start registry container + docker run -d \ + --name local-registry \ + --restart=always \ + -p $REGISTRY_PORT:5000 \ + -v $REGISTRY_DATA_DIR:/var/lib/registry \ + -v $REGISTRY_CONFIG_DIR/config.yml:/etc/docker/registry/config.yml \ + registry:2 >> $LOG_FILE 2>&1 + + # Wait for registry to be ready + echo "Waiting for registry to be ready..." | tee -a $LOG_FILE + sleep 10 + + # Test registry + if curl -s http://$REGISTRY_HOST:$REGISTRY_PORT/v2/ > /dev/null; then + echo "Local Docker registry is running at $REGISTRY_HOST:$REGISTRY_PORT" | tee -a $LOG_FILE + else + echo "ERROR: Failed to start local Docker registry" | tee -a $LOG_FILE + return 1 + fi +} + +# Function to download and push Mattermost images +setup_mattermost_images() { + echo "Downloading and pushing Mattermost Calls images..." | tee -a $LOG_FILE + + # Images to process + declare -A IMAGES=( + ["mattermost/calls-recorder"]="$CALLS_RECORDER_VERSION" + ["mattermost/calls-transcriber"]="$CALLS_TRANSCRIBER_VERSION" + ) + + for image in "${!IMAGES[@]}"; do + version="${IMAGES[$image]}" + echo "Processing $image:$version..." | tee -a $LOG_FILE + + # Pull from Docker Hub + echo " Pulling $image:$version from Docker Hub..." | tee -a $LOG_FILE + docker pull $image:$version >> $LOG_FILE 2>&1 + + # Tag for local registry + local_tag="$REGISTRY_HOST:$REGISTRY_PORT/$image:$version" + echo " Tagging as $local_tag..." | tee -a $LOG_FILE + docker tag $image:$version $local_tag >> $LOG_FILE 2>&1 + + # Push to local registry + echo " Pushing to local registry..." | tee -a $LOG_FILE + docker push $local_tag >> $LOG_FILE 2>&1 + + # Also tag as 'latest' for convenience + if [ "$version" != "latest" ]; then + latest_tag="$REGISTRY_HOST:$REGISTRY_PORT/$image:latest" + docker tag $image:$version $latest_tag >> $LOG_FILE 2>&1 + docker push $latest_tag >> $LOG_FILE 2>&1 + fi + + echo " Successfully pushed $image:$version to local registry" | tee -a $LOG_FILE + done + + # Create registry data archive for transfer + echo "Creating registry data archive..." | tee -a $LOG_FILE + sudo tar -czf docker-registry-data.tar.gz -C $REGISTRY_DATA_DIR . + + # Create registry container image archive + echo "Creating registry container image archive..." | tee -a $LOG_FILE + docker save -o registry-image.tar registry:2 + gzip registry-image.tar + + echo "Created archives for air-gap transfer:" | tee -a $LOG_FILE + echo " - docker-registry-data.tar.gz" | tee -a $LOG_FILE + echo " - registry-image.tar.gz" | tee -a $LOG_FILE +} + +# Function to configure calls-offloader for local registry +configure_calls_offloader() { + echo "Configuring calls-offloader for local registry..." | tee -a $LOG_FILE + + # Create a modified calls-offloader config + if [ -f "/opt/calls-offloader/calls-offloader.toml" ]; then + sudo cp /opt/calls-offloader/calls-offloader.toml /opt/calls-offloader/calls-offloader.toml.backup + + # Update image_registry setting + sudo sed -i "s/image_registry = \"mattermost\"/image_registry = \"$REGISTRY_HOST:$REGISTRY_PORT\/mattermost\"/" /opt/calls-offloader/calls-offloader.toml + + echo "Updated calls-offloader configuration to use local registry" | tee -a $LOG_FILE + echo "Backup created at /opt/calls-offloader/calls-offloader.toml.backup" | tee -a $LOG_FILE + else + echo "Warning: calls-offloader.toml not found. You'll need to manually configure:" | tee -a $LOG_FILE + echo " image_registry = \"$REGISTRY_HOST:$REGISTRY_PORT/mattermost\"" | tee -a $LOG_FILE + fi +} + +# Function to create docker daemon configuration for insecure registry +configure_docker_daemon() { + echo "Configuring Docker daemon for insecure registry..." | tee -a $LOG_FILE + + # Create or update Docker daemon configuration + sudo mkdir -p /etc/docker + + # Check if daemon.json exists + if [ -f "/etc/docker/daemon.json" ]; then + sudo cp /etc/docker/daemon.json /etc/docker/daemon.json.backup + echo "Backed up existing daemon.json" | tee -a $LOG_FILE + fi + + # Create new daemon.json with insecure registry configuration + cat > /tmp/daemon.json << EOF +{ + "insecure-registries": ["$REGISTRY_HOST:$REGISTRY_PORT"] +} +EOF + + sudo mv /tmp/daemon.json /etc/docker/daemon.json + + # Restart Docker daemon + echo "Restarting Docker daemon..." | tee -a $LOG_FILE + sudo systemctl restart docker >> $LOG_FILE 2>&1 + + # Wait for Docker to restart + sleep 10 + + echo "Docker daemon configured for insecure registry access" | tee -a $LOG_FILE +} + +# Function to verify setup +verify_setup() { + echo "Verifying air-gap setup..." | tee -a $LOG_FILE + + # Check registry is accessible + if curl -s http://$REGISTRY_HOST:$REGISTRY_PORT/v2/_catalog | grep -q repositories; then + echo "✓ Local registry is accessible" | tee -a $LOG_FILE + else + echo "✗ Local registry is not accessible" | tee -a $LOG_FILE + return 1 + fi + + # List available images + echo "Available images in local registry:" | tee -a $LOG_FILE + curl -s http://$REGISTRY_HOST:$REGISTRY_PORT/v2/_catalog | jq '.repositories[]' 2>/dev/null || echo "Could not list repositories (jq not available)" | tee -a $LOG_FILE + + # Test pulling from local registry + test_image="$REGISTRY_HOST:$REGISTRY_PORT/mattermost/calls-offloader:latest" + echo "Testing pull from local registry: $test_image" | tee -a $LOG_FILE + if docker pull $test_image >> $LOG_FILE 2>&1; then + echo "✓ Successfully pulled test image from local registry" | tee -a $LOG_FILE + else + echo "✗ Failed to pull test image from local registry" | tee -a $LOG_FILE + return 1 + fi +} + +# Function to create air-gap deployment script +create_airgap_deployment_script() { + echo "Creating air-gap deployment script..." | tee -a $LOG_FILE + + cat > /tmp/deploy-airgap-calls.sh << 'EOF' +#!/bin/bash + +# Air-Gap Calls Deployment Script +# Run this script on the air-gapped network after setting up the local registry + +REGISTRY_HOST="${REGISTRY_HOST:-localhost}" +REGISTRY_PORT="${REGISTRY_PORT:-5000}" + +echo "Deploying Mattermost Calls in air-gapped environment..." + +# Configure Docker for local registry +sudo mkdir -p /etc/docker +cat > /tmp/daemon.json << EOD +{ + "insecure-registries": ["$REGISTRY_HOST:$REGISTRY_PORT"] +} +EOD +sudo mv /tmp/daemon.json /etc/docker/daemon.json +sudo systemctl restart docker +sleep 10 + +# Update calls-offloader configuration +if [ -f "/opt/calls-offloader/calls-offloader.toml" ]; then + sudo sed -i "s/image_registry = \"mattermost\"/image_registry = \"$REGISTRY_HOST:$REGISTRY_PORT\/mattermost\"/" /opt/calls-offloader/calls-offloader.toml + + # Restart calls-offloader service + sudo systemctl restart calls-offloader + + echo "Calls-offloader configured for air-gap deployment" +else + echo "Warning: /opt/calls-offloader/calls-offloader.toml not found" + echo "Please manually configure image_registry = \"$REGISTRY_HOST:$REGISTRY_PORT/mattermost\"" +fi + +echo "Air-gap deployment configuration complete" +echo "" +echo "IMPORTANT: Additional configuration required on Mattermost server:" +echo "On your Mattermost server, add this environment variable:" +echo " MM_CALLS_JOB_SERVICE_IMAGE_REGISTRY=\"$REGISTRY_HOST:$REGISTRY_PORT/mattermost\"" +echo "" +echo "Add it to /opt/mattermost/config/mattermost.environment and restart Mattermost:" +echo " echo 'MM_CALLS_JOB_SERVICE_IMAGE_REGISTRY=\"$REGISTRY_HOST:$REGISTRY_PORT/mattermost\"' | sudo tee -a /opt/mattermost/config/mattermost.environment" +echo " sudo systemctl restart mattermost" +EOF + + chmod +x /tmp/deploy-airgap-calls.sh + mv /tmp/deploy-airgap-calls.sh ./deploy-airgap-calls.sh + + echo "Air-gap deployment script created at ./deploy-airgap-calls.sh" | tee -a $LOG_FILE +} + +# Main execution +main() { + echo "Starting air-gap Docker registry setup..." | tee -a $LOG_FILE + + # Check if we have internet access for image pulling + if ! check_internet; then + echo "Skipping image download phase - run this script with internet access first" | tee -a $LOG_FILE + else + # Setup local registry + setup_registry + + # Download and push images + setup_mattermost_images + fi + + # Configure Docker daemon + configure_docker_daemon + + # Configure calls-offloader + configure_calls_offloader + + # Verify setup + verify_setup + + # Create air-gap deployment script + create_airgap_deployment_script + + echo "" | tee -a $LOG_FILE + echo "=== Air-Gap Setup Complete ===" | tee -a $LOG_FILE + echo "Local registry running at: http://$REGISTRY_HOST:$REGISTRY_PORT" | tee -a $LOG_FILE + echo "Registry data stored at: $REGISTRY_DATA_DIR" | tee -a $LOG_FILE + echo "" | tee -a $LOG_FILE + echo "Next steps for air-gapped deployment:" | tee -a $LOG_FILE + echo "1. Transfer these files to your air-gapped network:" | tee -a $LOG_FILE + echo " - docker-registry-data.tar.gz" | tee -a $LOG_FILE + echo " - registry-image.tar.gz" | tee -a $LOG_FILE + echo " - deploy-airgap-calls.sh" | tee -a $LOG_FILE + echo "2. Run the local registry container in the air-gapped network" | tee -a $LOG_FILE + echo "3. Execute ./deploy-airgap-calls.sh on the air-gapped systems" | tee -a $LOG_FILE + echo "" | tee -a $LOG_FILE + echo "Log file: $LOG_FILE" | tee -a $LOG_FILE +} + +# Run main function +main "$@" \ No newline at end of file diff --git a/source/configure/calls-rtcd-ent-only.md b/source/_static/badges/calls-rtcd-ent-only.md similarity index 100% rename from source/configure/calls-rtcd-ent-only.md rename to source/_static/badges/calls-rtcd-ent-only.md diff --git a/source/collaborate/make-calls.rst b/source/collaborate/make-calls.rst index 831bc31108d..2441fbc97e8 100644 --- a/source/collaborate/make-calls.rst +++ b/source/collaborate/make-calls.rst @@ -10,7 +10,7 @@ Using a web browser, the desktop app, or the mobile app, you can `join a call <# - All Mattermost customers can start, join, and participate in 1:1 audio calls with optional screen sharing. - For group calls up to 50 concurrent users, Mattermost Enterprise, Professional, or Mattermost Cloud is required. - - Enterprise customers can also `record calls <#record-a-call>`__, enable :ref:`live text captions ` during calls, and `transcribe recorded calls <#transcribe-recorded-calls>`__. We recommend that Enterprise self-hosted customers looking for group calls beyond 50 concurrent users consider using the :ref:`dedicated rtcd service `. + - Enterprise customers can also `record calls <#record-a-call>`__, enable :ref:`live text captions ` during calls, and `transcribe recorded calls <#transcribe-recorded-calls>`__. We recommend that Enterprise self-hosted customers looking for group calls beyond 50 concurrent users consider using the :doc:`dedicated RTCD service `. - Mattermost Cloud users can start calling right out of the box. For Mattermost self-hosted deployments, System admins need to enable and configure the plugin :ref:`using the System Console `. .. include:: ../_static/badges/academy-calls.rst diff --git a/source/configure/calls-deployment.md b/source/configure/calls-deployment.md index 6ec820a9c7d..dff70c3f03c 100644 --- a/source/configure/calls-deployment.md +++ b/source/configure/calls-deployment.md @@ -1,30 +1,45 @@ -# Calls self-hosted deployment +# Calls Deployment Overview -```{include} ../_static/badges/allplans-selfhosted.md +```{include} ../_static/badges/allplans-cloud-selfhosted.md ``` -Mattermost Calls is an excellent option for organizations demanding enhanced security and control over their communication infrastructure. Calls is designed to operate securely in self-hosted deployments, including [air-gapped environments](https://docs.mattermost.com/configure/calls-deployment.html#air-gapped-deployments), ensuring private communication without reliance on public internet connectivity with flexible configuration options for complex network requirements. +This document provides an overview of Mattermost Calls deployment options for self-hosted environments, including [air-gapped environments](https://docs.mattermost.com/configure/calls-deployment.html#air-gapped-deployments), ensuring private communication without reliance on public internet connectivity with flexible configuration options for complex network requirements. -This document provides information on how to successfully make the Calls plugin work on self-hosted deployments. It also outlines some of the most common deployment strategies with example diagrams, and provides the deployment guidelines for the recording, transcription, and live captions service. -## Terminology +## Quick Links + +For detailed information on specific topics, please refer to these specialized guides: -- [WebRTC](https://bloggeek.me/webrtcglossary/webrtc-2/): The set of underlying protocols/specifications on top of which calls are implemented. -- **RTC (Real Time Connection)**: The real-time connection. This is the channel used to send media tracks (audio/video/screen). -- **WS (WebSocket)**: The WebSocket connection. This is the channel used to set up a connection (signaling process). -- [NAT (Network Address Translation)](https://bloggeek.me/webrtcglossary/nat/): A networking technique to map IP addresses. -- [STUN (Session Traversal Utilities for NAT)](https://bloggeek.me/webrtcglossary/stun/): A protocol/service used by WebRTC clients to help traversing NATs. On the server side it's mainly used to figure out the public IP of the instance. -- [TURN (Traversal Using Relays around NAT)](https://bloggeek.me/webrtcglossary/turn/): A protocol/service used to help WebRTC clients behind strict firewalls connect to a call through media relay. +- {doc}`RTCD Setup and Configuration `: Comprehensive guide for setting up the dedicated RTCD service +- {doc}`Calls Offloader Setup and Configuration `: Comprehensive guide for setting up the calls-offloader service for recording and transcription +- {doc}`Calls Troubleshooting `: Detailed troubleshooting steps and debugging techniques +- {doc}`Calls Metrics and Monitoring `: Guide to monitoring Calls performance using metrics and observability +- {doc}`Calls Deployment on Kubernetes `: Detailed guide for deploying Calls in Kubernetes environments -## Plugin components +## About Mattermost Calls -- **Calls plugin**: This is the main entry point and a requirement to enable channel calls. +Mattermost Calls provides integrated audio calling and screen sharing capabilities within Mattermost channels. It's built on WebRTC technology and can be deployed either: -- **rtcd**: This is an optional service that can be deployed to offload all the functionality and data processing involved with the WebRTC connections. Read more about when and why to use [rtcd](#the-rtcd-service) below. +1. **Integrated mode**: Built into the Calls plugin (simpler, suitable for smaller deployments) +2. **RTCD mode**: Using a dedicated service for improved performance and scalability (recommended for production environments) + +## Terminology -## Requirements +- [WebRTC](https://bloggeek.me/webrtcglossary/webrtc-2/): The set of protocols on which calls are built +- **RTC**: Real-Time Connection channel used for media (audio/video/screen) +- **WS**: WebSocket connection used for signaling and connection setup +- **SFU**: Selective Forwarding Unit, routes media between participants +- [NAT](https://bloggeek.me/webrtcglossary/nat/): Network Address Translation for mapping IP addresses +- [STUN](https://bloggeek.me/webrtcglossary/stun/): Protocol used by WebRTC clients to help traverse NATs +- [TURN](https://bloggeek.me/webrtcglossary/turn/): Protocol to relay media for clients behind strict firewalls -### Server +## Key Components + +- **Calls plugin**: The main plugin that enables calls functionality. Installed by default in Mattermost self-hosted deployments. +- **RTCD service**: Optional dedicated service for offloading media processing (Enterprise feature). Typically deployed to dedicated servers or containers. See [RTCD Setup and Configuration](calls-rtcd-setup.md) for details. +- **calls-offloader**: Service for call recording and transcription (if enabled). Typically deployed to dedicated servers. See [Calls Offloader Setup and Configuration](calls-offloader-setup.md) for setup and troubleshooting details. + +## Network Requirements - Run Mattermost server on a secure (HTTPs) connection. This is a necessary requirement on the client to allow capturing devices (e.g., microphone, screen). See the [config TLS](https://docs.mattermost.com/deploy/server/setup-tls.html) section for more info. - See [network requirements](#network) below. @@ -88,7 +103,7 @@ This document provides information on how to successfully make the Calls plugin RTC (Calls plugin or rtcd) 8443 UDP (incoming) -Mattermost clients (Web/Desktop/Mobile) +Mattermost clients (Web/Desktop/Mobile) and jobs spawned by Calls Offloader (Recorder, Transcriber) Mattermost instance or rtcd service To allow clients to establish connections that transport calls related media (e.g. audio, video). This should be open on any network component (e.g. NAT, firewalls) in between the instance running the plugin (or rtcd) and the clients joining calls so that UDP traffic is correctly routed both ways (from/to clients). @@ -96,7 +111,7 @@ This document provides information on how to successfully make the Calls plugin RTC (Calls plugin or rtcd) 8443 TCP (incoming) -Mattermost clients (Web/Desktop/Mobile) +Mattermost clients (Web/Desktop/Mobile) and jobs spawned by Calls Offloader (Recorder, Transcriber) Mattermost instance or rtcd service To allow clients to establish connections that transport calls related media (e.g. audio, video). This should be open on any network component (e.g. NAT, firewalls) in between the instance running the plugin (or rtcd) and the clients joining calls so that TCP traffic is correctly routed both ways (from/to clients). This can be used as a backup channel in case clients are unable to connect using UDP. It requires rtcd version >= v0.11 and Calls version >= v0.17. @@ -119,624 +134,160 @@ This document provides information on how to successfully make the Calls plugin -#### Air-gapped deployments - -Mattermost Calls can function in air-gapped environments. Exposing Calls to the public internet is only necessary when users need to connect from outside the local network, and no existing method supports that connection. In such setups: - -- Users should connect from within the private/local network. This can be done on-premises, through a VPN, or via virtual machines. -- Configuring a STUN server is unnecessary, as all connections occur within the local network. -- The [ICE Host Override](https://docs.mattermost.com/configure/plugins-configuration-settings.html#ice-host-override) configuration setting can be optionally set with a local IP address (e.g., 192.168.1.45), depending on the specific network configuration and topology. +For complete network requirements, see the [RTCD Setup and Configuration](calls-rtcd-setup.md) guide. ## Limitations - All Mattermost customers can start, join, and participate in 1:1 audio calls with optional screen sharing. - For group calls up to 50 concurrent users, Mattermost Enterprise, Professional, or Mattermost Cloud is required. -- Enterprise customers can also [record calls](https://docs.mattermost.com/collaborate/make-calls.html#record-a-call), enable [live text captions](https://docs.mattermost.com/collaborate/make-calls.html#live-captions-during-calls) during calls, and [transcribe recorded calls](https://docs.mattermost.com/collaborate/make-calls.html#transcribe-recorded-calls). We recommend that Enterprise self-hosted customers looking for group calls beyond 50 concurrent users consider using the [dedicated rtcd service](#the-rtcd-service). -- For Mattermost self-hosted deployments, System admins need to enable and configure the plugin [using the System Console](https://docs.mattermost.com/configure/plugins-configuration-settings.html#calls). The default maximum number of participants is unlimited; however, we recommend a maximum of 50 participants per call. Maximum call participants is configurable by going to **System Console > Plugin Management > Calls > Max call participants**. Call participant limits greatly depends on instance resources. For more details, refer to the [performance section](#performance) below. +- Enterprise customers can also [record calls](https://docs.mattermost.com/collaborate/make-calls.html#record-a-call), enable [live text captions](https://docs.mattermost.com/collaborate/make-calls.html#live-captions-during-calls) during calls, and [transcribe recorded calls](https://docs.mattermost.com/collaborate/make-calls.html#transcribe-recorded-calls). We recommend that Enterprise self-hosted customers looking for group calls beyond 50 concurrent users consider using the [dedicated RTCD service](#when-to-use-rtcd). +- For Mattermost self-hosted deployments, System admins need to enable and configure the plugin [using the System Console](https://docs.mattermost.com/configure/plugins-configuration-settings.html#calls). The default maximum number of participants is unlimited; however, we recommend a maximum of 50 participants per call. Maximum call participants is configurable by going to **System Console > Plugin Management > Calls > Max call participants**. Call participant limits greatly depends on instance resources. For more details, refer to the [Performance Considerations](#performance-considerations) section below. ## Configuration -For Mattermost self-hosted customers, the calls plugin is pre-packaged, installed, and enabled. Configuration to allow end-users to use it can be found in the [System Console](https://docs.mattermost.com/configure/plugins-configuration-settings.html#calls). - -## Modes of operation +For Mattermost self-hosted customers, the calls plugin is pre-packaged, installed, and enabled. Configuration to allow end-users to use it can be found in ``System Console > Plugins > Calls``. -Depending on how the Mattermost server is running, there are several modes under which the Calls plugin can operate. Please refer to the section below on [the rtcd service](#the-rtcd-service) to learn about the `rtcd` and the Selective Forwarding Unit (SFU). +## Deployment Architecture Options -| Mattermost deployment | SFU | SFU deployment | -|-----------------------|-----|----------------| -| Single instance | integrated | | -| Single instance | rtcd | | -| High availability cluster-based | integrated | clustered | -| High availability cluster-based | rtcd | | +Mattermost Calls can be deployed in several configurations: -### Single instance +### Single Instance Deployments -#### Integrated +#### Integrated Mode -This is the default mode when first installing the plugin on a single Mattermost instance setup. The WebRTC service is integrated in the plugin itself and runs alongside the Mattermost server. +The WebRTC service runs within the Calls plugin on the Mattermost server. This is the default mode when first installing the plugin on a single Mattermost instance setup. The WebRTC service is integrated in the plugin itself and runs alongside the Mattermost server. -![A diagram of the integrated configuration model of a single instance.](/images/calls-deployment-image3.png) +![Integrated configuration model of a single instance](../images/calls-deployment-image3.png) -#### rtcd +#### RTCD Mode -An external, dedicated and scalable WebRTC service (`rtcd`) is used to handle all calls media routing. +A dedicated RTCD service handles media routing, reducing load on the Mattermost server. -![A diagram of a Web RTC deployment configuration.](/images/calls-deployment-image7.png) +![Web RTC deployment configuration](../images/calls-deployment-image7.png) -### High availability cluster-based +### High Availability Deployments -#### Clustered +#### Clustered Mode This is the default mode when running the plugin in a high availability cluster-based deployment. Every Mattermost node will run an instance of the plugin that includes a WebRTC service. Calls are distributed across all available nodes through the existing load-balancer: a call is hosted on the instance where the initiating websocket connection (first client to join) is made. A single call will be hosted on a single cluster node. -![A diagram of a clustered calls deployment.](/images/calls-deployment-image5.png) +![Clustered calls deployment](../images/calls-deployment-image5.png) -#### rtcd (High Availability) +#### RTCD with High Availability -![A diagram of an rtcd deployment.](/images/calls-deployment-image2.png) +Dedicated RTCD services handle media routing for high availability. -## Performance +![RTCD deployment with high availability](../images/calls-deployment-image2.png) -Calls performance primarily depends on two resources: CPU and bandwidth (both network latency and overall throughput). The final consumption exhibits quadratic growth with the number of clients transmitting and receiving media. +### Kubernetes Deployments -As an example, a single call with 10 participants of which two are unmuted (transmitting voice data) will generally consume double the resources than the same call with a single participant unmuted. What ultimately counts towards performance is the overall number of concurrent media flows (in/out) across the server. +RTCD is the only officially supported approach for Kubernetes deployments. For detailed information on deploying Mattermost Calls in Kubernetes environments, including Helm chart configurations, resource requirements, and scaling considerations, see the [Calls Deployment on Kubernetes](calls-kubernetes.md) guide. -### Benchmarks +## When to Use RTCD -Here are the results from internally conducted performance and ceiling tests on a dedicated `rtcd` instance: - -#### Deployment specifications - -- 1x r6i.large nginx proxy -- 3x c5.large MM app nodes (HA) -- 2x db.x2g.xlarge RDS Aurora MySQL v8 (one writer, one reader) -- 1x (c7i.xlarge, c7i.2xlarge, c7i.4xlarge) RTCD -- 2x c7i.2xlarge load-test agents - -#### App specifications - -- Mattermost v9.6 -- Mattermost Calls v0.28.0 -- RTCD v0.16.0 -- load-test agent v0.28.0 - -#### Media specifications - -- Speech sample bitrate: 80Kbps -- Screen sharing sample bitrate: 1.6Mbps - -#### Results - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CallsParticipants/callUnmuted/callScreen sharingCPU (avg)Memory (avg)Bandwidth (in/out)Instance type (RTCD)
110002no47%1.46GB1Mbps / 194Mbpsc7i.xlarge
18001yes64%1.43GB2.7Mbps / 1.36Gbpsc7i.xlarge
110001yes79%1.54GB2.9Mbps / 1.68Gbpsc7i.xlarge
101001yes74%1.56GB18.2Mbps / 1.68Gbpsc7i.xlarge
100102no49%1.46GB18.7Mbps / 175Mbpsc7i.xlarge
100101yes84%1.73GB171Mbps / 1.53Gbpsc7i.xlarge
110002no20%1.44GB1.4Mbps / 194Mbpsc7i.2xlarge
110002yes49%1.53GB3.6Mbps / 1.79Gbpsc7i.2xlarge
210001yes73%2.38GB5.7Mbps / 3.06Gbpsc7i.2xlarge
100102yes60%1.74GB181Mbps / 1.62Gbpsc7i.2xlarge
150101yes72%2.26GB257Mbps / 2.30Gbpsc7i.2xlarge
150102yes79%2.34GB271Mbps / 2.41Gbpsc7i.2xlarge
250102no58%2.66GB47Mbps / 439Mbpsc7i.2xlarge
100022no78%2.31GB178Mbps / 195Mbpsc7i.2xlarge
210002yes41%2.6GB7.23Mbps / 3.60Gbpsc7i.4xlarge
310002yes63%3.53GB10.9Mbps / 5.38Gbpsc7i.4xlarge
410002yes83%4.40GB14.5Mbps / 7.17Gbpsc7i.4xlarge
250102yes79%3.49GB431Mbps / 3.73Gbpsc7i.4xlarge
50022yes71%2.54GB896Mbps / 919Mbpsc7i.4xlarge
+This section will help you understand when and why your organization would want to use the dedicated RTCD service. ```{note} -- The tests focused on a single, vertically scaled RTCD instance to understand the processing limits within a single node. Scaling the RTCD service horizontally should be sufficient to support a higher number of calls. -- RTCD processes were executed with all performance profiling enabled (including block and mutex). This resulted in some computational overhead. -- Both speech and screen samples have slightly higher bitrates than the average produced by a real client (e.g., a browser). This gives us some safety margin over real-world deployments. +RTCD is a standalone service, which adds operational complexity, maintenance costs, and requires an Enterprise license. For those who are evaluating Calls, and for many small instances of Mattermost, the integrated SFU (the one included in the Calls plugin) may be sufficient initially. ``` -### Dedicated service - -For Enterprise customers we offer a way to offload performance costs through a [dedicated service](https://github.com/mattermost/rtcd) that can be used to further scale up calls. - -### Load testing - -We provide a [load-test tool](https://github.com/mattermost/mattermost-plugin-calls/tree/main/lt) that can be used to simulate and measure the performance impact of calls. - -### Monitoring - -Both the plugin and the external `rtcd` service expose some Prometheus metrics to monitor performance. We provide an [official dashboard](https://grafana.com/grafana/dashboards/23225-mattermost-calls-performance-monitoring/) that can be imported in Grafana. You can refer to [Performance monitoring](https://docs.mattermost.com/scale/deploy-prometheus-grafana-for-performance-monitoring.html) for more information on how to set up Prometheus and visualize metrics through Grafana. - -#### Calls plugin metrics - -Metrics for the calls plugin are exposed through the `/plugins/com.mattermost.calls/metrics` subpath under the existing Mattermost server metrics endpoint. This is controlled by the [Listen address for performance](https://docs.mattermost.com/configure/environment-configuration-settings.html#listen-address-for-performance) configuration setting. It defaults to port `8067`. - -```{note} -- The [Metrics plugin](https://docs.mattermost.com/scale/collect-performance-metrics.html) collects application-level metrics only and does not make system or OS-level calls. As a result, data typically derived from system-level metrics may be missing in the Grafana panel. -- On Mattermost versions prior to v9.5, plugin metrics were exposed through the public `/plugins/com.mattermost.calls/metrics` API endpoint controlled by the [Web server listen address](https://docs.mattermost.com/configure/environment-configuration-settings.html#web-server-listen-address) configuration setting. This defaults to port `8065`. -``` - -**Process** - -- `mattermost_plugin_calls_process_cpu_seconds_total`: Total user and system CPU time spent in seconds. -- `mattermost_plugin_calls_process_max_fds`: Maximum number of open file descriptors. -- `mattermost_plugin_calls_process_open_fds`: Number of open file descriptors. -- `mattermost_plugin_calls_process_resident_memory_bytes`: Resident memory size in bytes. -- `mattermost_plugin_calls_process_virtual_memory_bytes`: Virtual memory size in bytes. - -**WebRTC connection** - -- `mattermost_plugin_calls_rtc_conn_states_total`: Total number of RTC connection state changes. -- `mattermost_plugin_calls_rtc_errors_total`: Total number of RTC errors. -- `mattermost_plugin_calls_rtc_rtp_bytes_total`: Total number of sent/received RTP packets in bytes. - - - Note: removed as of v0.16.0 - -- `mattermost_plugin_calls_rtc_rtp_packets_total`: Total number of sent/received RTP packets. - - - Note: removed as of v0.16.0 - -- `mattermost_plugin_calls_rtc_rtp_tracks_total`: Total number of incoming/outgoing RTP tracks. - - - Note: added as of v0.16.0 - -- `mattermost_plugin_calls_rtc_sessions_total`: Total number of active RTC sessions. - -**Application** - -- `mattermost_plugin_calls_app_handlers_time_bucket`: Time taken to execute app handlers. - - - `mattermost_plugin_calls_app_handlers_time_sum` - - - `mattermost_plugin_calls_app_handlers_time_count` - -**Database** - -- `mattermost_plugin_calls_store_ops_total`: Total number of db store operations. -- `mattermost_plugin_calls_store_methods_time_bucket`: Time taken to execute store methods. - - - `mattermost_plugin_calls_store_methods_time_sum` - - - `mattermost_plugin_calls_store_methods_time_count` -- `mattermost_plugin_calls_cluster_mutex_grab_time_bucket`: Time taken to grab global mutexes. - - - `mattermost_plugin_calls_cluster_mutex_grab_time_sum` - - - `mattermost_plugin_calls_cluster_mutex_grab_time_count` -- `mattermost_plugin_calls_cluster_mutex_locked_time_bucket`: Time spent locked in global mutexes. - - - `mattermost_plugin_calls_cluster_mutex_locked_time_sum` - - - `mattermost_plugin_calls_cluster_mutex_locked_time_count` - -**WebSocket** +The RTCD service is the recommended way to host Calls for the following reasons: -- `mattermost_plugin_calls_websocket_connections_total`: Total number of active WebSocket connections. -- `mattermost_plugin_calls_websocket_events_total`: Total number of WebSocket events. +- **Performance of the main Mattermost server(s)**: When the Calls plugin runs the SFU, calls traffic is added to the processing load of the server running the rest of your Mattermost services. If Calls traffic spikes, it can negatively affect the responsiveness of these services. Using an RTCD service isolates the calls traffic processing to those RTCD instances, and also reduces costs by minimizing CPU usage spikes. -**Jobs** +- **Performance, scalability, and stability of the Calls product**: If Calls traffic spikes, or more overall capacity is needed, RTCD servers can be added to balance the load. As an added benefit, if the Mattermost traffic spikes, or if a Mattermost instance needs to be restarted, those people in a current call will not be affected - current calls won't be dropped. -- `mattermost_plugin_calls_jobs_live_captions_new_audio_len_ms_bucket`: Duration (in ms) of new audio transcribed for live captions. + Some caveats apply here. WebSocket events (for example: emoji reactions, hand raising, muting/unmuting) will not be transmitted while the main Mattermost server is down. But the call itself will continue while the main server restarts. - - `mattermost_plugin_calls_jobs_live_captions_new_audio_len_ms_sum` +- **Kubernetes deployments**: In a Kubernetes deployment, RTCD is strongly recommended; it is currently the only officially supported way to run Calls. - - `mattermost_plugin_calls_jobs_live_captions_new_audio_len_ms_count` -- `mattermost_plugin_calls_jobs_live_captions_pktPayloadCh_buf_full`: Total packets of audio data dropped due to full channel. -- `mattermost_plugin_calls_jobs_live_captions_window_dropped`: Total windows of audio data dropped due to pressure on the transcriber. +- **Technical benefits**: The dedicated RTCD service has been optimized and tuned at the system/network level for real-time audio/video traffic, where latency is generally more important than throughput. -#### WebRTC service metrics +In general, RTCD is the preferred solution for a performant and scalable deployment. With RTCD, the Mattermost server will be minimally impacted when hosting a high number of calls. -Metrics for the `rtcd` service are exposed through the `/metrics` API endpoint under the `rtcd` API listener controlled by the `api.http.listen_address` configuration setting. It defaults to port `8045`. +For detailed RTCD setup instructions, see the [RTCD Setup and Configuration](calls-rtcd-setup.md) guide. -**Process** +## Call Recording and Transcription -- `rtcd_process_cpu_seconds_total`: Total user and system CPU time spent in seconds. -- `rtcd_plugin_calls_process_max_fds`: Maximum number of open file descriptors. -- `rtcd_plugin_calls_process_open_fds`: Number of open file descriptors. -- `rtcd_plugin_calls_process_resident_memory_bytes`: Resident memory size in bytes. -- `rtcd_plugin_calls_process_virtual_memory_bytes`: Virtual memory size in bytes. +For call recording and transcription, you need to: -**WebRTC Connection** +1. Deploy the `calls-offloader` service +2. Configure the service URL in the System Console +3. Enable call recordings and/or transcriptions in the plugin settings -- `rtcd_rtc_conn_states_total`: Total number of RTC connection state changes. -- `rtcd_rtc_errors_total`: Total number of RTC errors. -- `rtcd_rtc_rtp_bytes_total`: Total number of sent/received RTP packets in bytes. -- `rtcd_rtc_rtp_packets_total`: Total number of sent/received RTP packets. -- `rtcd_rtc_rtp_tracks_total`: Total number of incoming/outgoing RTP tracks. -- `rtcd_rtc_sessions_total`: Total number of active RTC sessions. -- `rtcd_rtc_rtp_tracks_writes_time_bucket`: Time taken to write to outgoing RTP tracks. +For detailed setup instructions, see the [Calls Offloader Setup and Configuration](calls-offloader-setup.md) guide. - - `rtcd_rtc_rtp_tracks_writes_time_sum` +## Air-Gapped Deployments - - `rtcd_rtc_rtp_tracks_writes_time_count` - -**WebSocket** - -- `rtcd_ws_connections_total`: Total number of active WebSocket connections. -- `rtcd_ws_messages_total`: Total number of received/sent WebSocket messages. - -#### Configuration - -A sample Prometheus configuration to scrape both plugin and `rtcd` metrics could look like this: - -``` -scrape_configs: -- job_name: node - static_configs: - - targets: ['rtcd-0:9100','rtcd-1:9100', 'calls-offloader-1:9100', 'calls-offloader-2:9100'] -- job_name: calls - metrics_path: /plugins/com.mattermost.calls/metrics - static_configs: - - targets: ['app-0:8067','app-1:8067','app-2:8067'] -- job_name: rtcd - static_configs: - - targets: ['rtcd-0:8045', 'rtcd-1:8045'] -``` - -### System tunings - -If you want to host many calls or calls with a large number of participants, take a look at the following platform specific (Linux) tunings (this is the only officially supported target for the plugin right now): - -``` -# Setting the maximum buffer size of the receiving UDP buffer to 16MB -net.core.rmem_max = 16777216 - -# Setting the maximum buffer size of the sending UDP buffer to 16MB -net.core.wmem_max = 16777216 - -# Allow to allocate more memory as needed for more control messages that need to be sent for each socket connected -net.core.optmem_max = 16777216 -``` - -## The rtcd service - -```{include} ./calls-rtcd-ent-only.md -``` - -The Calls plugin has a built-in [Selective Forwarding Unit (SFU)](https://bloggeek.me/webrtcglossary/sfu/) to route audio and screensharing data. This is the `integrated` option described in the [Modes of operation](#modes-of-operation) section above. But this SFU functionality can be deployed separately as an external `rtcd` instance. - -### Reasons to use the `rtcd` service - -This section will help you understand when and why your organization would want to use `rtcd`. - -```{note} -`rtcd` is a standalone service, which adds operational complexity, maintenance costs, and requires an enterprise licence. For those who are evaluating Calls, and for many small instances of Mattermost, the integrated SFU (the one included in the Calls plugin) may be sufficient initially. -``` - -The `rtcd` service is the recommended way to host Calls for the following reasons: - -- **Performance of the main Mattermost server(s).** When the Calls plugin runs the SFU, calls traffic is added to the processing load of the server running the rest of your Mattermost services. If Calls traffic spikes, it can negatively affect the responsiveness of these services. Using an rtcd service isolates the calls traffic processing to those rtcd instances, and also reduces costs by minimizing CPU usage spikes. - -- **Performance, scalability, and stability of the Calls product.** If Calls traffic spikes, or more overall capacity is needed, `rtcd` servers can be added to balance the load. As an added benefit, if the Mattermost traffic spikes, or if a Mattermost instance needs to be restarted, those people in a current call will not be affected - current calls won't be dropped. - -Some caveats apply here. Web socket events (for example: emoji reactions, hand raising, muting/unmuting) will not be transmitted while the main Mattermost server is down. But the call itself will continue while the main server restarts. - -- **Kubernetes deployments.** In a Kubernetes deployment, `rtcd` is strongly recommended; it is currently the only officially supported way to run Calls. -- **Technical benefits.** The dedicated `rtcd` service has been optimized and tuned at the system/network level for real-time audio/video traffic, where latency is generally more important than throughput. - -In general, `rtcd` is the preferred solution for a performant and scalable deployment. With `rtcd`, the Mattermost server will be minimally impacted when hosting a high number of calls. - -See the [Mattermost rtcd repository documentation](https://github.com/mattermost/rtcd/blob/master/README.md) on GitHub for details on [how to run calls through the service](https://github.com/mattermost/rtcd/blob/master/docs/getting_started.md), as well as: - -- [Key implementation details](https://github.com/mattermost/rtcd/blob/master/docs/implementation.md) -- [Project structure](https://github.com/mattermost/rtcd/blob/master/docs/project_structure.md) -- [Configuration overrides](https://github.com/mattermost/rtcd/blob/master/docs/env_config.md) -- [Authentication flow](https://github.com/mattermost/rtcd/blob/master/docs/security.md) - -### Horizontal scalability - -The supported way to enable horizontal scalability for Calls is through a form of DNS based load balancing. This can be achieved regardless of how the `rtcd` service is deployed (bare bone instance, Kubernetes, or an alternate way). - -In order for this to work, the [RTCD Service URL](https://docs.mattermost.com/configure/plugins-configuration-settings.html#rtcd-service-url) should point to a hostname that resolves to multiple IP addresses, each pointing to a running `rtcd` instance. The Mattermost Calls plugin will then automatically distribute calls amongst the available hosts. - -The expected requirements are the following: - -- When a new `rtcd` instance is deployed, it should be added to the DNS record. The plugin side will then be able to pick it up and start assigning calls to the new host. -- If a `rtcd` instance goes down, it should be removed from the DNS record. The plugin side can then detect the change and stop assigning new calls to that host. - -```{note} -- Load balancing is done at the call level. This means that a single call will always live on a single `rtcd` instance. -- There's currently no support for spreading sessions belonging to the same call across a fleet of instances. -``` - -## Configure recording, transcriptions, and live captions - -Before you can start recording, transcribing, and live captioning calls, you need to configure the `calls-offloader` job service. See the [calls-offloader](https://github.com/mattermost/calls-offloader/blob/master/docs/getting_started.md) documentation on GitHub for details on deploying and running this service. [Performance and scalability recommendations](https://github.com/mattermost/calls-offloader/blob/master/docs/performance.md) related to this service are also available on GitHub. - -```{note} -If deploying the service in a Kubernetes cluster, refer to the later section on [Helm charts](#helm-charts). -``` - -Once the `calls-offloader` service is running, recordings should be explicitly enabled through the [Enable call recordings](https://docs.mattermost.com/configure/plugins-configuration-settings.html#enable-call-recordings) config setting and the service's URL should be configured using [Job service URL](https://docs.mattermost.com/configure/plugins-configuration-settings.html#job-service-url). - -Call transcriptions can be enabled through the [Enable call transcriptions](https://docs.mattermost.com/configure/plugins-configuration-settings.html#enable-call-transcriptions) configuration setting. - -Live captions can be enabled through the [Enable live captions](https://docs.mattermost.com/configure/plugins-configuration-settings.html#enable-live-captions) configuration setting. - -```{note} -- The call transcriptions functionality is available starting in Calls version v0.22.0. -- The live captions functionality is available starting in Calls version v0.26.2. -``` - -## Kubernetes deployments - -The Calls plugin has been designed to integrate well with Kubernetes to offer improved scalability and control over the deployment. - -This is a sample diagram showing how the `rtcd` standalone service can be deployed in a Kubernetes cluster: - -![A diagram of calls deployed in a Kubernetes cluster.](/images/calls-deployment-kubernetes.png) - -If Mattermost isn't deployed in a Kubernetes cluster, and you want to use this deployment type, see the [Deploy Mattermost on Kubernetes](https://docs.mattermost.com/install/install-kubernetes.html) documentation. - -### Helm Charts - -The recommended way to deploy Calls related components and services in a Kubernetes deployment is to use the officially provided Helm charts. Related documentation including detailed information on how to deploy these services can be found in our `mattermost-helm` repository: - -- [rtcd Helm chart](https://github.com/mattermost/mattermost-helm/tree/master/charts/mattermost-rtcd) - -- [calls-offloader Helm chart](https://github.com/mattermost/mattermost-helm/tree/master/charts/mattermost-calls-offloader) - -### Limitations - -Due to the inherent complexities of hosting a WebRTC service, some limitations apply when deploying Calls in a Kubernetes environment. - -One key requirement is that each `rtcd` process live in a dedicated Kubernetes node. This is necessary to forward the data correctly while allowing for horizontal scaling. Data should generally not go through a standard ingress but directly to the pod running the `rtcd` process. - -The general recommendation is to expose one external IP address per `rtcd` instance (Kubernetes node). This makes it simpler to scale as the application is able to detect its own external address (through STUN) and advertise it to clients to achieve connectivity with minimal configuration. - -If, for some reason, exposing multiple IP addresses is not possible in your environment, port mapping (NAT) can be used. In this scenario different ports are used to map the respective `rtcd` nodes behind the single external IP. Example: +Mattermost Calls can function in air-gapped environments. Exposing Calls to the public internet is only necessary when users need to connect from outside the local network, and no existing method supports that connection. In such setups: -```sh -EXT_IP:8443 -> rtcdA:8443 -EXT_IP:8444 -> rtcdB:8443 -EXT_IP:8445 -> rtcdC:8443 -``` +- Users should connect from within the private/local network. This can be done on-premises, through a VPN, or via virtual machines. +- Configuring a STUN server is unnecessary, as all connections occur within the local network. +- The [ICE Host Override](https://docs.mattermost.com/configure/plugins-configuration-settings.html#ice-host-override) configuration setting can be optionally set with a local IP address (e.g., 192.168.1.45), depending on the specific network configuration and topology. +- For call recording and transcription in air-gapped environments, see the [Air-Gapped Deployments](calls-offloader-setup.md#air-gapped-deployments) section in the Calls Offloader Setup documentation. -This case requires a couple of extra configurations: +## Performance Considerations -- NAT mappings need to be in place for every `rtcd` node. This is usually done at the ingress point (e.g., ELB, NLB, etc). -- The `RTCD_RTC_ICEHOSTPORTOVERRIDE` config should be used to pass a full mapping of node IPs and their respective port. - - Example: `RTCD_RTC_ICEHOSTPORTOVERRIDE=rtcdA_IP/8443,rtcdB_IP/8444,rtcdC_IP/8445` -- The `RTCD_RTC_ICEHOSTOVERRIDE` should be used to set the external IP address. +Calls performance primarily depends on: -```{note} -One option to limit these static mappings is to reduce the size of the local subnet (e.g., to `/29`). -``` +- **CPU resources**: More participants require more processing power +- **Network bandwidth**: Both incoming and outgoing traffic increases with participant count. Due to the nature of the service, the bottleneck is always going to be the outgoing/egress path +- **Active speakers**: Unmuted participants require significantly more resources +- **Presenters**: Screen sharing participants require even more resources than active speakers -## Frequently asked questions +For detailed performance metrics, benchmarks, and monitoring guidance, see the [Calls Metrics and Monitoring](calls-metrics-monitoring.md) guide. -### Is there encryption? +## Frequently Asked Questions +**Is calls traffic encrypted?** Media (audio/video) is encrypted using security standards as part of WebRTC. It's mainly a combination of DTLS and SRTP. It's not e2e encrypted in the sense that in the current design all media needs to go through Mattermost which acts as a media router and has complete access to it. Media is then encrypted back to the clients so it's secured during transit. In short: only the participant clients and the Mattermost server have access to unencrypted call data. -### Are there any third-party services involved? +**Are there any third-party services involved?** +Only a Mattermost STUN server (`stun.global.calls.mattermost.com`) is used by default. This can be removed if you set the ICE Host Override configuration. -The only external service used is a Mattermost official STUN server (`stun.global.calls.mattermost.com`) which is configured as default. This is primarily used to find the public address of the Mattermost instance if none is provided through the [ICE Host Override](https://docs.mattermost.com/configure/plugins-configuration-settings.html#ice-host-override) option. The only information sent to this service is the IP addresses of clients connecting as no other traffic goes through it. It can be removed in cases where the [ICE Host Override](https://docs.mattermost.com/configure/plugins-configuration-settings.html#ice-host-override) setting is provided. - -```{note} -In air-gapped deployments, using STUN servers is not necessary since all connections remain within the local network. -``` +**Is using UDP a requirement?** +UDP is recommended protocol to serve real-time media as it allows for the lowest latency between peers, but TCP fallback is supported since plugin version 0.17 and RTCD version 0.11. -### Is using UDP a requirement? - -Yes, UDP is the recommended protocol to serve real-time media as it allows for the lowest latency between peers. However, there are a couple of possible solutions to cover clients that due to limitations or strict firewalls are unable to use UDP: +If clients are unable to connect using UDP (due to limitations or strict firewalls), you have a few options: - Since plugin version 0.17 and `rtcd` version 0.11 the RTC service will listen for TCP connections in addition to UDP ones. If configured correctly (e.g. using commonly allowed ports such as 80 or 443) it's possible to have clients connect directly through TCP when unable to do it through the preferred UDP channel. - Run calls through an external TURN server that listens on TCP and relays all media traffic between peers. However, this is a sub-optimal solution that should be avoided if possible as it will introduce extra latency along with added infrastructural cost. -### Do I need a TURN server? - -TURN becomes necessary when you expect to have clients that are unable to connect through the configured UDP port. This can happen due to very restrictive firewalls that either block non standard ports even in the outgoing direction or don't allow the use of the UDP protocol altogether (e.g. some corporate firewalls). In such cases TURN is needed to allow connectivity. +**Do I need a TURN server?** +Only if clients are behind restrictive firewalls that block UDP. We recommend (and officially support) [coturn](https://github.com/coturn/coturn) if needed. -We officially support and recommend using [coturn](https://github.com/coturn/coturn) for a stable and performant TURN service implementation. +**Can RTCD traffic be kept internal?** +Yes, and it's recommended. Only the media ports need to be accessible to end-users. -### How will this work with an existing reverse proxy sitting in front of Mattermost? +**How will this work with an existing reverse proxy sitting in front of Mattermost?** -Generally clients should connect directly to either Mattermost or, if deployed, the dedicated `rtcd` service through the configured UDP port . However, it's also possible to route the traffic through an existing load balancer as long as this has support for routing the UDP protocol (e.g. nginx). Of course this will require additional configuration and potential changes to how the plugin is run as it won't be possible to load balance the UDP flow across multiple instances like it happens for HTTP. +Generally clients should connect directly to either Mattermost or, if deployed, the dedicated `rtcd` service through the configured UDP port. However, it's also possible to route the traffic through an existing load balancer as long as this has support for routing the UDP protocol (e.g. nginx). Of course this will require additional configuration and potential changes to how the plugin is run as it won't be possible to load balance the UDP flow across multiple instances like it happens for HTTP. -### Do calls require a dedicated server to work or can they run alongside Mattermost? +**Do calls require a dedicated server to work or can they run alongside Mattermost?** The plugin can function in different modes. By default calls are handled completely by the plugin which runs as part of Mattermost. It's also possible to use a dedicated service to offload the computational and bandwidth costs and scale further (Enterprise only). -### Can the traffic between Mattermost and `rtcd` be kept internal or should it be opened to the public? +See [RTCD Setup and Configuration](calls-rtcd-setup.md) for more details on the dedicated RTCD service. + +**Can the traffic between Mattermost and `rtcd` be kept internal or should it be opened to the public?** When possible, it's recommended to keep communication between the Mattermost cluster and the dedicated `rtcd` service under the same private network as this can greatly simplify deployment and security. There's no requirement to expose `rtcd`'s HTTP API to the public internet. -### Can Calls be rolled out on a per-channel basis? +**Can Calls be rolled out on a per-channel basis?** ```{include} ../_static/badges/selfhosted-only.md ``` Yes. Mattermost system admins running self-hosted deployments can enable or disable call functionality per channel. Once [test mode](https://docs.mattermost.com/configure/plugins-configuration-settings.html#test-mode) is enabled for Mattermost Calls: -- Select **Enable calls** for each channel where you want Calls enabled -- Select **Disable calls** for all channels where you want Calls disabled. +1. **Navigate to the channel** where you want to enable or disable Calls +2. **Access the channel menu** by clicking the channel name at the top of the channel +3. **Select the Calls option** from the dropdown menu: + - Select **Enable calls** for each channel where you want Calls enabled + - Select **Disable calls** for all channels where you want Calls disabled + +![Channel menu showing Enable/Disable calls options](../images/calls-channel-enable-disable.png) Once Calls is enabled for specific channels, users can start making calls in those channels. @@ -746,48 +297,11 @@ When [test mode](https://docs.mattermost.com/configure/plugins-configuration-set ## Troubleshooting -### Connectivity issues - -If calls are failing to connect or timing out, it's likely there could be a misconfiguration at either the plugin config or networking level. - -For example, the [RTC Server Port (UDP)](https://docs.mattermost.com/configure/plugins-configuration-settings.html#rtc-server-port-udp) or the [RTC Server Port (TCP)](https://docs.mattermost.com/configure/plugins-configuration-settings.html#rtc-server-port-tcp) may not be open or forwarded correctly. - -#### Connectivity checks - -An easy way to check whether data can go through is to perform some tests using the `netcat` command line tool. - -On the host running Calls (could be the Mattermost instance itself or the one running `rtcd` depending on the chosen setup), run the following: - -```sh -nc -l -u -p 8443 -``` - -On the client side (i.e., the machine you would normally use to run the Mattermost desktop app or browser), run the following: +For comprehensive troubleshooting steps and debugging techniques, please refer to the [Calls Troubleshooting](calls-troubleshooting.md) guide. -```sh -nc -v -u HOST_IP 8443 -``` - -If connection succeeds, you should be able to send and receive text messages by typing and hitting enter on either side. - -```{note} -`HOST_IP` should generally be the public (client facing) IP of the Mattermost -(or `rtcd`) instance hosting the calls. When set, it should be the value of the [ICE Host Override](https://docs.mattermost.com/configure/plugins-configuration-settings.html#ice-host-override) -config setting. - -`8443` should be changed with the port configured in [RTC Server Port](https://docs.mattermost.com/configure/plugins-configuration-settings.html#rtc-server-port-udp). - -The same checks can be performed to test connectivity through the TCP port using the same commands with `-u` flag removed. -``` - -#### Network packets debugging - -A more advanced way to debug networking issues is to use the `tcpdump` command line utility to temporaily monitor network packets flowing in and out of the instance hosting calls. - -On the server side, run the following: - -```sh -sudo tcpdump -n port 8443 -``` +## Next Steps -This command will output information (i.e. source and destination addresses) for all the network packets being sent or received through port `8443`. This is a good way to check whether data is getting in and out of the instance and can be used to quickly identify network configuration issues. +1. For detailed setup instructions, see [RTCD Setup and Configuration](calls-rtcd-setup.md) +2. For monitoring guidance, see [Calls Metrics and Monitoring](calls-metrics-monitoring.md) +3. If you encounter issues, see [Calls Troubleshooting](calls-troubleshooting.md) +4. For Kubernetes deployments, see [Calls Deployment on Kubernetes](calls-kubernetes.md) diff --git a/source/configure/calls-kubernetes.md b/source/configure/calls-kubernetes.md new file mode 100644 index 00000000000..a8519f1dfc8 --- /dev/null +++ b/source/configure/calls-kubernetes.md @@ -0,0 +1,157 @@ +# Deploy Calls Kubernetes + +```{include} ../_static/badges/allplans-cloud-selfhosted.md +``` + +This guide provides detailed information for deploying Mattermost Calls on Kubernetes environments. + +## Overview + +Mattermost Calls has been designed to integrate well with Kubernetes to offer improved scalability and control over the deployment. For Kubernetes deployments, the RTCD service is strongly recommended and is the only officially supported approach. + +## Architecture + +![Calls deployed in a Kubernetes cluster](../images/calls-deployment-kubernetes.png) + +This diagram shows how the RTCD standalone service can be deployed in a Kubernetes cluster. In this architecture: + +1. Calls traffic is handled by dedicated RTCD pods +2. RTCD services are exposed through load balancers +3. Scaling is managed through Kubernetes deployment configurations +4. Call recording and transcription is handled by the calls-offloader service (see [Calls Offloader Setup and Configuration](calls-offloader-setup.md)) + +If Mattermost isn't already deployed in your Kubernetes cluster and you want to use this deployment type, visit the [Kubernetes operator guide](/install/mattermost-kubernetes-operator.md). + +## Helm Chart Deployment + +The recommended way to deploy Calls-related components in a Kubernetes environment is to use the officially provided Helm charts: + +### RTCD Helm Chart + +The RTCD Helm chart deploys the RTCD service needed for call media handling: + +```bash +helm repo add mattermost https://helm.mattermost.com +helm repo update + +helm install mattermost-rtcd mattermost/mattermost-rtcd \ + --set ingress.enabled=true \ + --set ingress.host=rtcd.example.com \ + --set service.annotations."service\\.beta\\.kubernetes\\.io/aws-load-balancer-backend-protocol"=udp \ + --set rtcd.ice.hostOverride=rtcd.example.com +``` + +For complete configuration options, see the [RTCD Helm chart documentation](https://github.com/mattermost/mattermost-helm/tree/master/charts/mattermost-rtcd). + +### Calls-Offloader Helm Chart + +If you need call recording and transcription capabilities, deploy the calls-offloader service: + +```bash +helm install mattermost-calls-offloader mattermost/mattermost-calls-offloader \ + --set ingress.enabled=true \ + --set ingress.host=calls-offloader.example.com +``` + +For complete configuration options, see the [Calls-Offloader Helm chart documentation](https://github.com/mattermost/mattermost-helm/tree/master/charts/mattermost-calls-offloader). + +## Kubernetes-Specific Configuration + +### Network Configuration + +For Kubernetes deployments, you need to ensure specific connectivity paths: + +1. **Client to RTCD connectivity**: UDP traffic on port 8443 is properly routed from clients to RTCD pods (for media) +2. **Mattermost to RTCD API connectivity**: TCP traffic on port 8045 must have a clear connectivity path from Mattermost pods to RTCD pods (for API communication) +3. **Client to RTCD TCP fallback**: TCP traffic on port 8443 can reach RTCD pods (for fallback connections when UDP fails) +4. **Load balancer configuration**: Load balancers must be properly configured to handle UDP traffic routing to RTCD pods +5. **Network policies**: Network policies must allow the required communications between Mattermost and RTCD services + +### Resource Requirements + +For optimal performance in Kubernetes environments: + +1. **CPU**: At least 2 CPU cores per RTCD pod +2. **Memory**: At least 1GB RAM per RTCD pod +3. **Network**: Sufficient bandwidth for expected call volume (see benchmarks) + +We recommend setting resource limits and requests in your deployment: + +```yaml +resources: + requests: + cpu: 1000m + memory: 1Gi + limits: + cpu: 2000m + memory: 2Gi +``` + +### Scaling Considerations + +Horizontal scaling of RTCD pods is possible, but remember: + +1. Each call is hosted entirely on a single RTCD pod +2. DNS-based load balancing should be used to distribute calls among pods +3. Health checks should ensure that only healthy pods receive new calls +4. Calls remain on their assigned pod for their entire duration + +### Limitations + +Due to the inherent complexities of hosting a WebRTC service, some limitations apply when deploying Calls in a Kubernetes environment. + +One key requirement is that each `rtcd` process must live in a dedicated Kubernetes node. This is necessary to forward the data correctly while allowing for horizontal scaling. Data should generally not go through a standard ingress but directly to the pod running the `rtcd` process. + +The general recommendation is to expose one external IP address per `rtcd` instance (Kubernetes node). This makes it simpler to scale as the application is able to detect its own external address (through STUN) and advertise it to clients to achieve connectivity with minimal configuration. + +If, for some reason, exposing multiple IP addresses is not possible in your environment, port mapping (NAT) can be used. In this scenario different ports are used to map the respective `rtcd` nodes behind the single external IP. Example: + +```text +EXT_IP:8443 -> rtcdA:8443 +EXT_IP:8444 -> rtcdB:8443 +EXT_IP:8445 -> rtcdC:8443 +``` + +This case requires a couple of extra configurations: + +* NAT mappings need to be in place for every `rtcd` node. This is usually done at the ingress point (e.g., ELB, NLB, etc). + +* The `RTCD_RTC_ICEHOSTPORTOVERRIDE` config should be used to pass a full mapping of node IPs and their respective port. + + * Example: `RTCD_RTC_ICEHOSTPORTOVERRIDE=rtcdA_IP/8443,rtcdB_IP/8444,rtcdC_IP/8445` + +* The `RTCD_RTC_ICEHOSTOVERRIDE` should be used to set the external IP address. + +```{note} +One option to limit these static mappings is to reduce the size of the local subnet (e.g., to `/29`). +``` + +## Monitoring and Metrics + +We recommend deploying Prometheus and Grafana alongside your Calls deployment: + +1. Configure Prometheus to scrape metrics from both Mattermost and RTCD pods +2. Import the official Mattermost Calls dashboard to Grafana +3. Set up alerts for CPU usage, connection failures, and error rates + +For detailed information on metrics collection and monitoring, see the [Calls Metrics and Monitoring](calls-metrics-monitoring.md) guide. + +## Troubleshooting + +For Kubernetes-specific troubleshooting: + +1. Check pod logs: `kubectl logs -f deployment/mattermost-rtcd` +2. Verify service connectivity: `kubectl port-forward service/mattermost-rtcd 8045:8045` +3. Ensure UDP traffic is properly routed through your ingress/load balancer +4. Verify network policies allow required communication paths + +For detailed troubleshooting steps, see the [Calls Troubleshooting](calls-troubleshooting.md) guide. + +## Other Calls Documentation + +- [Calls Overview](calls-deployment.md): Overview of deployment options and architecture +- [RTCD Setup and Configuration](calls-rtcd-setup.md): Comprehensive guide for setting up the dedicated RTCD service +- [Calls Offloader Setup and Configuration](calls-offloader-setup.md): Setup guide for call recording and transcription +- [Calls Metrics and Monitoring](calls-metrics-monitoring.md): Guide to monitoring Calls performance using metrics and observability +- [Calls Troubleshooting](calls-troubleshooting.md): Detailed troubleshooting steps and debugging techniques +3. If you encounter issues, see [Calls Troubleshooting](calls-troubleshooting.md) \ No newline at end of file diff --git a/source/configure/calls-metrics-monitoring.md b/source/configure/calls-metrics-monitoring.md new file mode 100644 index 00000000000..d5028896d27 --- /dev/null +++ b/source/configure/calls-metrics-monitoring.md @@ -0,0 +1,470 @@ +# Calls Metrics and Monitoring + +```{include} ../_static/badges/ent-only.md +``` + +This guide provides detailed information on monitoring Mattermost Calls performance and health through metrics and observability tools. Effective monitoring is essential for maintaining optimal call quality and quickly addressing any issues that arise. + +- [Metrics overview](#metrics-overview) +- [Setting up monitoring](#setting-up-monitoring) +- [Key metrics to monitor](#key-metrics-to-monitor) +- [Performance baselines](#performance-baselines) +- [Troubleshooting metrics collection](#troubleshooting-metrics-collection) + +## Metrics Overview + +Mattermost Calls provides metrics through Prometheus for both the Calls plugin and the RTCD service. These metrics help track: + +- Active call sessions and participants +- Media track statistics +- Connection states and errors +- Resource utilization (CPU, memory, network) +- WebSocket connections and events + +The metrics are exposed through HTTP endpoints: + +- **Calls Plugin**: `/plugins/com.mattermost.calls/metrics` +- **RTCD Service**: `/metrics` (default) or a configured endpoint + +## Setting Up Monitoring + +### Prerequisites + +To monitor Calls metrics, you'll need: + +1. **Prometheus**: For collecting and storing metrics +2. **Grafana**: For visualizing metrics (optional but recommended) + +### Installing Prometheus + +1. **Download and install Prometheus**: + + Visit the [Prometheus download page](https://prometheus.io/download/) for installation instructions. + +2. **Configure Prometheus** to scrape metrics from all Calls-related services: + + Complete `prometheus.yml` configuration for Calls monitoring: + + ```yaml + global: + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['PROMETHEUS_IP:9090'] + + - job_name: 'mattermost' + metrics_path: /metrics + static_configs: + - targets: ['MATTERMOST_SERVER_IP:8067'] + + - job_name: 'calls-plugin' + metrics_path: /plugins/com.mattermost.calls/metrics + static_configs: + - targets: ['MATTERMOST_SERVER_IP:8067'] + labels: + service_name: 'calls-plugin' + + - job_name: 'rtcd' + metrics_path: /metrics + static_configs: + - targets: ['RTCD_SERVER_IP:8045'] + labels: + service_name: 'rtcd' + + - job_name: 'rtcd-node-exporter' + metrics_path: /metrics + static_configs: + - targets: ['RTCD_SERVER_IP:9100'] + labels: + service_name: 'rtcd' + + - job_name: 'calls-offloader-node-exporter' + metrics_path: /metrics + static_configs: + - targets: ['CALLS_OFFLOADER_SERVER_IP:9100'] + labels: + service_name: 'offloader' + ``` + + Replace the placeholder IP addresses with your actual server addresses: + + - `MATTERMOST_SERVER_IP`: IP address of your Mattermost server + - `RTCD_SERVER_IP`: IP address of your RTCD server + - `CALLS_OFFLOADER_SERVER_IP`: IP address of your calls-offloader server (if deployed) + - `PROMETHEUS_IP`: IP address of your Prometheus server + - **Note**: The configuration above uses the default ports (RTCD: `8045`, Mattermost metrics: `8067`, etc.). Adjust these ports in `prometheus.yml` if you have customized them. + + ```{important} + **Metrics Path**: Ensure the metrics paths are correct. The RTCD service exposes metrics at `/metrics` by default, and the Calls plugin at `/plugins/com.mattermost.calls/metrics`. + ``` + + ```{important} + **Metrics Configuration Notice**: Use the `service_name` labels as shown in the configuration above. These labels help organize metrics in dashboards and enable proper service identification. + ``` + + ```{note} + - **node_exporter**: Optional but recommended for system-level metrics (CPU, memory, disk, network). See [node_exporter setup guide](https://prometheus.io/docs/guides/node-exporter/) for installation instructions. + - **calls-offloader**: Only needed if you have call recording/transcription enabled + ``` + +### Installing Grafana + +1. **Download and install Grafana**: + + Visit the [Grafana download page](https://grafana.com/grafana/download) for installation instructions. + +2. **Configure Grafana** to use Prometheus as a data source: + + - Add a new data source in Grafana + - Select Prometheus as the type + - Enter the URL of your Prometheus server + - Test and save the configuration + +3. **Import the Mattermost Calls dashboard**: + + - Navigate to Dashboards > Import in Grafana + - Enter dashboard ID: `23225` or use the direct link: [Mattermost Calls Performance Monitoring](https://grafana.com/grafana/dashboards/23225-mattermost-calls-performance-monitoring/) + - Select your Prometheus data source, and enter values for the + - Confirm the port used for RTCD metrics (default is `8045`), and the port used for the Calls plugin metrics (default is `8067`) + - Click Import to add the dashboard to your Grafana instance + + ```{note} + The dashboard is also available as JSON source from the [Mattermost performance assets repository](https://github.com/mattermost/mattermost-performance-assets/blob/master/grafana/mattermost-calls-performance-monitoring.json) for manual import or customization. + ``` + +## Key Metrics to Monitor + +### RTCD Metrics + +#### Process Metrics + +These metrics help monitor the health and resource usage of the RTCD process: + +- `rtcd_process_cpu_seconds_total`: Total CPU time spent +- `rtcd_process_open_fds`: Number of open file descriptors +- `rtcd_process_max_fds`: Maximum number of file descriptors +- `rtcd_process_resident_memory_bytes`: Memory usage in bytes +- `rtcd_process_virtual_memory_bytes`: Virtual memory used + +**Interpretation**: + +- High CPU usage (>70%) may indicate the need for additional RTCD instances +- Steadily increasing memory usage might indicate a memory leak +- High number of file descriptors could indicate connection handling issues + +#### WebRTC Connection Metrics + +These metrics track the WebRTC connections and media flow: + +- `rtcd_rtc_conn_states_total{state="X"}`: Count of connections in different states +- `rtcd_rtc_errors_total{type="X"}`: Count of RTC errors by type +- `rtcd_rtc_rtp_tracks_total{direction="X"}`: Count of RTP tracks (incoming/outgoing) +- `rtcd_rtc_sessions_total`: Total number of active RTC sessions + +**Interpretation**: + +- Increasing error counts may indicate connectivity or configuration issues +- Track by state to see if connections are failing to establish or dropping +- Larger track counts require proportionally more CPU and bandwidth + +#### WebSocket Metrics + +These metrics track the signaling channel: + +- `rtcd_ws_connections_total`: Total number of active WebSocket connections +- `rtcd_ws_messages_total{direction="X"}`: Count of WebSocket messages (sent/received) + +**Interpretation**: + +- Connection count should match expected participant numbers +- Unusually high message counts might indicate protocol issues +- Connection drops might indicate network issues + +### Calls Plugin Metrics + +Similar metrics are available for the Calls plugin with the following prefixes: + +- Process metrics: `mattermost_plugin_calls_process_*` +- WebRTC connection metrics: `mattermost_plugin_calls_rtc_*` +- WebSocket metrics: `mattermost_plugin_calls_websocket_*` +- Store metrics: `mattermost_plugin_calls_store_ops_total` + +## Performance Baselines + +The following performance benchmarks provide baseline metrics for RTCD deployments under various load conditions and configurations. + +**Deployment specifications** + +- 1x r6i.large nginx proxy +- 3x c5.large MM app nodes (HA) +- 2x db.x2g.xlarge RDS Aurora MySQL v8 (one writer, one reader) +- 1x (c7i.xlarge, c7i.2xlarge, c7i.4xlarge) RTCD +- 2x c7i.2xlarge load-test agents + +**App specifications** + +- Mattermost v9.6 +- Mattermost Calls v0.28.0 +- RTCD v0.16.0 +- load-test agent v0.28.0 + +**Media specifications** + +- Speech sample bitrate: 80Kbps +- Screen sharing sample bitrate: 1.6Mbps + +**Results** + +Below are the detailed benchmarks based on internal performance testing: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CallsParticipants/callUnmuted/callScreen sharingCPU (avg)Memory (avg)Bandwidth (in/out)Instance type (RTCD)
110002no47%1.46GB1Mbps / 194Mbpsc7i.xlarge
18001yes64%1.43GB2.7Mbps / 1.36Gbpsc7i.xlarge
110001yes79%1.54GB2.9Mbps / 1.68Gbpsc7i.xlarge
101001yes74%1.56GB18.2Mbps / 1.68Gbpsc7i.xlarge
100102no49%1.46GB18.7Mbps / 175Mbpsc7i.xlarge
100101yes84%1.73GB171Mbps / 1.53Gbpsc7i.xlarge
110002no20%1.44GB1.4Mbps / 194Mbpsc7i.2xlarge
110002yes49%1.53GB3.6Mbps / 1.79Gbpsc7i.2xlarge
210001yes73%2.38GB5.7Mbps / 3.06Gbpsc7i.2xlarge
100102yes60%1.74GB181Mbps / 1.62Gbpsc7i.2xlarge
150101yes72%2.26GB257Mbps / 2.30Gbpsc7i.2xlarge
150102yes79%2.34GB271Mbps / 2.41Gbpsc7i.2xlarge
250102no58%2.66GB47Mbps / 439Mbpsc7i.2xlarge
100022no78%2.31GB178Mbps / 195Mbpsc7i.2xlarge
210002yes41%2.6GB7.23Mbps / 3.60Gbpsc7i.4xlarge
310002yes63%3.53GB10.9Mbps / 5.38Gbpsc7i.4xlarge
410002yes83%4.40GB14.5Mbps / 7.17Gbpsc7i.4xlarge
250102yes79%3.49GB431Mbps / 3.73Gbpsc7i.4xlarge
50022yes71%2.54GB896Mbps / 919Mbpsc7i.4xlarge
+ +## Troubleshooting Metrics Collection + +### Verify RTCD Metrics are Being Collected + +To verify that Prometheus is successfully collecting RTCD metrics, use this command: + +```bash +curl http://PROMETHEUS_IP:9090/api/v1/label/__name__/values | jq '.' | grep rtcd +``` + +This command queries Prometheus for all available metric names and filters for RTCD-related metrics. + +If no RTCD metrics appear, check: +1. RTCD is running +2. Prometheus is configured to scrape the RTCD metrics endpoint +3. RTCD metrics port is accessible from Prometheus (default: 8045) + +### Check Prometheus Scrape Targets + +To verify all Calls-related services are being scraped successfully: + +1. Open the Prometheus web interface (typically `http://PROMETHEUS_IP:9090`) +2. Navigate to **Status > Targets** +3. Look for your configured Calls services: + - Mattermost server (for Calls plugin metrics) + - RTCD service + +Each target should show status "UP" in green. If a target shows "DOWN" or errors: +- Verify the service is running +- Check network connectivity between Prometheus and the target +- Verify the metrics endpoint is accessible + +## Other Calls Documentation + +- [Calls Overview](calls-deployment.md): Overview of deployment options and architecture +- [RTCD Setup and Configuration](calls-rtcd-setup.md): Comprehensive guide for setting up the dedicated RTCD service +- [Calls Offloader Setup and Configuration](calls-offloader-setup.md): Setup guide for call recording and transcription +- [Calls Deployment on Kubernetes](calls-kubernetes.md): Detailed guide for deploying Calls in Kubernetes environments +- [Calls Troubleshooting](calls-troubleshooting.md): Detailed troubleshooting steps and debugging techniques + +Configure Prometheus storage accordingly to balance disk usage with retention needs. \ No newline at end of file diff --git a/source/configure/calls-offloader-setup.md b/source/configure/calls-offloader-setup.md new file mode 100644 index 00000000000..420c3fee48b --- /dev/null +++ b/source/configure/calls-offloader-setup.md @@ -0,0 +1,929 @@ +# Calls Offloader Setup and Configuration + +```{include} ../_static/badges/ent-only.md +``` + + +This guide provides detailed instructions for setting up, configuring, and validating the Mattermost calls-offloader service used for call recording and transcription features. + +- [Overview](#overview) +- [Prerequisites](#prerequisites) +- [Installation and deployment](#installation-and-deployment) +- [Configuration](#configuration) +- [Validation and testing](#validation-and-testing) +- [Integration with Mattermost](#integration-with-mattermost) +- [Troubleshooting](#troubleshooting) +- [Air-Gapped Deployments](#air-gapped-deployments) + +## Overview + +The calls-offloader service is a dedicated microservice that handles resource-intensive tasks for Mattermost Calls, including: + +- **Call recording**: Captures audio and screen sharing content from calls +- **Call transcription**: Provides automated transcription of recorded calls +- **Live captions** (Experimental): Real-time transcription during active calls + +By offloading these tasks to a dedicated service, the main Mattermost server and RTCD service can focus on core functionality while maintaining optimal performance. + +## Prerequisites + +Before deploying calls-offloader, ensure you have: + +- A Mattermost Enterprise license +- A properly configured Mattermost Calls deployment (either integrated or with RTCD) +- Docker installed and running (for Docker-based job execution) +- Sufficient storage space for recordings (see [Storage Requirements](#storage-requirements)) +- A server or container environment with adequate resources + +### System Requirements + +For detailed system requirements and performance recommendations, refer to the [calls-offloader performance documentation](https://github.com/mattermost/calls-offloader/blob/master/docs/performance.md). + +### Storage Requirements + +Call recordings can consume significant storage space: + +- Audio-only recordings: ~1MB per minute per participant +- Screen sharing recordings: ~10-50MB per minute depending on content + +## Installation and Deployment + +### Bare Metal or VM Deployment + +1. Download the latest release from the [calls-offloader GitHub repository](https://github.com/mattermost/calls-offloader/releases) + +2. Create the necessary directories: + + ```bash + sudo mkdir -p /opt/calls-offloader/data/db + sudo useradd --system --home /opt/calls-offloader calls-offloader + sudo chown -R calls-offloader:calls-offloader /opt/calls-offloader + ``` + +3. Create a configuration file (`/opt/calls-offloader/config.toml`): + + ```toml + [api] + http.listen_address = ":4545" + http.tls.enable = false + http.tls.cert_file = "" + http.tls.cert_key = "" + security.allow_self_registration = true + security.enable_admin = false + security.admin_secret_key = "" + security.session_cache.expiration_minutes = 1440 + + [store] + data_source = "/opt/calls-offloader/data/db" + + [jobs] + api_type = "docker" + max_concurrent_jobs = 2 + failed_jobs_retention_time = "30d" + image_registry = "mattermost" + + [logger] + enable_console = true + console_json = false + console_level = "INFO" + enable_file = true + file_json = true + file_level = "INFO" + file_location = "/opt/calls-offloader/calls-offloader.log" + enable_color = true + ``` + +4. Create a systemd service file (`/etc/systemd/system/calls-offloader.service`): + + ```ini + [Unit] + Description=Mattermost Calls Offloader Service + After=network.target docker.service + Requires=docker.service + + [Service] + Type=simple + User=calls-offloader + WorkingDirectory=/opt/calls-offloader + ExecStart=/opt/calls-offloader/calls-offloader --config /opt/calls-offloader/config.toml + Restart=always + RestartSec=10 + LimitNOFILE=65536 + + [Install] + WantedBy=multi-user.target + ``` + +5. Enable and start the service: + + ```bash + sudo systemctl daemon-reload + sudo systemctl enable calls-offloader + sudo systemctl start calls-offloader + ``` + +6. Check the service status: + + ```bash + sudo systemctl status calls-offloader + ``` + +7. Verify the service is responding: + + ```bash + curl http://localhost:4545/version + # Example output: + # {"buildDate":"2025-03-10 19:13","buildVersion":"v0.9.2","buildHash":"a4bd418","goVersion":"go1.23.6"} + ``` + +## Configuration + +### API Configuration + +The API section controls how the service accepts requests: + +- **http.listen_address**: The address and port where the service listens (default: `:4545`) +- **http.tls.enable**: Whether to use TLS encryption for the API +- **security.allow_self_registration**: Allow clients to self-register for job management +- **security.enable_admin**: Enable admin functionality +- **security.admin_secret_key**: Secret key for admin authentication (change from default!) + +### Store Configuration + +Controls persistent data storage: + +- **data_source**: Path to directory for storing job metadata and state + +### Jobs Configuration + +Controls job processing behavior: + +- **api_type**: Job execution backend (`docker` or `kubernetes`) +- **max_concurrent_jobs**: Maximum number of simultaneous recording/transcription jobs +- **failed_jobs_retention_time**: How long to keep failed job data before cleanup +- **image_registry**: Docker registry for job runner images (typically `mattermost`) + +### Logger Configuration + +Controls logging output: + +- **enable_console**: Log to console output +- **console_json**: Use JSON format for console logs +- **console_level**: Log level for console (DEBUG, INFO, WARN, ERROR) +- **enable_file**: Log to file +- **file_location**: Path to log file +- **enable_color**: Use colored output for console logs + +### Private Network Configuration + +When the Mattermost deployment is running in a private network, additional configuration may be necessary for the jobs spawned by the calls-offloader service to reach the Mattermost server. + +In such cases, you can override the site URL used by recorder jobs or transcriber jobs to connect to Mattermost by setting the following environment variables on the Mattermost server: + +- **MM_CALLS_RECORDER_SITE_URL**: Override the site URL used by recording jobs +- **MM_CALLS_TRANSCRIBER_SITE_URL**: Override the site URL used by transcription jobs + +Example configuration: + +Create or edit the Mattermost environment file (`/opt/mattermost/config/mattermost.environment`): + +```bash +MM_CALLS_RECORDER_SITE_URL="http://internal-mattermost-server:8065" +MM_CALLS_TRANSCRIBER_SITE_URL="http://internal-mattermost-server:8065" +``` + +Then ensure your Mattermost systemd service references this environment file: + +```ini +[Unit] +Description=Mattermost +After=network.target + +[Service] +Type=notify +EnvironmentFile=/opt/mattermost/config/mattermost.environment +ExecStart=/opt/mattermost/bin/mattermost +TimeoutStartSec=3600 +KillMode=mixed +Restart=always +RestartSec=10 +WorkingDirectory=/opt/mattermost +User=mattermost +Group=mattermost + +[Install] +WantedBy=multi-user.target +``` + +This is particularly useful when: + +- The calls-offloader service runs in a different network segment than clients +- Internal DNS resolution differs from external URLs +- You need to use internal load balancer endpoints for job communication + +## Validation and Testing + +After deploying calls-offloader, validate the installation: + +1. **Check service status**: + + ```bash + # For systemd + sudo systemctl status calls-offloader + ``` + +2. **Test API connectivity**: + + **From the calls-offloader server (localhost test)**: + + ```bash + curl http://localhost:4545/version + # Should return version information + # Example: {"buildDate":"2025-03-10 19:13","buildVersion":"v0.9.2","buildHash":"a4bd418","goVersion":"go1.23.6"} + ``` + + **From the Mattermost server**: + + ```bash + curl http://YOUR_CALLS_OFFLOADER_SERVER:4545/version + # Should return the same version information + # This confirms network connectivity from Mattermost to calls-offloader + ``` + + If the localhost test works but the Mattermost server test fails, check: + + - Firewall rules or SELinux policies on the calls-offloader server (port 4545 must be accessible) + - Network connectivity between Mattermost and calls-offloader servers + - calls-offloader service binding configuration (ensure it's not bound to localhost only) + +3. **Verify Docker service** (if using docker api_type): + + ```bash + # Check that system user running calls-offloader can access Docker + sudo -u calls-offloader docker ps + ``` + +## Integration with Mattermost + +Once calls-offloader is properly set up and validated, configure Mattermost to use it: + +1. Go to **System Console > Plugins > Calls** + +2. In the **Job Service** section: + + - Set **Job Service URL** to your calls-offloader service (e.g., `http://calls-offloader-server:4545`) + +3. Enable recording and transcription features as needed: + + - **Enable Call Recordings**: Toggle to allow call recordings + - **Enable Call Transcriptions**: Toggle to allow call transcriptions + - **Enable Live Captions** (Experimental): Toggle to allow real-time transcription + +4. Save the configuration + +5. Restart the Calls plugin to re-establish state: + + - Go to **System Console > Plugins > Plugin Management** + - Find the **Calls** plugin and click **Disable** + - Wait a few seconds, then click **Enable** + +6. Test by starting a call and starting a recording + +## Troubleshooting + +### Common Issues + +**"failed to create recording job: max concurrent jobs reached"** + +This error occurs when the calls-offloader service has reached its configured job limit. + +Solutions: + +- Increase `max_concurrent_jobs` in the configuration +- Check if jobs are hanging and restart the service +- Monitor system resources and scale up if needed + +**Jobs not processing** + +Check the following: + +- Verify the calls-offloader service is running: `sudo systemctl status calls-offloader` +- Ensure network connectivity between Mattermost and calls-offloader +- Check Docker daemon is running and accessible by the user running the Calls Offloader service (E.g., user: ``calls-offloader``) +- Verify authentication configuration matches between services +- Review service logs for specific error messages + +**Docker permission issues** + +If using Docker API and seeing permission errors: + +```bash +# Add calls-offloader user to docker group +sudo usermod -a -G docker calls-offloader +sudo systemctl restart calls-offloader +``` + +### Debugging Commands + +Monitor calls-offloader job containers: + +```bash +# View running job containers +docker ps --format "{{.ID}} {{.Image}}" | grep "calls" + +# Follow logs for debugging +docker ps --format "{{.ID}} {{.Image}}" | grep "calls" | awk '{print $1}' | xargs -I {} docker logs -f {} + +# View completed job containers +docker ps -a --filter "status=exited" +``` + +Monitor service health: + +```bash +# Check service version and health +curl http://localhost:4545/version +``` + +Check service logs: + +```bash +# View recent logs +sudo journalctl -u calls-offloader -f + +# View log file (if file logging enabled) +tail -f /opt/calls-offloader/calls-offloader.log +``` + +### Performance Monitoring + +Monitor calls-offloader performance and resource usage to ensure optimal operation. See [Calls Metrics and Monitoring](calls-metrics-monitoring.md) for details on setting up metrics and observability. + +## Air-Gapped Deployments + +When deploying calls-offloader in air-gapped environments, you need to set up a local Docker registry since the service creates Docker containers that normally pull images from Docker Hub. + +### Overview + +The calls-offloader service creates Docker containers to handle: +- **Call Recording**: Creates containers to record audio/video from calls +- **Call Transcription**: Creates containers to transcribe recorded calls using speech-to-text + +These containers are typically pulled from the `mattermost` registry on Docker Hub, but in air-gapped networks, you need to: +1. Set up a local Docker registry +2. Pre-load the required Docker images +3. Configure the calls-offloader to use the local registry + +### Required Docker Images + +The following Docker images are needed for full calls functionality: + +- `mattermost/calls-recorder:v0.8.5` (or version matching your plugin) +- `mattermost/calls-transcriber:v0.6.3` (or version matching your plugin) +- `registry:2` (for the local Docker registry) + +```{warning} +**Disk Space Requirements**: Ensure you have sufficient disk space before starting the setup process. The Docker images can be quite large: +- **calls-recorder image**: ~1.5-2GB +- **calls-transcriber image**: ~1.5-2GB +- **Registry container + data**: ~500MB + +**Total recommended free space**: At least 5GB to accommodate image downloads, local registry data, and archive creation. +``` + +#### Determining the Correct Image Versions + +**Important**: The exact versions of `calls-offloader` and `calls-transcriber` images must match what your installed Calls plugin expects. These versions are defined in the Calls plugin source code. + +To find the correct versions for your Calls plugin: + +1. **Determine your Calls plugin version**: + - In Mattermost, go to **System Console > Plugins > Plugin Management** + - Find the **Calls** plugin and note the version number (e.g., `v1.9.0`) + +2. **Look up the required image versions**: + - Visit the Calls plugin repository: https://github.com/mattermost/mattermost-plugin-calls + - Navigate to the tag or branch corresponding to your plugin version + - Open the `plugin.json` file + - Find the `RecorderImage` and `TranscriberImage` entries (around line 719-720) + + Example from plugin.json: + ```json + "RecorderImage": "mattermost/calls-recorder:v0.8.5", + "TranscriberImage": "mattermost/calls-transcriber:v0.6.3" + ``` + +3. **Use these exact versions** in your air-gap setup instead of `latest` tags + +**Direct link format**: For plugin version `v1.9.0`, the plugin.json would be at: +`https://github.com/mattermost/mattermost-plugin-calls/blob/v1.9.0/plugin.json` + +### Setup Process + +#### Phase 1: Preparation (Internet-Connected Environment) + +Run this phase on a machine with internet access to download and prepare the Docker images. + +**Automated Setup Script** + +For convenience, you can use the automated setup script: + +```bash +# Download the setup script +curl -O https://docs.mattermost.com/scripts/air-gap-docker-registry-setup.sh +chmod +x air-gap-docker-registry-setup.sh + +# Run the setup script with the required image versions +sudo ./air-gap-docker-registry-setup.sh +``` + +The script will automatically create the required archive files (`docker-registry-data.tar.gz` and `registry-image.tar.gz`) for transfer to your air-gapped environment. + +**Manual Setup Steps** + +If you prefer to set up manually or need to customize the process: + +1. **Set up a local Docker registry**: + ```bash + # Create registry data directory + sudo mkdir -p /opt/docker-registry/data + + # Start a local Docker registry + docker run -d \ + --name local-registry \ + --restart=always \ + -p 5000:5000 \ + -v /opt/docker-registry/data:/var/lib/registry \ + registry:2 + ``` + +2. **Download and push required images**: + ```bash + # Pull required images from Docker Hub + docker pull mattermost/calls-recorder:v0.8.5 + docker pull mattermost/calls-transcriber:v0.6.3 + + # Tag images for local registry + docker tag mattermost/calls-recorder:v0.8.5 localhost:5000/mattermost/calls-recorder:v0.8.5 + docker tag mattermost/calls-transcriber:v0.6.3 localhost:5000/mattermost/calls-transcriber:v0.6.3 + + # Push images to local registry + docker push localhost:5000/mattermost/calls-recorder:v0.8.5 + docker push localhost:5000/mattermost/calls-transcriber:v0.6.3 + ``` + +3. **Export the registry data**: + ```bash + # Create an archive of the registry data + sudo tar -czf docker-registry-data.tar.gz -C /opt/docker-registry/data . + + # Also backup the registry container image + docker save registry:2 | gzip > registry-image.tar.gz + ``` + +#### Phase 2: Air-Gap Deployment + +Transfer the following files to your air-gapped network: +- `docker-registry-data.tar.gz` (contains the registry data with pre-loaded images) +- `registry-image.tar.gz` (contains the Docker registry container image) +- `deploy-airgap-calls.sh` (deployment script created by the setup script) + +**Complete Air-Gap Deployment Steps:** + +1. **Load the registry container image**: + ```bash + # Extract and load the registry container from the gzipped archive + gunzip registry-image.tar.gz + docker load -i registry-image.tar + ``` + +2. **Set up the registry data directory**: + ```bash + # Create the registry data directory + sudo mkdir -p /opt/docker-registry/data + + # Extract the pre-loaded registry data + sudo tar -xzf docker-registry-data.tar.gz -C /opt/docker-registry/data + ``` + +3. **Start the local registry with pre-loaded data**: + ```bash + # Start the registry container with the extracted data + docker run -d \ + --name local-registry \ + --restart=always \ + -p 5000:5000 \ + -v /opt/docker-registry/data:/var/lib/registry \ + registry:2 + ``` + +4. **Configure Docker daemon for insecure registry access**: + ```bash + # Create or update Docker daemon configuration + sudo mkdir -p /etc/docker + echo '{"insecure-registries": ["localhost:5000"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + ``` + +5. **Configure Mattermost server environment variable**: + ```bash + # Add the registry configuration to Mattermost environment + echo 'MM_CALLS_JOB_SERVICE_IMAGE_REGISTRY="localhost:5000/mattermost"' | sudo tee -a /opt/mattermost/config/mattermost.environment + + # Restart Mattermost to apply the environment variable + sudo systemctl restart mattermost + ``` + +6. **Run the air-gap deployment script** (if using the automated setup): + ```bash + # Make the deployment script executable and run it + chmod +x deploy-airgap-calls.sh + sudo ./deploy-airgap-calls.sh + ``` + + Or configure calls-offloader manually (see Manual Configuration section below). + +### Manual Configuration + +For reference, here are the individual configuration steps: + +#### 1. Docker Daemon Configuration + +Create or update `/etc/docker/daemon.json`: +```json +{ + "insecure-registries": ["localhost:5000"] +} +``` + +Restart Docker: +```bash +sudo systemctl restart docker +``` + +#### 2. Calls-Offloader Configuration + +Update `/opt/calls-offloader/calls-offloader.toml`: + +```toml +[jobs] +# Change this line: +image_registry = "mattermost" + +# To this: +image_registry = "localhost:5000/mattermost" +``` + +Restart the calls-offloader service: +```bash +sudo systemctl restart calls-offloader +``` + +### Verification + +#### Test Registry Access +```bash +# List available repositories +curl http://localhost:5000/v2/_catalog + +# Test pulling an image +docker pull localhost:5000/mattermost/calls-recorder:latest +``` + +#### Test Calls Functionality + +1. **Check calls-offloader logs**: + ```bash + sudo journalctl -u calls-offloader -f + ``` + +2. **Verify calls-offloader API**: + ```bash + curl http://localhost:4545/version + ``` + +3. **Test recording job creation** (requires proper Mattermost integration): + - Start a call in Mattermost + - Enable recording + - Check that Docker containers are created for recording jobs + +### Troubleshooting Air-Gap Deployments + +#### Common Issues + +1. **Registry not accessible**: + - Check that the registry container is running: `docker ps | grep registry` + - Verify Docker daemon configuration includes insecure registry + - Check firewall settings on port 5000 + +2. **Image pull failures**: + - Verify images are in the registry: `curl http://localhost:5000/v2/_catalog` + - Check Docker daemon logs: `sudo journalctl -u docker` + +3. **calls-offloader fails to create jobs**: + - Check calls-offloader logs: `sudo journalctl -u calls-offloader` + - Verify the `image_registry` configuration in calls-offloader.toml + - Ensure the calls-offloader service can reach the registry + +4. **"invalid Runner value: failed to validate runner" error**: + This error occurs when calls-offloader cannot validate Docker images from the local registry. + + **Common causes and solutions:** + - **Image not found**: Verify the exact image names and tags in your local registry: + ```bash + curl http://localhost:5000/v2/_catalog + curl http://localhost:5000/v2/mattermost/calls-recorder/tags/list + curl http://localhost:5000/v2/mattermost/calls-transcriber/tags/list + ``` + + - **Registry configuration mismatch**: Ensure the `image_registry` setting in calls-offloader.toml matches your registry: + ```bash + grep image_registry /opt/calls-offloader/calls-offloader.toml + # Should show: image_registry = "localhost:5000/mattermost" + ``` + + - **Docker daemon can't reach registry**: Test that Docker can pull from the local registry: + ```bash + docker pull localhost:5000/mattermost/calls-recorder:latest + docker pull localhost:5000/mattermost/calls-transcriber:latest + ``` + + - **Image tag mismatch**: The calls-offloader will be looking for specific image tags. Check what the plugin expects vs what's in your registry: + ```bash + # Check what tags are available + curl http://localhost:5000/v2/mattermost/calls-recorder/tags/list + # Compare with what your plugin.json specifies + ``` + + **Solution steps:** + 1. Restart calls-offloader after confirming registry configuration: `sudo systemctl restart calls-offloader` + 2. If the issue persists, check the exact image names and versions expected by your Calls plugin version + 3. Ensure both versioned tags (e.g., `v0.8.5`) and `latest` tags are present in your local registry + +#### Log Locations + +- Setup script logs: `/tmp/air-gap-registry-setup.log` +- calls-offloader logs: `/opt/calls-offloader/calls-offloader.log` +- Docker daemon logs: `sudo journalctl -u docker` +- Registry container logs: `docker logs local-registry` + +#### Advanced Configuration + +**Using a Different Registry Host** + +If you want to run the registry on a different host, replace `localhost:5000` with your registry host in all commands: + +```bash +# Example: using a dedicated registry server +REGISTRY_HOST="registry.internal.domain:5000" + +# Update Docker daemon configuration +echo "{\"insecure-registries\": [\"$REGISTRY_HOST\"]}" | sudo tee /etc/docker/daemon.json + +# Update calls-offloader configuration +sed -i "s|localhost:5000|$REGISTRY_HOST|g" /opt/calls-offloader/calls-offloader.toml +``` + +**Custom Image Versions** + +To use specific versions of the calls images, update the version tags in the docker commands: + +```bash +# Example: using specific versions +RECORDER_VERSION="v0.8.5" +TRANSCRIBER_VERSION="v0.6.3" + +docker pull mattermost/calls-recorder:$RECORDER_VERSION +docker pull mattermost/calls-transcriber:$TRANSCRIBER_VERSION +``` + +### Day 2 Operations: Upgrading Images in Air-Gap Environments + +When you upgrade your Mattermost Calls plugin to a newer version, you'll need to update the Docker images in your air-gapped registry to match the new plugin requirements. + +#### Upgrade Process Overview + +**When to upgrade**: After upgrading the Calls plugin in Mattermost, you must update the Docker images to match the versions expected by the new plugin version. + +**Two approaches available**: + +1. **Complete Rebuild Method**: Re-run the entire air-gap setup process with new image versions +2. **Incremental Update Method**: Transfer only the new images to minimize data transfer + +#### Method 1: Complete Rebuild (Recommended for Major Updates) + +This approach rebuilds the entire registry dataset with new image versions: + +1. **On internet-connected machine**, run the air-gap setup script with new versions: + ```bash + # Find new versions from your updated plugin.json + ./air-gap-docker-registry-setup.sh v0.8.5 v0.6.3 + ``` + +2. **Transfer new archives** to air-gapped environment: + - `docker-registry-data.tar.gz` (contains all images including new versions) + - `deploy-airgap-calls.sh` (updated deployment script) + +3. **In air-gapped environment**, replace the registry data: + ```bash + # Stop the existing registry + docker stop local-registry + docker rm local-registry + + # Backup existing data (optional) + sudo mv /opt/docker-registry/data /opt/docker-registry/data.backup + + # Extract new registry data + sudo mkdir -p /opt/docker-registry/data + sudo tar -xzf docker-registry-data.tar.gz -C /opt/docker-registry/data + + # Restart registry with new data + docker run -d \ + --name local-registry \ + --restart=always \ + -p 5000:5000 \ + -v /opt/docker-registry/data:/var/lib/registry \ + registry:2 + ``` + +#### Method 2: Incremental Update (Efficient for Minor Updates) + +This approach transfers only the new image versions without rebuilding the entire registry: + +**Phase 1: Preparation (Internet-Connected Environment)** + +1. **Download new images**: + ```bash + # Determine new versions from updated plugin.json + NEW_RECORDER_VERSION="v0.8.5" + NEW_TRANSCRIBER_VERSION="v0.6.3" + + # Pull new images + docker pull mattermost/calls-recorder:$NEW_RECORDER_VERSION + docker pull mattermost/calls-transcriber:$NEW_TRANSCRIBER_VERSION + ``` + +2. **Create individual image archives**: + ```bash + # Save each image as a separate tar file + docker save mattermost/calls-recorder:$NEW_RECORDER_VERSION -o calls-recorder-$NEW_RECORDER_VERSION.tar + docker save mattermost/calls-transcriber:$NEW_TRANSCRIBER_VERSION -o calls-transcriber-$NEW_TRANSCRIBER_VERSION.tar + + # Compress the files for transfer + gzip calls-recorder-$NEW_RECORDER_VERSION.tar + gzip calls-transcriber-$NEW_TRANSCRIBER_VERSION.tar + ``` + +3. **Create update script**: + ```bash + cat > update-air-gap-images.sh << 'EOF' + #!/bin/bash + + # Air-Gap Image Update Script + set -e + + NEW_RECORDER_VERSION="v0.8.5" + NEW_TRANSCRIBER_VERSION="v0.6.3" + REGISTRY_HOST="${REGISTRY_HOST:-localhost}" + REGISTRY_PORT="${REGISTRY_PORT:-5000}" + + echo "Updating air-gap registry with new image versions..." + echo "Recorder: $NEW_RECORDER_VERSION" + echo "Transcriber: $NEW_TRANSCRIBER_VERSION" + + # Load new images into Docker + echo "Loading new images..." + gunzip calls-recorder-$NEW_RECORDER_VERSION.tar.gz + gunzip calls-transcriber-$NEW_TRANSCRIBER_VERSION.tar.gz + + docker load -i calls-recorder-$NEW_RECORDER_VERSION.tar + docker load -i calls-transcriber-$NEW_TRANSCRIBER_VERSION.tar + + # Tag images for local registry + echo "Tagging images for local registry..." + docker tag mattermost/calls-recorder:$NEW_RECORDER_VERSION $REGISTRY_HOST:$REGISTRY_PORT/mattermost/calls-recorder:$NEW_RECORDER_VERSION + docker tag mattermost/calls-transcriber:$NEW_TRANSCRIBER_VERSION $REGISTRY_HOST:$REGISTRY_PORT/mattermost/calls-transcriber:$NEW_TRANSCRIBER_VERSION + + # Also update 'latest' tags + docker tag mattermost/calls-recorder:$NEW_RECORDER_VERSION $REGISTRY_HOST:$REGISTRY_PORT/mattermost/calls-recorder:latest + docker tag mattermost/calls-transcriber:$NEW_TRANSCRIBER_VERSION $REGISTRY_HOST:$REGISTRY_PORT/mattermost/calls-transcriber:latest + + # Push to local registry + echo "Pushing images to local registry..." + docker push $REGISTRY_HOST:$REGISTRY_PORT/mattermost/calls-recorder:$NEW_RECORDER_VERSION + docker push $REGISTRY_HOST:$REGISTRY_PORT/mattermost/calls-transcriber:$NEW_TRANSCRIBER_VERSION + docker push $REGISTRY_HOST:$REGISTRY_PORT/mattermost/calls-recorder:latest + docker push $REGISTRY_HOST:$REGISTRY_PORT/mattermost/calls-transcriber:latest + + echo "Image update complete!" + echo "" + echo "Verification commands:" + echo "curl http://$REGISTRY_HOST:$REGISTRY_PORT/v2/mattermost/calls-recorder/tags/list" + echo "curl http://$REGISTRY_HOST:$REGISTRY_PORT/v2/mattermost/calls-transcriber/tags/list" + + # Clean up temporary files + rm calls-recorder-$NEW_RECORDER_VERSION.tar + rm calls-transcriber-$NEW_TRANSCRIBER_VERSION.tar + + echo "" + echo "Next steps:" + echo "1. Restart calls-offloader service: sudo systemctl restart calls-offloader" + echo "2. Test call recording functionality to verify the update" + EOF + + chmod +x update-air-gap-images.sh + ``` + +**Phase 2: Air-Gap Update** + +1. **Transfer files to air-gapped environment**: + - `calls-recorder-v0.8.5.tar.gz` + - `calls-transcriber-v0.6.3.tar.gz` + - `update-air-gap-images.sh` + +2. **Run the update script**: + ```bash + chmod +x update-air-gap-images.sh + sudo ./update-air-gap-images.sh + ``` + +3. **Restart calls-offloader**: + ```bash + sudo systemctl restart calls-offloader + ``` + +4. **Verify the update**: + ```bash + # Check available image tags + curl http://localhost:5000/v2/mattermost/calls-recorder/tags/list + curl http://localhost:5000/v2/mattermost/calls-transcriber/tags/list + + # Test calls-offloader functionality + curl http://localhost:4545/version + ``` + +#### Advantages of Each Method + +**Complete Rebuild Method:** +- ✅ Ensures clean state with no leftover data +- ✅ Recommended for major version upgrades +- ✅ Simpler process (reuse existing automation) +- ❌ Larger data transfer requirements +- ❌ More downtime during registry replacement + +**Incremental Update Method:** +- ✅ Minimal data transfer (only new images) +- ✅ Faster deployment process +- ✅ Preserves existing registry data +- ✅ Less downtime (registry stays running) +- ❌ More complex process +- ❌ Potential for version conflicts if not managed carefully + +#### Choosing the Right Method + +**Use Complete Rebuild when:** +- Upgrading across major plugin versions (e.g., v1.8.x to v1.9.x) +- Registry has accumulated significant old/unused images +- You want to ensure a completely clean state +- Data transfer size is not a primary concern + +**Use Incremental Update when:** +- Applying minor version updates (e.g., v1.9.0 to v1.9.1) +- Bandwidth or transfer time is limited +- You need to minimize downtime +- The registry is working correctly and just needs new image versions + +#### Post-Update Verification + +After either upgrade method, perform these verification steps: + +1. **Test recording functionality**: + - Start a call in Mattermost + - Enable call recording + - Verify recording starts without errors + +2. **Check job container creation**: + ```bash + # Monitor for new job containers during recording + docker ps --format "{{.ID}} {{.Image}}" | grep calls + ``` + +3. **Monitor calls-offloader logs**: + ```bash + sudo journalctl -u calls-offloader -f + ``` + +4. **Verify image versions**: + ```bash + # Check that the new image versions are being used + docker ps --format "{{.Image}}" | grep calls + ``` + +## Other Calls Documentation + +- [Calls Overview](calls-deployment.md): Overview of deployment options and architecture +- [RTCD Setup and Configuration](calls-rtcd-setup.md): Comprehensive guide for setting up the dedicated RTCD service +- [Calls Metrics and Monitoring](calls-metrics-monitoring.md): Guide to monitoring Calls performance using metrics and observability +- [Calls Deployment on Kubernetes](calls-kubernetes.md): Detailed guide for deploying Calls in Kubernetes environments +- [Calls Troubleshooting](calls-troubleshooting.md): Detailed troubleshooting steps and debugging techniques +- [calls-offloader performance documentation](https://github.com/mattermost/calls-offloader/blob/master/docs/performance.md): Detailed performance tuning and monitoring recommendations \ No newline at end of file diff --git a/source/configure/calls-overview.rst b/source/configure/calls-overview.rst new file mode 100644 index 00000000000..e73f5bd7015 --- /dev/null +++ b/source/configure/calls-overview.rst @@ -0,0 +1,27 @@ +Mattermost Calls +================ + +.. include:: ../_static/badges/allplans-cloud-selfhosted.rst + :start-after: :nosearch: + +Mattermost Calls provides integrated audio calling and screen sharing capabilities within Mattermost channels. This section covers deployment, configuration, and management of Mattermost Calls. + +.. toctree:: + :maxdepth: 1 + :hidden: + + Deploy Mattermost Calls + RTCD Setup and Configuration + Calls Offloader Setup and Configuration + Calls Metrics and Monitoring + Deploy Calls on Kubernetes + Calls Troubleshooting + +**Calls Documentation:** + +* :doc:`Calls Deployment Overview ` - Main deployment overview and architecture guide for Mattermost Calls +* :doc:`RTCD Setup and Configuration ` - Real-time communication daemon setup for enterprise deployments +* :doc:`Calls Offloader Setup and Configuration ` - Configure call recording and transcription services +* :doc:`Calls Metrics and Monitoring ` - Performance monitoring with Prometheus and Grafana +* :doc:`Calls Deployment on Kubernetes ` - Kubernetes deployment guide for scalable Calls infrastructure +* :doc:`Calls Troubleshooting ` - Comprehensive troubleshooting guide for common issues diff --git a/source/configure/calls-rtcd-setup.md b/source/configure/calls-rtcd-setup.md new file mode 100644 index 00000000000..6fece97e085 --- /dev/null +++ b/source/configure/calls-rtcd-setup.md @@ -0,0 +1,499 @@ +# RTCD Setup and Configuration + +```{include} ../_static/badges/calls-rtcd-ent-only.md +``` + +This guide provides detailed instructions for setting up, configuring, and validating a Mattermost Calls deployment using the dedicated RTCD service. + +## Prerequisites + +Before deploying RTCD, ensure you have: + +- A Mattermost Enterprise license +- A server or VM with sufficient CPU and network capacity (see the [Performance](calls-deployment.html#performance) section for sizing guidance) + +## Network Requirements + +The following network connectivity is required: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ServicePortsProtocolsSourceTargetPurpose
API (Calls plugin)80,443TCP (incoming)Mattermost clients (web/desktop/mobile)Mattermost instance (Calls plugin)To allow for HTTP and WebSocket connectivity from clients to Calls plugin. This API is exposed on the same connection as Mattermost, so there's likely no need to change anything.
RTC (Calls plugin or rtcd)8443UDP (incoming)Mattermost clients (Web/Desktop/Mobile) and calls-offloaderMattermost instance or rtcd serviceTo allow clients to establish connections that transport calls related media (e.g. audio, video). This should be open on any network component (e.g. NAT, firewalls) in between the instance running the plugin (or rtcd) and the clients joining calls so that UDP traffic is correctly routed both ways (from/to clients).
RTC (Calls plugin or rtcd)8443TCP (incoming)Mattermost clients (Web/Desktop/Mobile) and calls-offloaderMattermost instance or rtcd serviceTo allow clients to establish connections that transport calls related media (e.g. audio, video). This should be open on any network component (e.g. NAT, firewalls) in between the instance running the plugin (or rtcd) and the clients joining calls so that TCP traffic is correctly routed both ways (from/to clients). This can be used as a backup channel in case clients are unable to connect using UDP. It requires rtcd version >= v0.11 and Calls version >= v0.17.
API (rtcd)8045TCP (incoming)Mattermost instance(s) (Calls plugin)rtcd serviceTo allow for HTTP/WebSocket connectivity from Calls plugin to rtcd service. Can be expose internally as the service only needs to be reachable by the instance(s) running the Mattermost server.
STUN (Calls plugin or rtcd)3478UDP (outgoing)Mattermost Instance(s) (Calls plugin) or rtcd serviceConfigured STUN servers(Optional) To allow for either Calls plugin or rtcd service to discover their instance public IP. Only needed if configuring STUN/TURN servers. This requirement does not apply when manually setting an IP or hostname through the ICE Host Override config option.
+ +## Installation and Deployment + +There are multiple ways to deploy RTCD, depending on your environment. We recommend the following order based on production readiness and operational control: + +### Bare Metal or VM Deployment (Recommended) + +This is the recommended deployment method for non-Kubernetes production environments as it provides the best performance and operational control. For Kubernetes deployments, see the [Calls Deployment on Kubernetes](calls-kubernetes.md) guide. + +1. **Download and install the RTCD binary**: + + Download the latest release from the [RTCD GitHub repository](https://github.com/mattermost/rtcd/releases): + + ```bash + # Create the RTCD directory structure + sudo mkdir -p /opt/rtcd + + # Download the latest RTCD binary (adjust URL for your architecture) + # For Linux x86_64: + wget https://github.com/mattermost/rtcd/releases/latest/download/rtcd-linux-amd64 + + # Make the binary executable and move it to the installation directory + chmod +x rtcd-linux-amd64 + sudo mv rtcd-linux-amd64 /opt/rtcd/rtcd + ``` + + ```{note} + Replace `rtcd-linux-amd64` with the appropriate binary for your system architecture (e.g., `rtcd-linux-arm64` for ARM64 systems). The binary should be placed at `/opt/rtcd/rtcd` as this is the expected location referenced in systemd service files and other documentation. + ``` + +2. **Create a configuration file** (`/opt/rtcd/rtcd.toml`) with the following settings: + + ```toml + [api] + http.listen_address = ":8045" + security.allow_self_registration = true + + [rtc] + ice_address_udp = "" + ice_port_udp = 8443 + ice_address_tcp = "" + ice_port_tcp = 8443 + ice_host_override = "YOUR_RTCD_SERVER_PUBLIC_IP" + + # UDP port range for WebRTC connections + ice.port_range.min = 9000 + ice.port_range.max = 10000 + + # STUN/TURN server configuration + ice_servers = [ + { urls = ["stun:stun.global.calls.mattermost.com:3478"] } + ] + + [store] + data_source = "/opt/rtcd/data/db" + + [logger] + enable_console = true + console_json = true + console_level = "INFO" + enable_file = true + file_json = true + file_level = "INFO" + file_location = "/opt/rtcd/rtcd.log" + enable_color = true + ``` + +3. Create a dedicated user for the RTCD service: + + ```bash + sudo useradd --system --no-create-home --shell /bin/false mattermost + ``` + +4. Create the data directory and set ownership: + + ```bash + sudo mkdir -p /opt/rtcd/data/db + sudo chown -R mattermost:mattermost /opt/rtcd + ``` + +5. Create a systemd service file (`/etc/systemd/system/rtcd.service`): + + ```ini + [Unit] + Description=Mattermost RTCD Server + After=network.target + + [Service] + Type=simple + User=mattermost + Group=mattermost + ExecStart=/opt/rtcd/rtcd --config /opt/rtcd/rtcd.toml + Restart=always + RestartSec=10 + LimitNOFILE=65536 + + [Install] + WantedBy=multi-user.target + ``` + +5. Enable and start the service: + + ```bash + sudo systemctl daemon-reload + sudo systemctl enable rtcd + sudo systemctl start rtcd + ``` + +6. Check the service status: + + ```bash + sudo systemctl status rtcd + ``` + +### Docker Deployment + +Docker deployment is suitable for development, testing, or containerized production environments: + +1. Run the RTCD container with basic configuration: + + ```bash + docker run -d --name rtcd \ + -e "RTCD_LOGGER_ENABLEFILE=true" \ + -e "RTCD_API_SECURITY_ALLOWSELFREGISTRATION=true" \ + -p 8443:8443/udp \ + -p 8443:8443/tcp \ + -p 8045:8045/tcp \ + mattermost/rtcd:latest + ``` + +2. For debugging purposes, you can enable more detailed logging: + + ```bash + docker run -d --name rtcd \ + -e "RTCD_LOGGER_ENABLEFILE=true" \ + -e "RTCD_LOGGER_CONSOLELEVEL=DEBUG" \ + -e "RTCD_API_SECURITY_ALLOWSELFREGISTRATION=true" \ + -p 8443:8443/udp \ + -p 8443:8443/tcp \ + -p 8045:8045/tcp \ + mattermost/rtcd:latest + ``` + + To view the logs: + + ```bash + docker logs -f rtcd + ``` + +You can also use a mounted configuration file instead of environment variables: + +```bash +docker run -d --name rtcd \ + -p 8045:8045 \ + -p 8443:8443/udp \ + -p 8443:8443/tcp \ + -v /path/to/config.toml:/rtcd/config/config.toml \ + mattermost/rtcd:latest +``` + +For a complete sample configuration file, see the [RTCD config.sample.toml](https://github.com/mattermost/rtcd/blob/master/config/config.sample.toml) in the official repository. + +### Kubernetes Deployment + +For Kubernetes deployments, use the official Helm chart: + +1. Add the Mattermost Helm repository: + + ```bash + helm repo add mattermost https://helm.mattermost.com + helm repo update + ``` + +2. Install the RTCD chart: + + ```bash + helm install mattermost-rtcd mattermost/mattermost-rtcd \ + --set ingress.enabled=true \ + --set ingress.host=rtcd.example.com \ + --set service.annotations."service\\.beta\\.kubernetes\\.io/aws-load-balancer-backend-protocol"=udp \ + --set rtcd.ice.hostOverride=rtcd.example.com + ``` + + Refer to the [RTCD Helm chart documentation](https://github.com/mattermost/mattermost-helm/tree/master/charts/mattermost-rtcd) for additional configuration options. + +## Configuration + +### RTCD Configuration File + +The RTCD service uses a TOML configuration file. Here's an example with commonly used settings: + +```toml +[api] +# The address and port to which the HTTP API server will listen +http.listen_address = ":8045" +# Security settings for authentication +security.allow_self_registration = true +security.enable_admin = true +security.admin_secret_key = "YOUR_API_KEY" + +[rtc] +# The UDP address and port for media traffic +ice_address_udp = "" +ice_port_udp = 8443 +# The TCP address and port for fallback connections +ice_address_tcp = "" +ice_port_tcp = 8443 +# Public hostname or IP that clients will use to connect +ice_host_override = "RTCD_SERVER_PUBLIC_IP" + +[logger] +# Logging configuration +enable_console = true +console_json = false +console_level = "INFO" +enable_file = true +file_json = true +file_level = "INFO" +file_location = "/opt/rtcd/rtcd.log" +``` + +Key Configuration Options: + +- **api.http.listen_address**: The address and port where the RTCD HTTP API service listens +- **rtc.ice_address_udp**: The UDP address for media traffic (empty means listen on all interfaces) +- **rtc.ice_port_udp**: The UDP port for media traffic +- **rtc.ice_address_tcp**: The TCP address for fallback media traffic +- **rtc.ice_port_tcp**: The TCP port for fallback media traffic +- **rtc.ice_host_override**: The public hostname or IP address clients will use to connect to RTCD +- **api.security.admin_secret_key**: API key for Mattermost servers to authenticate with RTCD + +### Required Mattermost Server Configuration + +When using RTCD, you must configure the Mattermost server's CORS settings to allow proper communication between the server and the RTCD service. + +#### CORS Configuration + +The `AllowCorsFrom` setting must include your SiteURL and, if using calls-offloader in a private network, the Mattermost server's private IP address: + +**Using mmctl:** +```bash +# Basic RTCD configuration - include your SiteURL +mmctl config set ServiceSettings.AllowCorsFrom "https://your-domain.com" + +# If using calls-offloader in a private network, also include Mattermost's private IP with port 8065 +mmctl config set ServiceSettings.AllowCorsFrom "https://your-domain.com http://192.168.1.100:8065" +``` + +**Using System Console:** +1. Go to **System Console > Environment > Web Server** +2. Set **Allow cross-origin requests from** to include: + - Your SiteURL (e.g., `https://your-domain.com`) + - If using calls-offloader in a private network: Also include Mattermost's private IP with port 8065 (e.g., `http://192.168.1.100:8065`) + +**Using config.json:** +```json +{ + "ServiceSettings": { + "AllowCorsFrom": "https://your-domain.com http://192.168.1.100:8065" + } +} +``` + +```{important} +This CORS configuration is specifically required for RTCD deployments and is not needed for integrated mode deployments. Multiple origins should be separated by spaces. +``` + +### STUN/TURN Configuration + +For clients behind strict firewalls, you may need to configure STUN/TURN servers. In the RTCD configuration file, reference your STUN/TURN servers as follows: + +```toml +[rtc] +# STUN/TURN server configuration + ice_servers = [ + { urls = ["stun:stun.global.calls.mattermost.com:3478"] } + # { urls = ["turn:turn.example.com:3478"], username = "turnuser", credential = "turnpassword" } + ] + +``` + +We recommend using [coturn](https://github.com/coturn/coturn) for your TURN server implementation. + +### System Tuning + +For high-volume deployments, tune your Linux system: + +1. Add the following to `/etc/sysctl.conf`: + + ```bash + # Increase UDP buffer sizes + net.core.rmem_max = 16777216 + net.core.wmem_max = 16777216 + net.core.optmem_max = 16777216 + ``` + +2. Apply the settings: + + ```bash + sudo sysctl -p + ``` + +## Validation and Testing + +After deploying RTCD, validate the installation: + +1. **Check service status and version**: + + ```bash + curl http://YOUR_RTCD_SERVER:8045/version + # Should return a JSON object with service information + # Example: {"build_hash":"abc123","build_date":"2023-01-15T12:00:00Z","build_version":"0.11.0","goVersion":"go1.20.4"} + ``` + +2. **Test UDP connectivity**: + + On the RTCD server: + + ```bash + nc -l -u -p 8443 + ``` + + On a client machine: + + ```bash + nc -v -u YOUR_RTCD_SERVER 8443 + ``` + + Type a message and hit Enter on either side. If messages are received on both ends, UDP connectivity is working. + + Note: This test must be run with the RTCD service stopped, as it binds to the same port. + + ```bash + sudo systemctl stop rtcd + ``` + +3. **Test TCP connectivity** (if enabled): + + Similar to the UDP test, but remove the `-u` flag from both commands. + +4. **Monitor metrics**: + + Refer to [Calls Metrics and Monitoring](calls-metrics-monitoring.md) for setting up Calls metrics and monitoring. + +## Horizontal Scaling + +To scale RTCD horizontally: + +1. **Deploy multiple RTCD instances**: + + Deploy multiple RTCD servers, each with their own unique IP address. + +2. **Configure DNS-based load balancing**: + + Set up a DNS record that points to multiple RTCD IP addresses: + + ```bash + rtcd.example.com. IN A 10.0.0.1 + rtcd.example.com. IN A 10.0.0.2 + rtcd.example.com. IN A 10.0.0.3 + ``` + +3. **Configure health checks**: + + Set up health checks to automatically remove unhealthy RTCD instances from DNS. + +4. **Configure Mattermost**: + + In the Mattermost System Console, set the **RTCD Service URL** to your DNS name (e.g., `rtcd.example.com`). + +The Mattermost Calls plugin will distribute calls among the available RTCD hosts. Remember that a single call will always be hosted on one RTCD instance; sessions belonging to the same call are not spread across different instances. + +## Integration with Mattermost + +Once RTCD is properly set up and validated, configure Mattermost to use it: + +1. Go to **System Console > Plugins > Calls** + +2. Enable the **Enable RTCD Service** option + +3. Set the **RTCD Service URL** to your RTCD service address (either a single server or DNS load-balanced hostname) + +4. If configured, enter the **RTCD API Key** that matches the one in your RTCD configuration + +5. Save the configuration + +6. Test by creating a new call in any Mattermost channel + +7. Verify that the call is being routed through RTCD by checking the RTCD logs and metrics + +## Other Calls Documentation + +- [Calls Overview](calls-deployment.md): Overview of deployment options and architecture +- [Calls Offloader Setup and Configuration](calls-offloader-setup.md): Setup guide for call recording and transcription +- [Calls Metrics and Monitoring](calls-metrics-monitoring.md): Guide to monitoring Calls performance using metrics and observability +- [Calls Deployment on Kubernetes](calls-kubernetes.md): Detailed guide for deploying Calls in Kubernetes environments +- [Calls Troubleshooting](calls-troubleshooting.md): Detailed troubleshooting steps and debugging techniques + +For detailed Mattermost Calls configuration options, see the [Calls Plugin Configuration Settings](plugins-configuration-settings.rst#calls) documentation. \ No newline at end of file diff --git a/source/configure/calls-troubleshooting.md b/source/configure/calls-troubleshooting.md new file mode 100644 index 00000000000..f42420b1862 --- /dev/null +++ b/source/configure/calls-troubleshooting.md @@ -0,0 +1,386 @@ +# Troubleshooting Mattermost Calls + +```{include} ../_static/badges/allplans-cloud-selfhosted.md +``` + +This guide provides comprehensive troubleshooting steps for Mattermost Calls, particularly focusing on the dedicated RTCD deployment model. Follow these steps to identify and resolve common issues. + +- [Common issues](#common-issues) +- [Connectivity troubleshooting](#connectivity-troubleshooting) +- [Log analysis](#log-analysis) +- [Performance issues](#performance-issues) + +## Common Issues + +### Calls Not Connecting + +**Symptoms**: Users can start calls but cannot connect, or calls connect but drop quickly. + +**Possible causes and solutions**: + +1. **Network connectivity issues**: + - Verify that UDP port 8443 (or your configured port) is open between clients and RTCD servers + - Ensure TCP port 8045 is open between Mattermost and RTCD servers + - Check that any load balancers are properly configured for UDP traffic + +2. **ICE configuration issues**: + - Verify the `rtc.ice_host_override` setting in RTCD configuration matches the publicly accessible hostname or IP of the RTCD server + - If this setting is incorrect, client browser console may show errors like: `com.mattermost.calls: peer error timed out waiting for rtc connection` + - Meanwhile, RTCD `trace` level logs might show internal IP addresses in ICE connection logs: + + ```json + {"timestamp":"2025-05-14 10:29:08.935 Z","level":"trace","msg":"Ping STUN from udp4 host 172.31.29.117:8443 (resolved: 172.31.29.117:8443) to udp4 host 192.168.64.1:59737 (resolved: 192.168.64.1:59737)","caller":"rtc/logger.go:54","origin":"ice/v4.(*Agent).sendBindingRequest github.com/pion/ice/v4@v4.0.3/agent.go:921"} + ``` + +3. **API connectivity**: + - Verify that Mattermost servers can reach the RTCD API endpoint + - Check that the API key is correctly configured in both Mattermost and RTCD + +4. **Plugin configuration**: + - Ensure the Calls plugin is enabled and properly configured + - Verify the RTCD service URL is correct in the System Console + +### Audio Issues + +**Symptoms**: Users can connect to calls, but audio is one-way, choppy, or not working. + +**Possible causes and solutions**: + +1. **Client permissions**: + - Ensure browser/app has microphone permissions + - Check if users are using multiple audio devices that might interfere + +2. **Network quality**: + - High latency or packet loss can cause audio issues + - Try testing with TCP fallback enabled (requires RTCD v0.11+ and Calls v0.17+) + +3. **Audio device configuration**: + - Users should verify their audio input/output settings + - Try different browsers or the desktop app + +### Call Quality Issues + +**Symptoms**: Calls connect but quality is poor, with latency, echo, or distortion. + +**Possible causes and solutions**: + +1. **Server resources**: + - Check CPU usage on RTCD servers - high CPU can cause quality issues + - Refer to the [Calls Metrics and Monitoring](calls-metrics-monitoring.md) guide for detailed instructions on monitoring and optimizing performance + - Monitor network bandwidth usage + +2. **Network congestion**: + - Check for packet loss between clients and RTCD + - Consider network QoS settings to prioritize real-time traffic + +3. **Client-side issues**: + - Browser or app limitations + - Hardware limitations (CPU, memory) + - Network congestion at the user's location + +## Connectivity Troubleshooting + +### Basic Connectivity Tests + +1. **HTTP API connectivity test**: + + Test if the RTCD API is reachable: + + ```bash + curl http://YOUR_RTCD_SERVER:8045/version + # Example response: {"buildDate":"2025-04-02 21:33","buildVersion":"v1.1.0","buildHash":"7bc1f7a","goVersion":"go1.23.6","goOS":"linux","goArch":"amd64"} ``` + +2. **UDP connectivity test**: + + On the RTCD server: + + ```bash + nc -l -u -p 8443 + ``` + + On a client machine: + + ```bash + nc -v -u YOUR_RTCD_SERVER 8443 + ``` + + Type a message and press Enter. If you see the message on both sides, UDP connectivity is working. + +3. **TCP fallback connectivity test**: + + Same as the UDP test, but without the `-u` flag: + + On the RTCD server: + + ```bash + nc -l -p 8443 + ``` + + On a client machine: + + ```bash + nc -v YOUR_RTCD_SERVER 8443 + ``` + +### Network Packet Analysis + +To capture and analyze network traffic: + +1. **Capture UDP traffic on the RTCD server**: + + ```bash + sudo tcpdump -n 'udp port 8443' -i any + ``` + +2. **Capture TCP API traffic**: + + ```bash + sudo tcpdump -n 'tcp port 8045' -i any + ``` + +3. **Analyze traffic patterns**: + + - Verify packets are flowing both ways + - Look for ICMP errors that might indicate firewall issues + - Check for patterns of packet loss + +4. **Use Wireshark for deeper analysis**: + + For more detailed packet inspection, capture traffic with tcpdump and analyze with Wireshark: + + ```bash + sudo tcpdump -n -w calls_traffic.pcap 'port 8443' + ``` + + Then analyze the `calls_traffic.pcap` file with Wireshark. + +### Firewall Configuration Checks + +1. **Check iptables rules** (Linux): + + ```bash + sudo iptables -L -n + ``` + + Ensure there are no rules blocking UDP port 8443 or TCP ports 8045/8443. + +2. **Check cloud provider security groups**: + + Verify that security groups or network ACLs allow: + - Inbound UDP on port 8443 from client networks + - Inbound TCP on port 8045 from Mattermost server networks + - Inbound TCP on port 8443 (if TCP fallback is enabled) + +3. **Check intermediate firewalls**: + + - Corporate firewalls might block UDP traffic + - Some networks might require TURN servers for traversal + +## Log Analysis + +### RTCD Logs + +The RTCD service logs important events and errors. Set the log level to "debug" for troubleshooting: + +1. **In the configuration file**: + + ```toml + [logger] + enable_file = true + file_level = "DEBUG" + ``` + + Restart the RTCD service after making these changes + +2. **Common log patterns to look for**: + + - **Connection errors**: Look for "failed to connect" or "connection error" messages + - **ICE negotiation failures**: Look for "ICE failed" or "ICE timeout" messages + - **API authentication issues**: Look for "unauthorized" or "invalid API key" messages + +### Mattermost Logs + +Check the Mattermost server logs for Calls plugin related issues: + +1. **Enable debug logging** in System Console > Environment > Logging > File Log Level + +2. **Filter for Calls-related logs**: + + ```bash + grep -i "calls" /path/to/mattermost.log + ``` + +3. **Look for common patterns**: + + - Connection errors to RTCD + - Plugin initialization issues + - WebSocket connection problems + +### Browser Console Logs + +Instruct users to check their browser console logs: + +1. **In Chrome/Edge**: + - Press F12 to open Developer Tools + - Go to the Console tab + - Look for errors related to WebRTC, Calls, or media permissions + +2. **Specific patterns to look for**: + + - "getUserMedia" errors (microphone permission issues) + - "ICE connection" failures + - WebSocket connection errors + +## Performance Issues + +### Diagnosing High CPU Usage + +If RTCD servers show high CPU usage: + +1. **Check concurrent calls and participants**: + + - Access the Prometheus metrics endpoint to see active sessions + - Compare with the benchmark data in the {doc}`Calls Metrics and Monitoring ` documentation's Performance Baselines section + +2. **Profile CPU usage** (Linux): + + ```bash + top -p $(pgrep rtcd) + ``` + + Or for detailed per-thread usage: + + ```bash + ps -eLo pid,ppid,tid,pcpu,comm | grep rtcd + ``` + +3. **Enable pprof profiling** (if needed): + + Add to your RTCD configuration: + + ```json + { + "debug": { + "pprof": true, + "pprofPort": 6060 + } + } + ``` + + Then capture a CPU profile: + + ```bash + curl http://localhost:6060/debug/pprof/profile > cpu.profile + ``` + + Analyze with: + + ```bash + go tool pprof -http=:8080 cpu.profile + ``` + +### Diagnosing Network Bottlenecks + +If you suspect network bandwidth issues: + +1. **Monitor network utilization**: + + ```bash + iftop -n + ``` + +2. **Check for packet drops**: + + ```bash + netstat -su | grep -E 'drop|error' + ``` + +3. **Verify system network buffers**: + + ```bash + sysctl -a | grep net.core.rmem + sysctl -a | grep net.core.wmem + ``` + + Ensure these match the recommended values: + + ```bash + net.core.rmem_max = 16777216 + net.core.wmem_max = 16777216 + net.core.optmem_max = 16777216 + ``` + +## Recording and Transcription Issues + +For troubleshooting calls-offloader service issues including recording and transcription problems, see the [Calls Offloader Setup and Configuration](calls-offloader-setup.md#troubleshooting) guide. + +### Calls-Offloader Docker Debugging + +If you're running calls-offloader in Docker, use these commands for debugging: + +#### Monitor Live Logs + +For easier log management, configure calls-offloader to capture job logs directly in the calls-offloader log file by adding the following to your `calls-offloader.toml`: + +```toml +[jobs.docker] +# Whether to output job logs to the console. Default is false. +output_logs = true +``` + +With this configuration enabled, all logs from recorder and transcriber docker containers will be written to the main calls-offloader log file. + +#### View Completed Jobs + +To view completed calls-offloader job containers (useful for debugging failed jobs): + +```bash +# List all exited containers to see completed jobs +docker ps -a --filter "status=exited" +``` + +Look for containers with recorder and transcriber image names that have exited. You can then examine their logs: + +```bash +# View logs from a specific completed container +docker logs +``` + +#### Additional Docker Debugging Tips + +- **Check container resource usage**: `docker stats` to see if containers are hitting resource limits +- **Inspect container configuration**: `docker inspect ` for detailed container settings +- **Check container health**: `docker inspect | grep Health` if health checks are configured + +## Prometheus Metrics Analysis + +Use Prometheus metrics for real-time and historical performance data: + +For detailed setup instructions on configuring Prometheus and Grafana for Calls monitoring, see the {doc}`Calls Metrics and Monitoring ` guide. + +## When to Contact Support + +Consider contacting Mattermost Support when: + +1. You've tried troubleshooting steps without resolution +2. You're experiencing persistent connection failures across multiple clients +3. You notice unexpected or degraded performance despite proper configuration +4. You need help interpreting diagnostic information +5. You suspect a bug in the Calls plugin or RTCD service + +When contacting support, please include: + +- RTCD version and configuration (with sensitive information redacted) +- Mattermost server version +- Calls plugin version +- Client environments (browsers, OS versions) +- Relevant logs and diagnostic information +- Detailed description of the issue and steps to reproduce + +## Other Calls Documentation + +- [Calls Overview](calls-deployment.md): Overview of deployment options and architecture +- [RTCD Setup and Configuration](calls-rtcd-setup.md): Comprehensive guide for setting up the dedicated RTCD service +- [Calls Offloader Setup and Configuration](calls-offloader-setup.md): Setup guide for call recording and transcription +- [Calls Metrics and Monitoring](calls-metrics-monitoring.md): Guide to monitoring Calls performance using metrics and observability +- [Calls Deployment on Kubernetes](calls-kubernetes.md): Detailed guide for deploying Calls in Kubernetes environments \ No newline at end of file diff --git a/source/configure/plugins-configuration-settings.rst b/source/configure/plugins-configuration-settings.rst index fed0238ea68..d92c4159a26 100644 --- a/source/configure/plugins-configuration-settings.rst +++ b/source/configure/plugins-configuration-settings.rst @@ -532,7 +532,7 @@ ICE servers configurations - The configurations above, containing STUN and TURN servers, are sent to the clients and used to generate local candidates. - If hosting calls through the plugin (i.e. not using the |rtcd_service|) any configured STUN server may also be used to find the instance's public IP when none is provided through the |ice_host_override_link| option. -.. |rtcd_service| replace:: :ref:`rtcd service ` +.. |rtcd_service| replace:: :doc:`RTCD service ` **Example** @@ -764,7 +764,7 @@ Call recording quality +-----------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------+ .. note:: - The quality setting will affect the performance of the job service and the file size of recordings. Refer to the :ref:`deployment section ` for more information. + The quality setting will affect the performance of the job service and the file size of recordings. Refer to the :ref:`call recording and transcription section ` for more information. .. config:setting:: enable-pluginscalltranscriptions :displayname: Enable call transcriptions (Plugins - Calls) @@ -814,7 +814,7 @@ Transcriber model size +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ .. note:: - This setting is available starting in plugin version 0.22. The model size setting will affect the performance of the job service. Refer to the :ref:`configure call recordings, transcriptions, and live captions ` documentation for more information. + This setting is available starting in plugin version 0.22. The model size setting will affect the performance of the job service. Refer to the :ref:`call recording and transcription section ` documentation for more information. .. config:setting:: call-transcriber-threads :displayname: Call transcriber threads (Plugins - Calls) @@ -837,7 +837,7 @@ Call transcriber threads +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------+ .. note:: - The call transcriber threads setting will affect the performance of the job service. Refer to the :ref:`configure call recordings, transcriptions, and live captions ` documentation for more information. This setting is available starting in plugin version 0.26.2. + The call transcriber threads setting will affect the performance of the job service. Refer to the :ref:`call recording and transcription section ` documentation for more information. This setting is available starting in plugin version 0.26.2. .. config:setting:: enable-pluginslivecaptions :displayname: (Experimental) Enable live captions (Plugins - Calls) diff --git a/source/deploy/server/containers/install-docker.rst b/source/deploy/server/containers/install-docker.rst index 84d29e0d35d..dfd60ddb921 100644 --- a/source/deploy/server/containers/install-docker.rst +++ b/source/deploy/server/containers/install-docker.rst @@ -183,7 +183,7 @@ Looking for a way to evaluate Mattermost on a single local machine using Docker? - This local image is self-contained (i.e., it has an internal database and works out of the box). Dropping a container using this image removes data and configuration as expected. You can see the :doc:`configuration settings ` documentation to learn more about customizing your trial deployment. - **Preview Mode** shouldn't be used in a production environment, as it uses a known password string, contains other non-production configuration settings, has email disabled, keeps no persistent data (all data lives inside the container), and doesn't support upgrades. - - If you are planning to use the calling functionality in **Preview Mode** on a non-local environment, you should ensure that the server is running on a secure (HTTPs) connection and that the :ref:`network requirements ` to run calls are met. + - If you are planning to use the calling functionality in **Preview Mode** on a non-local environment, you should ensure that the server is running on a secure (HTTPs) connection and that the :ref:`network requirements ` to run calls are met. 1. Install `Docker `__. diff --git a/source/deploy/server/preparations.rst b/source/deploy/server/preparations.rst index 7a0e8c10661..d693905ae12 100644 --- a/source/deploy/server/preparations.rst +++ b/source/deploy/server/preparations.rst @@ -10,7 +10,7 @@ This guide outlines the key preparation steps required before installing the Mat Review software and hardware requirements Set up an NGINX proxy - Configure Mattermost Calls + Configure Mattermost Calls Set up TLS Use an image proxy diff --git a/source/images/calls-channel-enable-disable.png b/source/images/calls-channel-enable-disable.png new file mode 100644 index 00000000000..a2751779dad Binary files /dev/null and b/source/images/calls-channel-enable-disable.png differ