Add quest support for local loco-manipulation

* Switch delta time to use wall clock time instead of physics step time. Update config values to compensate.
* Add support for decoupling anchor rotation as a toggle when pressing right controller A button.
* Fix reset not working

Refactor motion controller specific code into OpenXRDeviceMotionController. If we are doing only "handtracking" as teleop_device, then we don't want all the headset anchoring code.

Change the pre render callbacks to be a list so other systems can hook into it as well

Rename classes from Controller to MotionController to limit confusion with Robotics controllers

Move the code to alter the stage until after the telop device has been instantiated so we know what device we dealing with and if we need to change the ground plane material

Format with ./isaaclab -f

Update changelog, contributors and extension.toml

Switch to rendering_dt instead of wall clock for determinism

Address AI code review comments.

Instead of having fixed heigh on by default, make it optional in the case of being able to go into crouch.

We don't need to add pre render to ManagerBasedEnv since that can be done through XRCore.

Moving new motion controller retargeters into new files

Moving OpenXRDevice motion controller logic backup into OpenXRDevice, and moving interface enums into DeviceBase to remove retargeter dependancy on OpenXRDevice

Adding retargeter requiremnts call

Removing bespoke xr env adjustments

Undo xr anchor height offset

Cleaning up teleop script

Code cleanup

Update code comments

Fix typo in RetargeterBase

Properly set sim.device for retargeters

Fixing greptile feedback

Fix logic error if xr anchor rotation is disabled on AR startup

Follow naming conventions for get_requirments

Author: hougantc-nvda

CONTRIBUTORS.md

@@ -72,6 +72,7 @@ Guidelines for modifications:
 * HoJin Jeon
 * Hongwei Xiong
 * Hongyu Li
+* Hougant Chen
 * Huihua Zhao
 * Iretiayo Akinola
 * Jack Zeng

docs/source/how-to/cloudxr_teleoperation.rst

@@ -721,11 +721,11 @@ Here's an example of setting up hand tracking:
 
    # Create retargeters
    position_retargeter = Se3AbsRetargeter(
-       bound_hand=OpenXRDevice.TrackingTarget.HAND_RIGHT,
+       bound_hand=DeviceBase.TrackingTarget.HAND_RIGHT,
        zero_out_xy_rotation=True,
        use_wrist_position=False  # Use pinch position (thumb-index midpoint) instead of wrist
    )
-   gripper_retargeter = GripperRetargeter(bound_hand=OpenXRDevice.TrackingTarget.HAND_RIGHT)
+   gripper_retargeter = GripperRetargeter(bound_hand=DeviceBase.TrackingTarget.HAND_RIGHT)
 
    # Create OpenXR device with hand tracking and both retargeters
    device = OpenXRDevice(
@@ -919,15 +919,15 @@ The retargeting system is designed to be extensible. You can create custom retar
                Any: The transformed control commands for the robot.
            """
            # Access hand tracking data using TrackingTarget enum
-           right_hand_data = data[OpenXRDevice.TrackingTarget.HAND_RIGHT]
+           right_hand_data = data[DeviceBase.TrackingTarget.HAND_RIGHT]
 
            # Extract specific joint positions and orientations
            wrist_pose = right_hand_data.get("wrist")
            thumb_tip_pose = right_hand_data.get("thumb_tip")
            index_tip_pose = right_hand_data.get("index_tip")
 
            # Access head tracking data
-           head_pose = data[OpenXRDevice.TrackingTarget.HEAD]
+           head_pose = data[DeviceBase.TrackingTarget.HEAD]
 
            # Process the tracking data and apply your custom logic
            # ...

scripts/environments/teleoperation/teleop_se3_agent.py

@@ -3,7 +3,11 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-"""Script to run a keyboard teleoperation with Isaac Lab manipulation environments."""
+"""Script to run teleoperation with Isaac Lab manipulation environments.
+
+Supports multiple input devices (e.g., keyboard, spacemouse, gamepad) and devices
+configured within the environment (including OpenXR-based hand tracking or motion
+controllers)."""
 
 """Launch Isaac Sim Simulator first."""
 
@@ -13,7 +17,7 @@
 from isaaclab.app import AppLauncher
 
 # add argparse arguments
-parser = argparse.ArgumentParser(description="Keyboard teleoperation for Isaac Lab environments.")
+parser = argparse.ArgumentParser(description="Teleoperation for Isaac Lab environments.")
 parser.add_argument("--num_envs", type=int, default=1, help="Number of environments to simulate.")
 parser.add_argument(
     "--teleop_device",
@@ -78,7 +82,7 @@
 
 def main() -> None:
     """
-    Run keyboard teleoperation with Isaac Lab manipulation environment.
+    Run teleoperation with an Isaac Lab manipulation environment.
 
     Creates the environment, sets up teleoperation interfaces and callbacks,
     and runs the main simulation loop until the application is closed.
@@ -98,8 +102,6 @@ def main() -> None:
         env_cfg.terminations.object_reached_goal = DoneTerm(func=mdp.object_reached_goal)
 
     if args_cli.xr:
-        # External cameras are not supported with XR teleop
-        # Check for any camera configs and disable them
         env_cfg = remove_camera_configs(env_cfg)
         env_cfg.sim.render.antialiasing_mode = "DLSS"
 
@@ -204,7 +206,7 @@ def stop_teleoperation() -> None:
                 )
             else:
                 logger.error(f"Unsupported teleop device: {args_cli.teleop_device}")
-                logger.error("Supported devices: keyboard, spacemouse, gamepad, handtracking")
+                logger.error("Configure the teleop device in the environment config.")
                 env.close()
                 simulation_app.close()
                 return
@@ -254,6 +256,7 @@ def stop_teleoperation() -> None:
 
                 if should_reset_recording_instance:
                     env.reset()
+                    teleop_interface.reset()
                     should_reset_recording_instance = False
                     print("Environment reset complete")
         except Exception as e:

source/isaaclab/config/extension.toml

@@ -1,7 +1,7 @@
 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "0.48.3"
+version = "0.48.4"
 
 # Description
 title = "Isaac Lab framework for Robot Learning"

source/isaaclab/docs/CHANGELOG.rst

@@ -1,6 +1,21 @@
 Changelog
 ---------
 
+0.48.4 (2025-11-13)
+~~~~~~~~~~~~~~~~~~~
+
+Added
+^^^^^
+
+* Added OpenXR motion controller support for the G1 robot locomanipulation environment
+  ``Isaac-PickPlace-Locomanipulation-G1-Abs-v0``. This enables teleoperation using XR motion controllers
+  in addition to hand tracking.
+* Added :class:`OpenXRDeviceMotionController` for motion controller-based teleoperation with headset anchoring control.
+* Added motion controller-specific retargeters:
+  * :class:`G1TriHandControllerUpperBodyRetargeterCfg` for upper body and hand control using motion controllers.
+  * :class:`G1LowerBodyStandingControllerRetargeterCfg` for lower body control using motion controllers.
+
+
 0.48.3 (2025-11-13)
 ~~~~~~~~~~~~~~~~~~~
 

source/isaaclab/isaaclab/devices/device_base.py

@@ -9,6 +9,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from dataclasses import dataclass, field
+from enum import Enum
 from typing import Any
 
 from isaaclab.devices.retargeter_base import RetargeterBase, RetargeterCfg
@@ -58,9 +59,13 @@ def __init__(self, retargeters: list[RetargeterBase] | None = None):
         """
         # Initialize empty list if None is provided
         self._retargeters = retargeters or []
+        # Aggregate required features across all retargeters
+        self._required_features = set()
+        for retargeter in self._retargeters:
+            self._required_features.update(retargeter.get_requirements())
 
     def __str__(self) -> str:
-        """Returns: A string containing the information of joystick."""
+        """Returns: A string identifier for the device."""
         return f"{self.__class__.__name__}"
 
     """
@@ -123,3 +128,32 @@ def advance(self) -> torch.Tensor:
         # With multiple retargeters, return a tuple of outputs
         # Concatenate retargeted outputs into a single tensor
         return torch.cat([retargeter.retarget(raw_data) for retargeter in self._retargeters], dim=-1)
+
+    # -----------------------------
+    # Shared data layout helpers (for retargeters across devices)
+    # -----------------------------
+    class TrackingTarget(Enum):
+        """Standard tracking targets shared across devices."""
+
+        HAND_LEFT = 0
+        HAND_RIGHT = 1
+        HEAD = 2
+        CONTROLLER_LEFT = 3
+        CONTROLLER_RIGHT = 4
+
+    class MotionControllerDataRowIndex(Enum):
+        """Rows in the motion-controller 2x7 array."""
+
+        POSE = 0
+        INPUTS = 1
+
+    class MotionControllerInputIndex(Enum):
+        """Indices in the motion-controller input row."""
+
+        THUMBSTICK_X = 0
+        THUMBSTICK_Y = 1
+        TRIGGER = 2
+        SQUEEZE = 3
+        BUTTON_0 = 4
+        BUTTON_1 = 5
+        PADDING = 6

source/isaaclab/isaaclab/devices/openxr/__init__.py

@@ -7,4 +7,4 @@
 
 from .manus_vive import ManusVive, ManusViveCfg
 from .openxr_device import OpenXRDevice, OpenXRDeviceCfg
-from .xr_cfg import XrCfg, remove_camera_configs
+from .xr_cfg import XrAnchorRotationMode, XrCfg, remove_camera_configs

source/isaaclab/isaaclab/devices/openxr/manus_vive.py

@@ -13,7 +13,6 @@
 import numpy as np
 from collections.abc import Callable
 from dataclasses import dataclass
-from enum import Enum
 
 import carb
 from isaacsim.core.version import get_version
@@ -22,7 +21,6 @@
 from isaaclab.devices.retargeter_base import RetargeterBase
 
 from ..device_base import DeviceBase, DeviceCfg
-from .openxr_device import OpenXRDevice
 from .xr_cfg import XrCfg
 
 # For testing purposes, we need to mock the XRCore
@@ -61,13 +59,6 @@ class ManusVive(DeviceBase):
     data is transformed into robot control commands suitable for teleoperation.
     """
 
-    class TrackingTarget(Enum):
-        """Enum class specifying what to track with Manus+Vive. Consistent with :class:`OpenXRDevice.TrackingTarget`."""
-
-        HAND_LEFT = 0
-        HAND_RIGHT = 1
-        HEAD = 2
-
     TELEOP_COMMAND_EVENT_TYPE = "teleop_command"
 
     def __init__(self, cfg: ManusViveCfg, retargeters: list[RetargeterBase] | None = None):
@@ -192,9 +183,9 @@ def _get_raw_data(self) -> dict:
             joint_name = HAND_JOINT_MAP[int(index)]
             result[hand][joint_name] = np.array(pose["position"] + pose["orientation"], dtype=np.float32)
         return {
-            OpenXRDevice.TrackingTarget.HAND_LEFT: result["left"],
-            OpenXRDevice.TrackingTarget.HAND_RIGHT: result["right"],
-            OpenXRDevice.TrackingTarget.HEAD: self._calculate_headpose(),
+            DeviceBase.TrackingTarget.HAND_LEFT: result["left"],
+            DeviceBase.TrackingTarget.HAND_RIGHT: result["right"],
+            DeviceBase.TrackingTarget.HEAD: self._calculate_headpose(),
         }
 
     def _calculate_headpose(self) -> np.ndarray: