From 7f1d3ee64841188d6368b99ee0d2ccaf1e6ced75 Mon Sep 17 00:00:00 2001 From: "adam.rashid96" Date: Sat, 2 Nov 2024 03:34:37 -0400 Subject: [PATCH 1/2] Controller Support for localAnchor + debugging MotionControllerActuator --- docs/tutorials/physics/mocap_control.md | 78 +------------------------ docs/tutorials/physics/mocap_control.py | 7 ++- vuer/schemas/physics_components.py | 31 ++++++++++ vuer/schemas/scene_components.py | 44 ++++++++++++++ 4 files changed, 80 insertions(+), 80 deletions(-) diff --git a/docs/tutorials/physics/mocap_control.md b/docs/tutorials/physics/mocap_control.md index 2b64469..e87ad62 100644 --- a/docs/tutorials/physics/mocap_control.md +++ b/docs/tutorials/physics/mocap_control.md @@ -10,7 +10,7 @@ from textwrap import dedent from killport import kill_ports from vuer import Vuer, VuerSession from vuer.events import ClientEvent -from vuer.schemas import MuJoCo, Scene, Fog, Sphere, Hands, HandActuator +from vuer.schemas import MuJoCo, Scene, Fog, Sphere, Hands, HandActuator, MotionControllers, MotionControllerActuator ``` @@ -74,79 +74,3 @@ Now, to get the mujoco updates, we can listen to the `ON_MUJOCO_FRAME` event. async def on_mujoco_frame(event: ClientEvent, sess: VuerSession): print("ON_MUJOCO_FRAME", event) ``` - -You should get events that looks like these: - -![mujoco_frame_event](figures/mujoco_frame_event.png) - -and the main session handler. - -```python -@app.spawn(start=True) -async def main(sess: VuerSession): - # here we setup the staging area. Use Fog to simulate MuJoCo's - # default style. - sess.set @ Scene( - # grid=False, - bgChildren=[ - Fog(color=0x2C3F57, near=10, far=20), - Hands(), - Sphere( - args=[50, 10, 10], - materialType="basic", - material=dict(color=0x2C3F57, side=1), - ), - ], - ) - await sleep(0.0005) - sess.upsert @ MuJoCo( - - HandActuator(key="pinch-on-squeeze"), - - key="franka-gripper", - src=asset_pref + "scene.xml", - assets=[asset_pref + fn for fn in ASSETS_LIST], - useLights=True, - timeout=1000000, - scale=0.1, - ) - - await sleep(100.0) -``` - - -```{admonition} Note -:class: tip - -Note the `HandActuator` that is used to add control to the -gripper fingers. This is currently hand-specific. - - class HandActuator(SceneElement): - tag = "HandActuator" - - ctrlId = -1 - offset = 0.01 - low = 0.01 - high = 1.0 - cond = 'right-squeeze' - value = "right:thumb-tip,right:index-finger-tip" - scale = 1.0 -``` - -This correspond to the following actuator definition -in the MJCF file: - -```xml -... - - - - - -... - - - - -``` diff --git a/docs/tutorials/physics/mocap_control.py b/docs/tutorials/physics/mocap_control.py index 08315f9..c2555ea 100644 --- a/docs/tutorials/physics/mocap_control.py +++ b/docs/tutorials/physics/mocap_control.py @@ -3,7 +3,7 @@ from cmx import doc -MAKE_DOCS = os.getenv("MAKE_DOCS", True) +MAKE_DOCS = os.getenv("MAKE_DOCS", False) doc @ """ # MuJoCo VR Mocap Example @@ -17,7 +17,7 @@ from killport import kill_ports from vuer import Vuer, VuerSession from vuer.events import ClientEvent - from vuer.schemas import MuJoCo, Scene, Fog, Sphere, Hands, HandActuator + from vuer.schemas import MuJoCo, Scene, Fog, Sphere, Hands, HandActuator, MotionControllers, MotionControllerActuator doc @ """ @@ -98,6 +98,7 @@ async def main(sess: VuerSession): bgChildren=[ Fog(color=0x2C3F57, near=10, far=20), Hands(), + MotionControllers(), Sphere( args=[50, 10, 10], materialType="basic", @@ -109,7 +110,7 @@ async def main(sess: VuerSession): sess.upsert @ MuJoCo( HandActuator(key="pinch-on-squeeze"), - + MotionControllerActuator(), key="franka-gripper", src=asset_pref + "scene.xml", assets=[asset_pref + fn for fn in ASSETS_LIST], diff --git a/vuer/schemas/physics_components.py b/vuer/schemas/physics_components.py index b5f2bda..00a17a8 100644 --- a/vuer/schemas/physics_components.py +++ b/vuer/schemas/physics_components.py @@ -49,3 +49,34 @@ class HandActuator(SceneElement): cond = 'right-squeeze' value = "right:thumb-tip,right:index-finger-tip" scale = 1.0 + +class MotionControllerActuator(SceneElement): + """ + MotionControllerActuator component for actuating the MuJoCo simulation based on motion controller inputs. + + :param ctrlId: The control ID in the MuJoCo simulation to actuate. + :type ctrlId: int + :param low: The minimum value for actuation. + :type low: float + :param high: The maximum value for actuation. + :type high: float + :param cond: The condition for actuation, e.g., 'right-trigger'. + :type cond: str + :param scale: The scaling factor applied to the input value for actuation. + :type scale: float + """ + tag = "MotionControllerActuator" + + ctrlId: int = -1 + low: float = 0.0 + high: float = 1.0 + cond: str = 'right-trigger' + scale: float = 1.0 + + def __init__(self, ctrlId=-1, low=0.0, high=1.0, cond='right-trigger', scale=1.0, **kwargs): + super().__init__(**kwargs) + self.ctrlId = ctrlId + self.low = low + self.high = high + self.cond = cond + self.scale = scale \ No newline at end of file diff --git a/vuer/schemas/scene_components.py b/vuer/schemas/scene_components.py index 8c5e688..f83fa15 100644 --- a/vuer/schemas/scene_components.py +++ b/vuer/schemas/scene_components.py @@ -628,6 +628,49 @@ def __init__( **kwargs, ) +class MotionControllers(SceneElement): + """ + MotionController for tracking XR controller poses and button states. + + :param key: Unique identifier for the controller. + :type key: str, optional + :param eventTypes: A tuple or list of events to track (e.g., "trigger", "squeeze"). + :type eventTypes: tuple or list, optional + :param stream: Whether to enable streaming of controller data. + :type stream: bool, optional + :param left: Boolean indicating if the left controller should be tracked. + :type left: bool, optional + :param right: Boolean indicating if the right controller should be tracked. + :type right: bool, optional + :param showLeft: Boolean indicating if the left controller visualization should be shown. + :type showLeft: bool, optional + :param showRight: Boolean indicating if the right controller visualization should be shown. + :type showRight: bool, optional + """ + + tag = "MotionControllers" + + def __init__( + self, + key="motionControllers", + eventTypes=("trigger", "squeeze"), + stream=True, + left=None, + right=None, + showLeft=True, + showRight=True, + **kwargs, + ): + super().__init__( + key=key, + eventTypes=eventTypes, + stream=stream, + left=left, + right=right, + showLeft=showLeft, + showRight=showRight, + **kwargs, + ) class Obj(SceneElement): tag = "Obj" @@ -771,6 +814,7 @@ def __init__( AmbientLight(key="ambient", intensity=0.25), PointLight(key="spot", intensity=1, position=[0, 1, 1]), Hands(fps=30, eventType=["squeeze"], stream=True), + MotionControllers(fps=30, eventType=["trigger", "squeeze"], stream=True), ] self.up = up From 30ce95786c76d88955a4189354c822345700c4bd Mon Sep 17 00:00:00 2001 From: "adam.rashid96" Date: Mon, 4 Nov 2024 22:18:22 -0500 Subject: [PATCH 2/2] Fixed motionControllerActuator --- docs/tutorials/physics/mocap_control.md | 40 +++++++++++++++++++++++++ docs/tutorials/physics/mocap_control.py | 6 ++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/physics/mocap_control.md b/docs/tutorials/physics/mocap_control.md index e87ad62..d65b5a0 100644 --- a/docs/tutorials/physics/mocap_control.md +++ b/docs/tutorials/physics/mocap_control.md @@ -74,3 +74,43 @@ Now, to get the mujoco updates, we can listen to the `ON_MUJOCO_FRAME` event. async def on_mujoco_frame(event: ClientEvent, sess: VuerSession): print("ON_MUJOCO_FRAME", event) ``` + +You should get events that looks like these: + +![mujoco_frame_event](figures/mujoco_frame_event.png) + +and the main session handler. + +```python +@app.spawn(start=True) +async def main(sess: VuerSession): + # here we setup the staging area. Use Fog to simulate MuJoCo's + # default style. + sess.set @ Scene( + # grid=False, + bgChildren=[ + Fog(color=0x2C3F57, near=10, far=20), + # Hands(), + MotionControllers(), + Sphere( + args=[50, 10, 10], + materialType="basic", + material=dict(color=0x2C3F57, side=1), + ), + ], + ) + await sleep(0.0005) + sess.upsert @ MuJoCo( + + # HandActuator(key="pinch-on-squeeze"), + MotionControllerActuator(high=0.15, low=0.01, ctrlId=-1), + key="franka-gripper", + src=asset_pref + "scene.xml", + assets=[asset_pref + fn for fn in ASSETS_LIST], + useLights=True, + timeout=1000000, + scale=0.1, + ) + + await sleep(100.0) +``` diff --git a/docs/tutorials/physics/mocap_control.py b/docs/tutorials/physics/mocap_control.py index c2555ea..e2c3e6f 100644 --- a/docs/tutorials/physics/mocap_control.py +++ b/docs/tutorials/physics/mocap_control.py @@ -97,7 +97,7 @@ async def main(sess: VuerSession): # grid=False, bgChildren=[ Fog(color=0x2C3F57, near=10, far=20), - Hands(), + # Hands(), MotionControllers(), Sphere( args=[50, 10, 10], @@ -109,8 +109,8 @@ async def main(sess: VuerSession): await sleep(0.0005) sess.upsert @ MuJoCo( - HandActuator(key="pinch-on-squeeze"), - MotionControllerActuator(), + # HandActuator(key="pinch-on-squeeze"), + MotionControllerActuator(high=0.15, low=0.01, ctrlId=-1), key="franka-gripper", src=asset_pref + "scene.xml", assets=[asset_pref + fn for fn in ASSETS_LIST],