Fix OVRTX scene-camera black frames from unscoped render-var lookups

source/isaaclab_ov/changelog.d/mtrepte-ovrtx-render-var-scope.rst

@@ -0,0 +1,9 @@
+Fixed
+^^^^^
+
+* Fixed :class:`~isaaclab_ov.renderers.ovrtx_renderer.OVRTXRenderer` producing all-black frames
+  for scene camera sensors on OVRTX 0.5+. Render-var lookups used a fixed, unscoped
+  ``"/Render/Vars/<name>"`` key, but OVRTX 0.5+ authors each camera's render vars under its own
+  scope (``"/RenderCamera_<id>/Vars/<name>"``), so the lookup silently missed every render var and
+  left the output buffer at its zero-initialized state. Render-var keys are now resolved per
+  camera from its actual render scope.

source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_compat.py

@@ -6,10 +6,13 @@
 """Version compatibility between the public OVRTX 0.4 API and OVRTX 0.5 and later.
 
 OVRTX 0.4 keys ``frame.render_vars`` by render-var source name (``"LdrColor"``), while
-0.5 keys it by the authored RenderVar prim path (``"/Render/Vars/LdrColor"``). The
-installed version cannot change while the process runs, so the key form is resolved once
-at import and published as :data:`RENDER_VAR_FRAME_KEYS`; per-frame code indexes that
-mapping instead of re-checking the version.
+0.5 keys it by the authored RenderVar prim path. :data:`RENDER_VAR_FRAME_KEYS` gives the
+*single-camera* (unscoped) form of that path (``"/Render/Vars/LdrColor"``) for callers
+that do not have a specific camera's render scope. OVRTX 0.5+ actually authors render
+vars under a *per-camera* scope (``"/RenderCamera_<id>/Vars/LdrColor"``, see
+``render_scope_name`` in :func:`build_render_product_as_string`), so per-frame reads that
+know which camera they are reading must use :func:`resolve_render_var_key` with that
+camera's scope name instead of indexing :data:`RENDER_VAR_FRAME_KEYS` directly.
 
 Missing or invalid version metadata selects source-name render-var keys.
 """
@@ -85,4 +88,37 @@ def build_render_var_frame_keys(version: Version | None) -> Mapping[str, str]:
 """Installed OVRTX version, or ``None`` when it is unavailable or unparsable."""
 
 RENDER_VAR_FRAME_KEYS: Mapping[str, str] = build_render_var_frame_keys(OVRTX_VERSION)
-"""Maps render-var source name to its ``frame.render_vars`` key for the installed OVRTX."""
+"""Maps render-var source name to its ``frame.render_vars`` key for the installed OVRTX.
+
+Uses the unscoped ``"/Render/Vars/<name>"`` path on OVRTX 0.5+, which does not match any
+render var actually authored for a specific camera (those live under
+``"/RenderCamera_<id>/Vars/<name>"``). Per-frame reads for a known camera must use
+:func:`resolve_render_var_key` with that camera's render scope instead.
+"""
+
+
+def resolve_render_var_key(source: str, render_scope_name: str | None) -> str:
+    """Return the ``frame.render_vars`` key for *source* on one camera's render product.
+
+    On OVRTX 0.4, render vars are keyed by source name everywhere, so *render_scope_name*
+    is unused. On OVRTX 0.5+, each camera's render vars are authored under its own scope
+    (``"/RenderCamera_<id>/Vars/<name>"``, see ``render_scope_name`` in
+    :func:`~isaaclab_ov.renderers.ovrtx_usd.build_render_product_as_string`), so the key
+    must be rebuilt per camera rather than read from :data:`RENDER_VAR_FRAME_KEYS`, whose
+    unscoped path never matches an authored render var.
+
+    Args:
+        source: Render-var source name (e.g. ``"LdrColor"``).
+        render_scope_name: The camera's render scope (e.g. ``"RenderCamera_0"``), or
+            ``None`` when the caller does not know it -- falls back to the unscoped path,
+            which is only correct on OVRTX 0.4.
+
+    Returns:
+        The key to index into ``frame.render_vars`` for this camera and source.
+    """
+    if not uses_prim_path_render_vars(OVRTX_VERSION):
+        return source
+    unscoped_path = render_var_prim_paths_by_source().get(source, source)
+    if render_scope_name is None:
+        return unscoped_path
+    return unscoped_path.replace("/Render/", f"/{render_scope_name}/", 1)

source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py

@@ -79,7 +79,7 @@
     decode_stable_id_map,
     decode_stable_id_semantic_id_map,
 )
-from isaaclab_ov.renderers.ovrtx_compat import RENDER_VAR_FRAME_KEYS
+from isaaclab_ov.renderers.ovrtx_compat import resolve_render_var_key
 from isaaclab_ov.renderers.ovrtx_renderer_cfg import OVRTXBackendCfg, OVRTXRendererCfg
 from isaaclab_ov.renderers.ovrtx_renderer_kernels import (
     compute_cable_points_world_kernel,
@@ -112,33 +112,40 @@
 
 from isaaclab.renderers.camera_render_spec import CameraRenderSpec
 
-# ``frame.render_vars`` keys of the render vars read below. Baked at import from the installed
-# OVRTX version, which decides whether frames are keyed by source name or RenderVar prim path.
-_LDR_COLOR_VAR = RENDER_VAR_FRAME_KEYS["LdrColor"]
+# Render-var source names read below. The ``frame.render_vars`` key for a given source depends
+# on the installed OVRTX version *and*, on OVRTX 0.5+, on which camera's render product is being
+# read (each camera authors its render vars under its own scope) -- see
+# ``resolve_render_var_key`` and ``OVRTXCameraRenderData.render_scope_name``.
+_LDR_COLOR_SOURCE = "LdrColor"
+_HDR_COLOR_SOURCE = "HdrColor"
+_ALBEDO_SOURCE = "DiffuseAlbedoSD"
+_NORMALS_SOURCE = "NormalSD"
+_MOTION_VECTORS_SOURCE = "TargetMotionSD"
+_SEMANTIC_SEGMENTATION_SOURCE = "SemanticSegmentation"
+_INSTANCE_SEGMENTATION_SOURCE = "NonStableInstanceSegmentation"
+_SEMANTIC_ID_MAP_SOURCE = "SemanticIdMap"
+_STABLE_ID_MAP_SOURCE = "StableIdMap"
+_STABLE_ID_SEMANTIC_ID_MAP_SOURCE = "StableIdSemanticIdMap"
+
+# Render-var sources needed to decode the instance-segmentation info dicts.
+_INSTANCE_SEGMENTATION_MAP_SOURCES = (
+    _STABLE_ID_SEMANTIC_ID_MAP_SOURCE,
+    _STABLE_ID_MAP_SOURCE,
+    _SEMANTIC_ID_MAP_SOURCE,
+)
+
 _CAMERA_INTRINSIC_ATTRIBUTES = (
     "focalLength",
     "horizontalAperture",
     "verticalAperture",
     "horizontalApertureOffset",
     "verticalApertureOffset",
 )
-_HDR_COLOR_VAR = RENDER_VAR_FRAME_KEYS["HdrColor"]
-_ALBEDO_VAR = RENDER_VAR_FRAME_KEYS["DiffuseAlbedoSD"]
-_NORMALS_VAR = RENDER_VAR_FRAME_KEYS["NormalSD"]
-_MOTION_VECTORS_VAR = RENDER_VAR_FRAME_KEYS["TargetMotionSD"]
-_SEMANTIC_SEGMENTATION_VAR = RENDER_VAR_FRAME_KEYS["SemanticSegmentation"]
-_INSTANCE_SEGMENTATION_VAR = RENDER_VAR_FRAME_KEYS["NonStableInstanceSegmentation"]
-_SEMANTIC_ID_MAP_VAR = RENDER_VAR_FRAME_KEYS["SemanticIdMap"]
-_STABLE_ID_MAP_VAR = RENDER_VAR_FRAME_KEYS["StableIdMap"]
-_STABLE_ID_SEMANTIC_ID_MAP_VAR = RENDER_VAR_FRAME_KEYS["StableIdSemanticIdMap"]
-
-# Map render vars needed to decode the instance-segmentation info dicts.
-_INSTANCE_SEGMENTATION_MAP_VARS = (_STABLE_ID_SEMANTIC_ID_MAP_VAR, _STABLE_ID_MAP_VAR, _SEMANTIC_ID_MAP_VAR)
-
-# Maps depth render vars to compatible output buffers.
+
+# Maps depth render-var sources to compatible output buffers.
 _DEPTH_VAR_BUFFER_KEYS: dict[str, tuple[str, ...]] = {
-    RENDER_VAR_FRAME_KEYS["DistanceToImagePlaneSD"]: ("depth", "distance_to_image_plane"),
-    RENDER_VAR_FRAME_KEYS["DistanceToCameraSD"]: ("distance_to_camera",),
+    "DistanceToImagePlaneSD": ("depth", "distance_to_image_plane"),
+    "DistanceToCameraSD": ("distance_to_camera",),
 }
 
 # The resolved integer value is assigned to the ``omni:rtx:minimal:mode`` attribute of the render product.
@@ -304,6 +311,10 @@ class OVRTXCameraRenderData:
     def __init__(self, spec: CameraRenderSpec, device):
         """Create render data from a camera render specification."""
         self.render_product_path: str | None = None
+        # Set by the renderer right after computing this camera's render scope (e.g.
+        # "RenderCamera_0"). Needed to resolve this camera's frame.render_vars keys on OVRTX 0.5+,
+        # which authors render vars per-camera instead of under one shared scope.
+        self.render_scope_name: str | None = None
         self.camera_xform_binding = None
         self.camera_xform_query = None
         self.resources = contextlib.ExitStack()
@@ -578,6 +589,7 @@ def _initialize_camera_render_data_from_spec_legacy(
             raise RuntimeError("Expected an exported USD string from stage")
 
         scope = f"RenderCamera_{self._next_camera_id}"
+        render_data.render_scope_name = scope
         render_product_string, render_product_path = build_render_product_as_string(
             width=width,
             height=height,
@@ -994,6 +1006,7 @@ def _register_camera(self, spec: CameraRenderSpec, render_data: OVRTXCameraRende
         if not camera_paths or not camera_paths[0].startswith("/World/envs/env_0/"):
             raise ValueError("OVRTX cameras must be under /World/envs/env_0/.")
         scope = f"RenderCamera_{self._next_camera_id}"
+        render_data.render_scope_name = scope
         data_types = list(spec.cfg.data_types or ["rgb"])
         if spec.cfg.isp_cfg is not None and "rgb_hdr" not in data_types:
             data_types.append("rgb_hdr")
@@ -1344,7 +1357,7 @@ def _process_id_segmentation_render_var(
         render_data: OVRTXCameraRenderData,
         frame,
         output_buffers: dict,
-        render_var_key: str,
+        render_var_source: str,
         buffer_key: str,
         colorize: bool,
     ) -> None:
@@ -1358,11 +1371,12 @@ def _process_id_segmentation_render_var(
             render_data: OVRTX render data for the current frame.
             frame: OVRTX frame holding the mapped render vars.
             output_buffers: Destination warp buffers, keyed by data type.
-            render_var_key: ``frame.render_vars`` key of the OVRTX render var to read.
+            render_var_source: Source name of the OVRTX render var to read, resolved against
+                ``render_data``'s render scope.
             buffer_key: Data type key into ``output_buffers``.
             colorize: If True, IDs are mapped to RGBA colors; otherwise raw uint32 IDs are copied.
         """
-        render_var = frame.render_vars.get(render_var_key)
+        render_var = frame.render_vars.get(self._render_var_key(render_data, render_var_source))
         if render_var is None or buffer_key not in output_buffers:
             return
 
@@ -1403,7 +1417,7 @@ def _process_semantic_id_map(self, render_data: OVRTXCameraRenderData, frame) ->
             render_data: OVRTX render data for the current frame.
             frame: OVRTX frame holding the mapped render vars.
         """
-        semantic_id_map = frame.render_vars.get(_SEMANTIC_ID_MAP_VAR)
+        semantic_id_map = frame.render_vars.get(self._render_var_key(render_data, _SEMANTIC_ID_MAP_SOURCE))
         if semantic_id_map is None:
             return
 
@@ -1440,19 +1454,22 @@ def _process_instance_segmentation_maps(self, render_data: OVRTXCameraRenderData
             render_data: OVRTX render data for the current frame.
             frame: OVRTX frame holding the mapped render vars.
         """
-        resolved = {key: frame.render_vars.get(key) for key in _INSTANCE_SEGMENTATION_MAP_VARS}
-        missing = [key for key, render_var in resolved.items() if render_var is None]
+        resolved = {
+            source: frame.render_vars.get(self._render_var_key(render_data, source))
+            for source in _INSTANCE_SEGMENTATION_MAP_SOURCES
+        }
+        missing = [source for source, render_var in resolved.items() if render_var is None]
         if missing:
             raise RuntimeError(
                 f"instance_segmentation was requested but the following render vars are missing from the "
                 f"OVRTX frame: {missing}. Available vars: {list(frame.render_vars.keys())}"
             )
 
-        with resolved[_STABLE_ID_SEMANTIC_ID_MAP_VAR].map(device=Device.CPU) as mapping:
+        with resolved[_STABLE_ID_SEMANTIC_ID_MAP_SOURCE].map(device=Device.CPU) as mapping:
             stable_id_semantic_id_map = decode_stable_id_semantic_id_map(np.from_dlpack(mapping))
-        with resolved[_STABLE_ID_MAP_VAR].map(device=Device.CPU) as mapping:
+        with resolved[_STABLE_ID_MAP_SOURCE].map(device=Device.CPU) as mapping:
             stable_id_to_path = decode_stable_id_map(np.from_dlpack(mapping))
-        with resolved[_SEMANTIC_ID_MAP_VAR].map(device=Device.CPU) as mapping:
+        with resolved[_SEMANTIC_ID_MAP_SOURCE].map(device=Device.CPU) as mapping:
             semantic_id_to_labels = decode_semantic_id_map(np.from_dlpack(mapping))
 
         id_to_labels, id_to_semantics = build_instance_id_to_labels_and_semantics(
@@ -1570,14 +1587,18 @@ def _prepare_ppisp_hdr_source(
         # assignment.
         return wp.clone(tiled_data, device=output_device)
 
+    def _render_var_key(self, render_data: OVRTXCameraRenderData, source: str) -> str:
+        """Return *render_data*'s camera's ``frame.render_vars`` key for a render-var source name."""
+        return resolve_render_var_key(source, render_data.render_scope_name)
+
     def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, output_buffers: dict) -> None:
         """Extract RGB, depth, albedo, and semantic from a single render frame into output_buffers."""
         # Reset per-output metadata so it is a snapshot of this frame only. Unlike pixel AOVs (always
         # present), metadata like the semantic ``idToLabels`` is only repopulated below when its render var
         # is available, so without this a missing SemanticIdMap on a later frame would leave a stale mapping.
         render_data.renderer_info.clear()
 
-        ldr_color = frame.render_vars.get(_LDR_COLOR_VAR)
+        ldr_color = frame.render_vars.get(self._render_var_key(render_data, _LDR_COLOR_SOURCE))
         if ldr_color is not None:
             buffer_key = None
 
@@ -1595,8 +1616,8 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu
                 with self._map_render_var_to_dlpack(ldr_color) as tiled_data:
                     self._extract_rgba_tiles(render_data, tiled_data, output_buffers, buffer_key)
 
-        for depth_var, buffer_keys in _DEPTH_VAR_BUFFER_KEYS.items():
-            depth_render_var = frame.render_vars.get(depth_var)
+        for depth_source, buffer_keys in _DEPTH_VAR_BUFFER_KEYS.items():
+            depth_render_var = frame.render_vars.get(self._render_var_key(render_data, depth_source))
             if depth_render_var is None:
                 continue
             if not any(buffer_key in output_buffers for buffer_key in buffer_keys):
@@ -1608,12 +1629,12 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu
                     )
                 self._extract_depth_tiles(render_data, tiled_depth_data, output_buffers, buffer_keys)
 
-        albedo_var = frame.render_vars.get(_ALBEDO_VAR)
+        albedo_var = frame.render_vars.get(self._render_var_key(render_data, _ALBEDO_SOURCE))
         if albedo_var is not None and "albedo" in output_buffers:
             with self._map_render_var_to_dlpack(albedo_var) as tiled_albedo_data:
                 self._extract_rgba_tiles(render_data, tiled_albedo_data, output_buffers, "albedo", suffix="albedo")
 
-        hdr_color = frame.render_vars.get(_HDR_COLOR_VAR)
+        hdr_color = frame.render_vars.get(self._render_var_key(render_data, _HDR_COLOR_SOURCE))
         if hdr_color is not None and "rgb_hdr" in output_buffers:
             with self._map_render_var_to_dlpack(hdr_color) as tiled_hdr_data:
                 tiled_hdr_data = self._prepare_ppisp_hdr_source(render_data, tiled_hdr_data, output_buffers)
@@ -1623,7 +1644,7 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu
             render_data,
             frame,
             output_buffers,
-            _SEMANTIC_SEGMENTATION_VAR,
+            _SEMANTIC_SEGMENTATION_SOURCE,
             "semantic_segmentation",
             self.cfg.colorize_semantic_segmentation,
         )
@@ -1635,7 +1656,7 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu
             render_data,
             frame,
             output_buffers,
-            _INSTANCE_SEGMENTATION_VAR,
+            _INSTANCE_SEGMENTATION_SOURCE,
             "instance_segmentation",
             self.cfg.colorize_instance_segmentation,
         )
@@ -1644,15 +1665,15 @@ def _process_render_frame(self, render_data: OVRTXCameraRenderData, frame, outpu
         if "instance_segmentation" in output_buffers:
             self._process_instance_segmentation_maps(render_data, frame)
 
-        normals_var = frame.render_vars.get(_NORMALS_VAR)
+        normals_var = frame.render_vars.get(self._render_var_key(render_data, _NORMALS_SOURCE))
         if normals_var is not None and "normals" in output_buffers:
             with self._map_render_var_to_dlpack(normals_var) as tiled_normals_data:
                 self._launch_extract_all_tiles(render_data, tiled_normals_data, output_buffers["normals"])
 
         # For motion vectors, extract only the first two (u, v) channels from the tiled buffer.
         # Note: mirrors the Isaac RTX renderer's handling of the "TargetMotionSD" AOV
         # (check: https://github.com/isaac-sim/IsaacLab/issues/2003).
-        motion_var = frame.render_vars.get(_MOTION_VECTORS_VAR)
+        motion_var = frame.render_vars.get(self._render_var_key(render_data, _MOTION_VECTORS_SOURCE))
         if motion_var is not None and "motion_vectors" in output_buffers:
             with self._map_render_var_to_dlpack(motion_var) as tiled_motion_vectors_data:
                 self._launch_extract_all_tiles(render_data, tiled_motion_vectors_data, output_buffers["motion_vectors"])
@@ -1955,6 +1976,7 @@ def _initialize_camera_render_data_from_spec_ovstage(
             raise RuntimeError("Expected an exported USD string from stage")
 
         scope = f"RenderCamera_{self._next_camera_id}"
+        render_data.render_scope_name = scope
         render_product_string, render_product_path = build_render_product_as_string(
             width=width,
             height=height,

source/isaaclab_ov/test/test_ovrtx_compat.py

@@ -25,18 +25,22 @@
 ]
 
 if not _MISSING_MODULES:
+    from isaaclab_ov.renderers import ovrtx_compat as ovrtx_compat_module  # noqa: E402
     from isaaclab_ov.renderers.ovrtx_compat import (  # noqa: E402
         RENDER_VAR_FRAME_KEYS,
         build_render_var_frame_keys,
         detect_ovrtx_version,
+        resolve_render_var_key,
         uses_prim_path_render_vars,
     )
     from isaaclab_ov.renderers.ovrtx_usd import render_var_prim_paths_by_source  # noqa: E402
 else:
+    ovrtx_compat_module = None
     RENDER_VAR_FRAME_KEYS = None
     build_render_var_frame_keys = None
     detect_ovrtx_version = None
     render_var_prim_paths_by_source = None
+    resolve_render_var_key = None
     uses_prim_path_render_vars = None
 
 
@@ -106,3 +110,20 @@ def test_installed_frame_keys_match_the_installed_version():
 def test_published_frame_keys_are_read_only():
     with pytest.raises(TypeError):
         RENDER_VAR_FRAME_KEYS["LdrColor"] = "mutated"  # type: ignore[index]
+
+
+def test_resolve_render_var_key_ignores_scope_on_ovrtx_04(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr(ovrtx_compat_module, "OVRTX_VERSION", None)
+    assert resolve_render_var_key("LdrColor", "RenderCamera_3") == "LdrColor"
+    assert resolve_render_var_key("LdrColor", None) == "LdrColor"
+
+
+def test_resolve_render_var_key_falls_back_to_the_unscoped_path_on_ovrtx_05(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr(ovrtx_compat_module, "OVRTX_VERSION", Version("0.5"))
+    assert resolve_render_var_key("LdrColor", None) == "/Render/Vars/LdrColor"
+
+
+def test_resolve_render_var_key_rewrites_the_scope_prefix_on_ovrtx_05(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr(ovrtx_compat_module, "OVRTX_VERSION", Version("0.5"))
+    assert resolve_render_var_key("LdrColor", "RenderCamera_3") == "/RenderCamera_3/Vars/LdrColor"
+    assert resolve_render_var_key("DiffuseAlbedoSD", "RenderCamera_3") == "/RenderCamera_3/Vars/albedo"