Visualizing Worlds

This tutorial explains you how to visualize a world. There are two recommended ways of doing it. One light weight way through RVIZ2 and a more heavy weight way through simulation with multiverse. Let’s load a world first to get started.

import logging
import os

from importlib.resources import files
from pathlib import Path

from semantic_digital_twin.adapters.urdf import URDFParser 

logging.disable(logging.CRITICAL)
apartment = os.path.join(Path(files("semantic_digital_twin")).parent.parent, "resources", "urdf", "apartment.urdf")
world = URDFParser.from_file(apartment).parse()

Unknown tag "material" in /robot[@name='apartment']/link[@name='coffe_machine']/collision[1]

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[1], line 11
      7 from semantic_digital_twin.adapters.urdf import URDFParser
      8 
      9 logging.disable(logging.CRITICAL)
     10 apartment = os.path.join(Path(files("semantic_digital_twin")).parent.parent, "resources", "urdf", "apartment.urdf")
---> 11 world = URDFParser.from_file(apartment).parse()

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/adapters/urdf.py:194, in URDFParser.parse(self)
    192 world = World()
    193 world.name = self.prefix
--> 194 with world.modify_world():
    195     world.add_kinematic_structure_entity(root)
    196     main_joints = []

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/world.py:314, in WorldModelUpdateContextManager.__exit__(self, exc_type, exc_val, exc_tb)
    312 try:
    313     if exc_type is None:
--> 314         self.world._notify_model_change(
    315             publish_changes=self.publish_changes
    316         )
    317         run_pending_publications = True
    318 finally:

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/world.py:2129, in World._notify_model_change(self, publish_changes, **kwargs)
   2123 def _notify_model_change(self, publish_changes: bool = True, **kwargs) -> None:
   2124     """
   2125     Notifies the system of a model change and updates the necessary states, caches,
   2126     and forward kinematics expressions while also triggering registered callbacks
   2127     for model changes.
   2128     """
-> 2129     self._model_manager.update_model_version_and_notify_callbacks(
   2130         publish_changes=publish_changes, **kwargs
   2131     )
   2132     self.notify_state_change(
   2133         publish_changes=publish_changes, force_republish=True, **kwargs
   2134     )
   2136     for callback in list(self.state.state_change_callbacks):

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/world.py:523, in WorldModelManager.update_model_version_and_notify_callbacks(self, **kwargs)
    521 self.version += 1
    522 for callback in list(self.model_change_callbacks):
--> 523     callback.notify_model_change(**kwargs)

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/callbacks/callback.py:171, in ModelChangeCallback.notify_model_change(self, **kwargs)
    169 def notify_model_change(self, **kwargs):
    170     if not self._is_paused:
--> 171         self.on_model_change(**kwargs)

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/collision_checking/collision_detector.py:142, in CollisionDetectorModelUpdater.on_model_change(self, **kwargs)
    140 if self._world.is_empty():
    141     return
--> 142 self.collision_detector.sync_world_model()
    143 self.compile_collision_fks()

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/collision_checking/pybullet_collision_detector.py:390, in BulletCollisionDetector.sync_world_model(self)
    388     return
    389 for body in self._world.bodies_with_collision:
--> 390     self.add_body(body)
    391 self._ordered_bullet_objects = list(self.body_to_bullet_object.values())

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/collision_checking/pybullet_collision_detector.py:405, in BulletCollisionDetector.add_body(self, body)
    404 def add_body(self, body: Body):
--> 405     o = create_shape_from_body(body=body, mesh_decomposer=self.mesh_decomposer)
    406     self.kineverse_world.add_collision_object(o)
    407     self.body_to_bullet_object[body] = o

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/collision_checking/pybullet_collision_detector.py:226, in create_shape_from_body(body, mesh_decomposer)
    224 shapes = []
    225 for collision_id, geometry in enumerate(body.collision):
--> 226     shape = create_shape_from_geometry(
    227         geometry=geometry, mesh_decomposer=mesh_decomposer
    228     )
    229     link_T_geometry = bullet.Transform.from_np(geometry.origin.to_np())
    230     shapes.append((link_T_geometry, shape))

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/collision_checking/pybullet_collision_detector.py:199, in create_shape_from_geometry(geometry, mesh_decomposer)
    194     shape = create_cylinder_shape(
    195         diameter=geometry.width, height=geometry.height
    196     )
    198 case Mesh():
--> 199     shape = load_convex_mesh_shape(
    200         mesh=geometry,
    201         single_shape=False,
    202         scale=geometry.scale,
    203         mesh_decomposer=mesh_decomposer,
    204     )
    206 case _:
    207     raise NotImplementedError()

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/collision_checking/pybullet_collision_detector.py:266, in load_convex_mesh_shape(mesh, single_shape, scale, mesh_decomposer)
    256 """
    257 Loads a convex mesh shape from a mesh.
    258 
   (...)    263 :return: the bullet convex shape.
    264 """
    265 if not mesh.mesh.is_convex and mesh_decomposer is not None:
--> 266     obj_pkg_filename = convert_to_decomposed_obj_and_save_in_tmp(
    267         mesh=mesh, mesh_decomposer=mesh_decomposer
    268     )
    269 else:
    270     obj_pkg_filename = str(mesh.local_file)

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/collision_checking/pybullet_collision_detector.py:313, in convert_to_decomposed_obj_and_save_in_tmp(mesh, mesh_decomposer, cache_dir, log_path)
    311 if not trimesh_obj.is_convex and mesh_decomposer is not None:
    312     with suppress_stdout_stderr():
--> 313         mesh_decomposer.apply_to_mesh_and_save(mesh, obj_file_name)
    314     logging.info(f'Saved convex decomposition to "{obj_file_name}".')
    315 else:

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/pipeline/mesh_decomposition/vhacd.py:132, in VHACDMeshDecomposer.apply_to_mesh_and_save(self, mesh, output_path)
    131 def apply_to_mesh_and_save(self, mesh: Mesh, output_path: str) -> str:
--> 132     parts = self.apply_to_mesh(mesh)
    133     trimesh.Scene([p.mesh for p in parts]).export(output_path, file_type="obj")
    134     return output_path

File /__w/cognitive_robot_abstract_machine/cognitive_robot_abstract_machine/semantic_digital_twin/src/semantic_digital_twin/pipeline/mesh_decomposition/vhacd.py:113, in VHACDMeshDecomposer.apply_to_mesh(self, mesh)
    112 def apply_to_mesh(self, mesh: Mesh) -> List[Mesh]:
--> 113     decomposed = mesh.mesh.convex_decomposition(
    114         maxConvexHulls=self.max_convex_hulls,
    115         resolution=self.resolution,
    116         minimumVolumePercentErrorAllowed=self.minimum_volume_percent_error_allowed,
    117         maxRecursionDepth=self.max_recursion_depth,
    118         shrinkWrap=self.shrink_wrap,
    119         fillMode=self.fill_mode.value,
    120         maxNumVerticesPerCH=self.max_vertices_per_convex_hull,
    121         asyncACD=self.asynchronous,
    122         minEdgeLength=self.min_edge_length,
    123         findBestPlane=self.find_best_plane,
    124     )
    125     new_geometry = [
    126         Mesh.from_trimesh(mesh=decomposed_part, origin=mesh.origin)
    127         for decomposed_part in decomposed
    128     ]
    129     return new_geometry

File /opt/ros/cram-env/lib/python3.12/site-packages/trimesh/base.py:3048, in Trimesh.convex_decomposition(self, **kwargs)
   3035 def convex_decomposition(self, **kwargs) -> list["Trimesh"]:
   3036     """
   3037     Compute an approximate convex decomposition of a mesh
   3038     using `pip install pyVHACD`.
   (...)   3044     **kwargs : VHACD keyword arguments
   3045     """
   3046     return [
   3047         Trimesh(**kwargs)
-> 3048         for kwargs in decomposition.convex_decomposition(self, **kwargs)
   3049     ]

File /opt/ros/cram-env/lib/python3.12/site-packages/trimesh/decomposition.py:48, in convex_decomposition(mesh, **kwargs)
     37 # the faces are triangulated in a (len(face), ...vertex-index)
     38 # for vtkPolyData
     39 # i.e. so if shaped to four columns the first column is all 3
     40 faces = (
     41     np.column_stack((np.ones(len(mesh.faces), dtype=np.int64) * 3, mesh.faces))
     42     .ravel()
     43     .astype(np.uint32)
     44 )
     46 return [
     47     {"vertices": v, "faces": f}
---> 48     for v, f in compute_vhacd(mesh.vertices, faces, **kwargs)
     49 ]

KeyboardInterrupt: 

For the RVIZ2 way, ROS2 and the TFPublisher is needed. A caveat of this approach is that you have to manage the lifecycle of a ROS2 node yourself. We recommend to put the spinning into sperate threads and just shutdown the thread when exiting the system.

from semantic_digital_twin.adapters.ros.tf_publisher import TFPublisher
from semantic_digital_twin.adapters.ros.visualization.viz_marker import VizMarkerPublisher
import threading
import rclpy
rclpy.init()

node = rclpy.create_node("semantic_digital_twin")
thread = threading.Thread(target=rclpy.spin, args=(node,), daemon=True)
thread.start()

tf_publisher = TFPublisher(_world=world, node=node)
viz = VizMarkerPublisher(_world=world, node=node)

When you want to stop visualizing, you have to stop the visualizer and afterwards clean up ROS2.

node.destroy_node()
rclpy.shutdown()

The world can also be visualized directly through a running simulation. Although this approach is computationally heavier, it provides the important advantage of enabling interaction with the environment. In addition to visualization, the physics engine can be used to simulate dynamics, contacts, and collisions. Further details are provided in the physics simulators section.

If you have followed the guide until here, you have probably noticed that we have used the RayTracer to visualize the world a few times. This is a convenient way of visualizing a world inside a notebook, like in these guides, but it is not recommended for normal usage.