Issues with using pytorch_kinematics

[sean1295]

When you do pd_ee_pose control which necessitates the usage of pytorch_kinematics, it throws an error when the xml file contains capsule mesh. Here is the minimal reproducible code.

import gymnasium as gym import mani_skill.envs import torch num_envs = 2 env = gym.make("PushT-v1", control_mode='pd_ee_delta_pose', num_envs=num_envs)

---

Exception Traceback (most recent call last) Cell In[5], line 5 3 import torch 4 num_envs = 2 # you can go up to 4096 on better GPUs ----> 5 env = gym.make("PushT-v1", control_mode='pd_ee_delta_pose', num_envs=num_envs)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/gymnasium/envs/registration.py:802, in make(id, max_episode_steps, autoreset, apply_api_compatibility, disable_env_checker, kwargs) 799 render_mode = None 801 try: --> 802 env = env_creator(env_spec_kwargs) 803 except TypeError as e: 804 if ( 805 str(e).find("got an unexpected keyword argument 'render_mode'") >= 0 806 and apply_human_rendering 807 ):

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/utils/registration.py:167, in make(env_id, kwargs) 165 raise KeyError("Env {} not found in registry".format(env_id)) 166 env_spec = REGISTERED_ENVS[env_id] --> 167 env = env_spec.make(kwargs) 168 return env

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/utils/registration.py:78, in EnvSpec.make(self, kwargs) 76 print("Exiting as assets are not found or downloaded") 77 exit() ---> 78 return self.cls(_kwargs)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/envs/tasks/tabletop/push_t.py:125, in PushTEnv.init(self, robot_uids, robot_init_qpos_noise, *args, kwargs) 123 def init(self, *args, robot_uids="panda_stick", robot_init_qpos_noise=0.02,*kwargs): 124 self.robot_init_qpos_noise = robot_init_qpos_noise --> 125 super().init(args, robot_uids=robot_uids, kwargs)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/envs/sapien_env.py:318, in BaseEnv.init(self, num_envs, obs_mode, reward_mode, control_mode, render_mode, shader_dir, enable_shadow, sensor_configs, human_render_camera_configs, robot_uids, sim_cfg, reconfiguration_freq, sim_backend, render_backend, parallel_in_single_scene) 314 self._set_main_rng(2022) 315 self._elapsed_steps = ( 316 torch.zeros(self.numenvs, device=self.device, dtype=torch.int32) 317 ) --> 318 obs, = self.reset(seed=2022, options=dict(reconfigure=True)) 319 self._init_raw_obs = common.to_cpu_tensor(obs) 320 """the raw observation returned by the env.reset (a cpu torch tensor/dict of tensors). Useful for future observation wrappers to use to auto generate observation spaces"""

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/envs/sapien_env.py:728, in BaseEnv.reset(self, seed, options) 726 with torch.random.fork_rng(): 727 torch.manual_seed(seed=self._episode_seed) --> 728 self._reconfigure(options) 729 self._after_reconfigure(options) 731 # TODO (stao): Reconfiguration when there is partial reset might not make sense and certainly broken here now. 732 # Solution to resolve that would be to ensure tasks that do reconfigure more than once are single-env only / cpu sim only 733 # or disable partial reset features explicitly for tasks that have a reconfiguration frequency

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/envs/sapien_env.py:587, in BaseEnv._reconfigure(self, options) 585 # load everything into the scene first before initializing anything 586 self._setup_scene() --> 587 self._load_agent(options) 588 self._load_scene(options) 589 self._load_lighting(options)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/envs/sapien_env.py:370, in BaseEnv._load_agent(self, options) 366 raise RuntimeError( 367 f"Agent {robot_uid} not found in the dict of registered agents. If the id is not a typo then make sure to apply the @register_agent() decorator." 368 ) 369 agent_cls = REGISTERED_AGENTS[robot_uid].agent_cls --> 370 agent: BaseAgent = agent_cls( 371 self.scene, 372 self._control_freq, 373 self._control_mode, 374 agent_idx=i if len(robot_uids) > 1 else None, 375 ) 376 agents.append(agent) 377 if len(agents) == 1:

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/agents/base_agent.py:101, in BaseAgent.init(self, scene, control_freq, control_mode, agent_idx) 99 # The control mode after reset for consistency 100 self._default_control_mode = control_mode --> 101 self.set_control_mode() 103 self._after_init()

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/agents/base_agent.py:220, in BaseAgent.set_control_mode(self, control_mode) 218 if "balance_passive_force" in config: 219 balance_passive_force = config.pop("balance_passive_force") --> 220 self.controllers[control_mode] = CombinedController( 221 config, 222 self.robot, 223 self._control_freq, 224 scene=self.scene, 225 ) 226 else: 227 self.controllers[control_mode] = config.controller_cls( 228 config, self.robot, self._control_freq, scene=self.scene 229 )

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/agents/controllers/base_controller.py:202, in DictController.init(self, configs, articulation, control_freq, sim_freq, scene) 200 self.controllers: Dict[str, BaseController] = dict() 201 for uid, config in configs.items(): --> 202 self.controllers[uid] = config.controller_cls( 203 config, articulation, control_freq, sim_freq=sim_freq, scene=scene 204 ) 205 self._initialize_action_space() 206 self._initialize_joints()

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/agents/controllers/base_controller.py:64, in BaseController.init(self, config, articulation, control_freq, sim_freq, scene) 61 self._sim_steps = sim_freq // control_freq 62 """Number of simulation steps per control step""" ---> 64 self._initialize_joints() 65 self._initialize_action_space() 66 # NOTE(jigu): It is intended not to be a required field in config.

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/agents/controllers/pd_ee_pose.py:45, in PDEEPosController._initialize_joints(self) 43 if self.device.type == "cuda": 44 self._check_gpu_sim_works() ---> 45 self.kinematics = Kinematics( 46 self.config.urdf_path, 47 self.config.ee_link, 48 self.articulation, 49 self.active_joint_indices, 50 ) 52 self.ee_link = self.kinematics.end_link

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/agents/controllers/utils/kinematics.py:69, in Kinematics.init(self, urdf_path, end_link_name, articulation, active_joint_indices) 67 if self.device.type == "cuda": 68 self.use_gpu_ik = True ---> 69 self._setup_gpu() 70 else: 71 self.use_gpu_ik = False

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/mani_skill/agents/controllers/utils/kinematics.py:101, in Kinematics._setup_gpu(self) 98 yield (err, out) 100 with suppress_stdout_stderr(): --> 101 self.pk_chain = pk.build_serial_chain_from_urdf( 102 urdf_str, 103 end_link_name=self.end_link.name, 104 ).to(device=self.device) 106 self.qmask = torch.zeros( 107 len(self.active_ancestor_joints), dtype=bool, device=self.device 108 ) 109 self.qmask[self.controlled_joints_idx_in_qmask] = 1

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf.py:135, in build_serial_chain_from_urdf(data, end_link_name, root_link_name) 117 def build_serial_chain_from_urdf(data, end_link_name, root_link_name=""): 118 """ 119 Build a SerialChain object from urdf data. 120 (...) 133 SerialChain object created from URDF. 134 """ --> 135 urdf_chain = build_chain_from_urdf(data) 136 return chain.SerialChain(urdf_chain, end_link_name, 137 "" if root_link_name == "" else root_link_name)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf.py:94, in build_chain_from_urdf(data) 63 def build_chain_from_urdf(data): 64 """ 65 Build a Chain object from URDF data. 66 (...) 92 93 """ ---> 94 robot = URDF.from_xml_string(data) 95 lmap = robot.link_map 96 joints = robot.joints

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:617, in Object.from_xml_string(cls, xml_string) 615 node = etree.fromstring(xml_string) 616 path = Path(cls.XML_REFL.tag, tree=etree.ElementTree(node)) --> 617 return cls.from_xml(node, path)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:611, in Object.from_xml(cls, node, path) 608 @classmethod 609 def from_xml(cls, node, path): 610 cur_type = get_type(cls) --> 611 return cur_type.from_xml(node, path)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:232, in ObjectType.from_xml(self, node, path) 230 def from_xml(self, node, path): 231 obj = self.type() --> 232 obj.read_xml(node, path) 233 return obj

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:599, in Object.read_xml(self, node, path) 598 def read_xml(self, node, path): --> 599 self.XML_REFL.set_from_xml(self, node, path) 600 self.post_read_xml() 601 try:

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:514, in Reflection.set_from_xml(self, obj, node, path, info) 512 element_path = get_element_path(element) 513 if element.is_aggregate: --> 514 element.add_from_xml(obj, child, element_path) 515 else: 516 if tag in unset_scalars:

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:391, in AggregateElement.add_from_xml(self, obj, node, path) 390 def add_from_xml(self, obj, node, path): --> 391 value = self.value_type.from_xml(node, path) 392 obj.add_aggregate(self.xml_var, value)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:232, in ObjectType.from_xml(self, node, path) 230 def from_xml(self, node, path): 231 obj = self.type() --> 232 obj.read_xml(node, path) 233 return obj

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:599, in Object.read_xml(self, node, path) 598 def read_xml(self, node, path): --> 599 self.XML_REFL.set_from_xml(self, node, path) 600 self.post_read_xml() 601 try:

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:514, in Reflection.set_from_xml(self, obj, node, path, info) 512 element_path = get_element_path(element) 513 if element.is_aggregate: --> 514 element.add_from_xml(obj, child, element_path) 515 else: 516 if tag in unset_scalars:

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:391, in AggregateElement.add_from_xml(self, obj, node, path) 390 def add_from_xml(self, obj, node, path): --> 391 value = self.value_type.from_xml(node, path) 392 obj.add_aggregate(self.xml_var, value)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:232, in ObjectType.from_xml(self, node, path) 230 def from_xml(self, node, path): 231 obj = self.type() --> 232 obj.read_xml(node, path) 233 return obj

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:599, in Object.read_xml(self, node, path) 598 def read_xml(self, node, path): --> 599 self.XML_REFL.set_from_xml(self, node, path) 600 self.post_read_xml() 601 try:

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:517, in Reflection.set_from_xml(self, obj, node, path, info) 515 else: 516 if tag in unset_scalars: --> 517 element.set_from_xml(obj, child, element_path) 518 unset_scalars.remove(tag) 519 else:

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:361, in Element.set_from_xml(self, obj, node, path) 360 def set_from_xml(self, obj, node, path): --> 361 value = self.value_type.from_xml(node, path) 362 setattr(obj, self.var, value)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/urdf.py:139, in GeometricType.from_xml(self, node, path) 137 children = xml_children(node) 138 assert len(children) == 1, 'One element only for geometric' --> 139 return self.factory.from_xml(children[0], path=path)

File ~/anaconda3/envs/maniskill/lib/python3.12/site-packages/pytorch_kinematics/urdf_parser_py/xml_reflection/core.py:251, in FactoryType.from_xml(self, node, path) 249 cur_type = self.typeMap.get(node.tag) 250 if cur_type is None: --> 251 raise Exception("Invalid {} tag: {}".format(self.name, node.tag)) 252 value_type = get_type(cur_type) 253 return value_type.from_xml(node, path)

Exception: Invalid geometric tag: capsule

[StoneT2000]

try pip uninstall pytorch_kinematics_ms and then reinstall it

[sean1295]

Thanks, it works!