[Question] Question on Setting up the camera on robot head, but the pose is not as what I have setting

[cidxb]

Question

I am using isaac Sim 4.2.0.2 and omni-isaac-lab 0.24.19

I was using the following code for setting up a camera on my robot's head

@configclass
class TableSceneCfg(InteractiveSceneCfg):
    """Scene configuration with robot, table, and objects."""
    camera = CameraCfg(
        prim_path="{ENV_REGEX_NS}/Robot/head_Link/front_cam",
        update_period=0.1,
        height=720,
        width=1280,
        data_types=["rgb", "distance_to_image_plane"],
        spawn=sim_utils.PinholeCameraCfg(
            focal_length=24.0,
            focus_distance=400.0,
            horizontal_aperture=20.955,
            clipping_range=(0.1, 1.0e5),
        ),
        offset=CameraCfg.OffsetCfg(
            rot=(0.12279, 0.69636, -0.12279, 0.69636),  
            pos=(-0.1, 0.0, 0.0),
            convention="ros",
        ),
    )

The rotation value acquired first by using the slider inside the simulation , and I adjust the value slider to suitable pose and marked it down. But after I filled the value up on the cfg. When I got inside the sim ,the value still not as what as I set.

Should I do a some what operation similar to robot.write_joint_state_to_sim(joint_pos, joint_vel)?

Since I found out that, for robot , if I want the initial joint set as the value in ArticulationCfg, I must do a such operation first to reset it before the simulation (B.T.W, I am also not sure, if this the correct usage, I do learn it from examples which they use it inside the simulation loop as reset procedure, but not before it. But if I don't reset it first, the simulation will start from 0 point and slowly move to the initial pose, which is not desired).

Wish someone has solution or experience on it.

Thanks ahead for kindly help from the community. Hat tip to you!

[RandomOakForest]

Thank you for posting this. Maybe you forgot to do a sim.reset() as explained in this tutorial.

[cidxb]

Thank you for posting this. Maybe you forgot to do a sim.reset() as explained in this tutorial.

Thank you for reply, but I did have the reset method on my script, here is my whole script :

def get_smooth_random_joint_positions(robot, time, amplitude=0.5, frequency=0.5):

    default_pos = robot.data.default_joint_pos.clone()
    num_joints = len(default_pos)
    device = default_pos.device  

    if not hasattr(robot, "phase_offsets"):

        robot.phase_offsets = (
            torch.rand(num_joints, device=device) * 2 * np.pi
        ) 

    joint_offsets = (
        amplitude
        * torch.sin(2 * np.pi * frequency * time + robot.phase_offsets.to(device))
    ).to(device)

    return default_pos + 0

@configclass
class TableSceneCfg(InteractiveSceneCfg):
    """Scene configuration with robot, table, and objects."""

    @configclass
    class RobotInitialState(AssetBaseCfg.InitialStateCfg):
        """Extended initial state configuration for the robot."""

        pos: tuple[float, float, float] = (0.0, 0.5, 0.0)
        rot: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0)
        lin_vel: tuple[float, float, float] = (0.0, 0.0, 0.0)
        ang_vel: tuple[float, float, float] = (0.0, 0.0, 0.0)

    # Ground plane
    ground = AssetBaseCfg(
        prim_path="/World/defaultGroundPlane", spawn=sim_utils.GroundPlaneCfg()
    )
    # lights
    dome_light = AssetBaseCfg(
        prim_path="/World/Light",
        spawn=sim_utils.DomeLightCfg(intensity=3000.0, color=(0.75, 0.75, 0.75)),
    )
    # Robot with adjusted origin
    robot: ArticulationCfg = TORA_CFG.replace(
        prim_path="{ENV_REGEX_NS}/Robot",
    )

    # # Table adjusted relative to robot
    # table = AssetBaseCfg(
    #     prim_path="{ENV_REGEX_NS}/table",
    #     spawn=sim_utils.UsdFileCfg(
    #         usd_path="../../USD_res/Assets/Table_01/rh_table_short.usd",
    #         scale=np.array([0.02, 0.02, 0.02]),
    #     ),
    #     init_state=AssetBaseCfg.InitialStateCfg(
    #         pos=(0.0, -0.9, 0.0)  
    #     ),
    # )
    table = AssetBaseCfg(
        prim_path="{ENV_REGEX_NS}/table",
        spawn=sim_utils.UsdFileCfg(
            usd_path="./table_white.usd",
            scale=np.array([0.1, 0.1, 0.1]),
        ),
        init_state=AssetBaseCfg.InitialStateCfg(
            pos=(0.0, -0.9, 0.0),  
            rot=(0.7071, 0.7071, 0.0, 0.0),
        ),
    )

    # Objects
    object1 = AssetBaseCfg(
        prim_path="{ENV_REGEX_NS}/object1",
        spawn=sim_utils.UsdFileCfg(
            usd_path="./ImageToStl.com_obj_000030.usd",
            scale=np.array([0.001, 0.001, 0.001]),
        ),
        init_state=AssetBaseCfg.InitialStateCfg(
            pos=(0.0, -0.7, 1.3),
            rot=(0.707, 0.707, 0.0, 0.0),  # 90度绕X轴旋转
        ),
    )
    ###Problematic
    # object2 = AssetBaseCfg(
    #     prim_path="{ENV_REGEX_NS}/object2",
    #     spawn=sim_utils.UsdFileCfg(
    #         usd_path="./ImageToStl.com_obj_000031.usd",
    #         scale=np.array([0.001, 0.001, 0.001]),
    #     ),
    #     init_state=AssetBaseCfg.InitialStateCfg(
    #         pos=(-0.5, -0.7, 1.3), rot=(0.707, 0.707, 0.0, 0.0)
    #     ),
    # )
    # Camera
    camera = CameraCfg(
        prim_path="{ENV_REGEX_NS}/Robot/head_Link/front_cam",
        update_period=0.1,
        height=720,
        width=1280,
        data_types=["rgb", "distance_to_image_plane"],
        spawn=sim_utils.PinholeCameraCfg(
            focal_length=24.0,
            focus_distance=400.0,
            horizontal_aperture=20.955,
            clipping_range=(0.1, 1.0e5),
        ),
        offset=CameraCfg.OffsetCfg(
            rot=(0.12279, 0.69636, -0.12279, 0.69636),  
            pos=(-0.1, 0.0, 0.0),
            convention="ros",
        ),
    )

def run_simulator(sim: sim_utils.SimulationContext, scene: InteractiveScene):
    """Run the simulator."""
    sim_dt = sim.get_physics_dt()
    sim_time = 0.0
    count = 0
    robot = scene["robot"]
    robot_state = robot.data.default_root_state.clone()
    robot_state[:, :3] += scene.env_origins
    robot.write_root_state_to_sim(robot_state)
    # joint state
    joint_pos, joint_vel = (
        robot.data.default_joint_pos.clone(),
        robot.data.default_joint_vel.clone(),
    )
    robot.write_joint_state_to_sim(joint_pos, joint_vel)
    # reset the internal state
    robot.reset()
    while simulation_app.is_running():
        # Generate smooth random positions
        targets = scene["robot"].data.default_joint_pos
        # Apply actions to robot
        scene["robot"].set_joint_position_target(targets)
        # Write data to sim
        scene.write_data_to_sim()
        # Step simulation
        sim.step()
        # Update time
        sim_time += sim_dt
        count += 1
        # Update scene
        scene.update(sim_dt)

def main():
    """Main function."""
    # Initialize simulation
    sim_cfg = sim_utils.SimulationCfg(dt=0.005, device=args_cli.device)
    sim = sim_utils.SimulationContext(sim_cfg)

    # Set camera view
    sim.set_camera_view(eye=[3.0, -3.0, 2.25], target=[0.0, 0.0, 0.0])

    # Create scene
    scene_cfg = TableSceneCfg(num_envs=args_cli.num_envs, env_spacing=5.0)
    scene = InteractiveScene(scene_cfg)

    # Reset simulation
    sim.reset()

    print("[INFO]: Setup complete...")
    # Run simulator
    run_simulator(sim, scene)

if __name__ == "__main__":
    main()
    simulation_app.close()

For example, by using the slider of orientation in isaac sim, I have acquired the quaternion are (0.12279, 0.69636, -0.12279, 0.69636), and after I filled in the numbers, and ran the script, I went back to check the value, it shows that the camera is on (0.69636,-0.12279,-0.69636,-0.12279),and the camera is upside down and inside out flipped. My best guess is the order problem, since they values are right but not the order and sign ? But I am not sure how to fix it ...