How to export a robot navigation video based on a baseline

[2201957]

I have made changes to the [default.py as follows:

_C.VIDEO_OPTION = ["disk"] # options: "disk", "tensorboard" _C.VIDEO_DIR = "data/videos/debug"

but this does no help and there is something wrong happen

[MarSaKi]

Sorry, I didn't implement the video visualization pipeline of BEVBert. However, the visualization of ETPNav is well-implemented. You can take a look at this repo and fix those vis bugs, currently, I don't have time to implement BEVBert video visualization.

[2201957]

Both ETPnav & BEVbert are interesting jobs！

After debugging, I solved the problem by uncommenting /bevbert_ce/vlnce_baselines/common/environments.py in the function Step

` def step(self, action, vis_info, *args, **kwargs): act = action['act']

    if act == 4: # high to low
        if self.video_option:
            self.get_plan_frame(vis_info)

        # 1. back to front node
        if action['back_path'] is None:
            self.teleport(action['front_pos'])
        else:
            self.multi_step_control(action['back_path'], action['tryout'], vis_info)
        agent_state = self._env.sim.get_agent_state()
        observations = self.get_observation_at(agent_state.position, agent_state.rotation)

        # 2. forward to ghost node
        self.single_step_control(action['ghost_pos'], action['tryout'], vis_info)
        agent_state = self._env.sim.get_agent_state()
        observations = self.get_observation_at(agent_state.position, agent_state.rotation)

    elif act == 0:   # stop
        if self.video_option:
            self.get_plan_frame(vis_info)

        # 1. back to stop node
        if action['back_path'] is None:
            self.teleport(action['stop_pos'])
        else:
            self.multi_step_control(action['back_path'], action['tryout'], vis_info)

        # 2. stop
        observations = self._env.step(act)
        if self.video_option:
            info = self.get_info(observations)
            self.video_frames.append(
                navigator_video_frame(
                    observations,
                    info,
                    vis_info,
                )
            )
            self.get_plan_frame(vis_info)

    else:
        raise NotImplementedError                

    reward = self.get_reward(observations)
    done = self.get_done(observations)
    info = self.get_info(observations)

    if self.video_option and done:
        # if 0 < info["spl"] <= 0.6:
        # generate_video(
        #     video_option=self.video_option,
        #     video_dir=self.video_dir,
        #     images=self.video_frames,
        #     episode_id=self._env.current_episode.episode_id,
        #     scene_id=self._env.current_episode.scene_id.split('/')[-1].split('.')[-2],
        #     checkpoint_idx=0,
        #     metrics={"SPL": round(info["spl"], 3)},
        #     tb_writer=None,
        #     fps=8,
        # )
        # # for pano visualization
        # metrics={
        #             # "sr": round(info["success"], 3), 
        #             "spl": round(info["spl"], 3),
        #             # "ndtw": round(info["ndtw"], 3),
        #             # "sdtw": round(info["sdtw"], 3),
        #         }
        # metric_strs = []
        # for k, v in metrics.items():
        #     metric_strs.append(f"{k}{v:.2f}")
        # episode_id=self._env.current_episode.episode_id
        # scene_id=self._env.current_episode.scene_id.split('/')[-1].split('.')[-2]
        # tmp_name = f"{scene_id}-{episode_id}-" + "-".join(metric_strs)
        # tmp_name = tmp_name.replace(" ", "_").replace("\n", "_") + ".png"
        # tmp_fn = os.path.join(self.video_dir, tmp_name)
        # tmp = np.concatenate(self.plan_frames, axis=0)
        # cv2.imwrite(tmp_fn, tmp)
        self.plan_frames = []

    return observations, reward, done, info

`

Thank you very much for your work in the VLN field, and I hope that open-source contributors like you wil receive the greatest help in the coming future !!!!!!!