Open XxsStrongest opened 6 months ago
python tasks/run.py --config=egs/datasets/woman/lm3d_radnerf_sr.yaml --exp_name=woman/may_head --reset
nerf_sr.yaml --exp_name=woman/may_head --reset | set_hparams Unknow hparams: [] | Hparams chains: ['egs/egs_bases/radnerf/base.yaml', 'egs/egs_bases/radnerf/lm3d_radnerf.yaml', 'egs/datasets/woman/lm3d_radnerf.yaml', 'egs/datasets/woman/lm3d_radnerf_sr.yaml'] | Hparams: { "accumulate_grad_batches": 1, "add_eye_blink_cond": true, "ambient_coord_dim": 3, "ambient_loss_mode": "mae", "amp": true, "base_config": [ "./lm3d_radnerf.yaml" ], "binary_data_dir": "data/binary/videos", "bound": 1, "camera_offset": [ 0, 0, 0 ], "camera_scale": 4.0, "clip_grad_norm": 0.0, "clip_grad_value": 0, "cond_dropout_rate": 0.0, "cond_out_dim": 64, "cond_type": "idexp_lm3d_normalized", "cond_win_size": 1, "cuda_ray": true, "debug": false, "density_thresh": 10, "density_thresh_torso": 0.01, "desired_resolution": 2048, "dt_gamma": 0.00390625, "eval_max_batches": 100, "exp_name": "woman/may_head", "eye_blink_dim": 2, "far": 0.9, "finetune_lips": true, "finetune_lips_start_iter": 200000, "geo_feat_dim": 128, "grid_interpolation_type": "linear", "grid_size": 128, "grid_type": "tiledgrid", "gui_fovy": 21.24, "gui_h": 512, "gui_max_spp": 1, "gui_radius": 3.35, "gui_w": 512, "hidden_dim_ambient": 128, "hidden_dim_color": 128, "hidden_dim_sigma": 128, "individual_embedding_dim": 4, "individual_embedding_num": 13000, "infer": false, "infer_audio_source_name": "", "infer_bg_img_fname": "", "infer_c2w_name": "", "infer_cond_name": "", "infer_lm3d_clamp_std": 1.5, "infer_lm3d_lle_percent": 0.25, "infer_lm3d_smooth_sigma": 0.0, "infer_out_video_name": "", "infer_scale_factor": 1.0, "infer_smo_std": 0.0, "infer_smooth_camera_path": true, "infer_smooth_camera_path_kernel_size": 7, "init_method": "tcp", "lambda_ambient": null, "lambda_dual_fm": 0.0, "lambda_lap_ambient_loss": 0.0, "lambda_lpips_loss": 0.001, "lambda_weights_entropy": 0.0001, "load_ckpt": "", "load_imgs_to_memory": false, "log2_hashmap_size": 16, "lpips_mode": "vgg19_v2", "lpips_start_iters": 200000, "lr": 0.0005, "lr_lambda_ambient": 0.01, "max_ray_batch": 4096, "max_steps": 16, "max_updates": 250000, "min_near": 0.05, "n_rays": 65536, "near": 0.3, "nerf_keypoint_mode": "lm68", "not_save_modules": [ "criterion_lpips", "dual_disc" ], "num_ckpt_keep": 1, "num_layers_ambient": 3, "num_layers_color": 2, "num_layers_sigma": 3, "num_sanity_val_steps": 2, "num_steps": 16, "num_valid_plots": 5, "optimizer_adam_beta1": 0.9, "optimizer_adam_beta2": 0.999, "polygon_face_mask": true, "print_nan_grads": false, "processed_data_dir": "data/processed/videos", "raw_data_dir": "data/raw/videos", "resume_from_checkpoint": 0, "save_best": true, "save_codes": [ "tasks", "modules", "egs" ], "save_gt": true, "scheduler": "exponential", "seed": 9999, "smo_win_size": 3, "smooth_lips": false, "sr_start_iters": 0, "start_rank": 0, "target_ambient_loss": 1e-08, "task_cls": "tasks.radnerfs.radnerf_sr.RADNeRFTask", "tb_log_interval": 100, "torso_head_aware": false, "torso_individual_embedding_dim": 8, "torso_shrink": 0.8, "update_extra_interval": 16, "upsample_steps": 0, "use_window_cond": true, "val_check_interval": 2000, "valid_infer_interval": 10000, "valid_monitor_key": "val_loss", "valid_monitor_mode": "min", "validate": false, "video_id": "woman", "warmup_updates": 0, "weight_decay": 0, "with_att": true, "with_sr": true, "work_dir": "checkpoints/woman/may_head", "world_size": -1, "zero_dummy": true } 05/29 03:37:08 PM GPU available: True, GPU used: [0], world_size: 1, multi-machine training: False /data/workpace/GeneFacePlusPlus/tasks/radnerfs/dataset_utils.py:266: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requiresgrad(True), rather than torch.tensor(sourceTensor). self.lm68s = torch.tensor(self.lm2ds[:, index_lm68_from_lm478, :]) val: Smooth head trajectory (rotation and translation) with a window size of 7 | Copied codes to checkpoints/woman/may_head/codes/20240529153717. | cond_prenet Trainable Parameters: 0.050M | blink_embedding Trainable Parameters: 0.000M | blink_encoder Trainable Parameters: 0.001M | cond_att_net Trainable Parameters: 0.004M | position_embedder Trainable Parameters: 1.807M | ambient_net Trainable Parameters: 0.029M | ambient_embedder Trainable Parameters: 1.807M | sigma_net Trainable Parameters: 0.041M | direction_embedder Trainable Parameters: 0.000M | color_net Trainable Parameters: 0.019M | dropout Trainable Parameters: 0.000M | sr_net Trainable Parameters: 0.271M Sanity Val: 0%| | 0/2 [00:00<?, ?step/s]/data/workpace/GeneFacePlusPlus/tasks/radnerfs/dataset_utils.py:427: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requiresgrad(True), rather than torch.tensor(sourceTensor). sample['lm68'] = torch.tensor(self.lm68s[idx].reshape([68*2]))
训练时一直卡在这个地方不动，请问有大佬知道问题原因吗。
去安装一下rysnc试试
raymondren1982
commented
5 months ago raymondren1982
commented
5 months ago
nerf_sr.yaml --exp_name=woman/may_head --reset | set_hparams Unknow hparams: [] | Hparams chains: ['egs/egs_bases/radnerf/base.yaml', 'egs/egs_bases/radnerf/lm3d_radnerf.yaml', 'egs/datasets/woman/lm3d_radnerf.yaml', 'egs/datasets/woman/lm3d_radnerf_sr.yaml'] | Hparams: { "accumulate_grad_batches": 1, "add_eye_blink_cond": true, "ambient_coord_dim": 3, "ambient_loss_mode": "mae", "amp": true, "base_config": [ "./lm3d_radnerf.yaml" ], "binary_data_dir": "data/binary/videos", "bound": 1, "camera_offset": [ 0, 0, 0 ], "camera_scale": 4.0, "clip_grad_norm": 0.0, "clip_grad_value": 0, "cond_dropout_rate": 0.0, "cond_out_dim": 64, "cond_type": "idexp_lm3d_normalized", "cond_win_size": 1, "cuda_ray": true, "debug": false, "density_thresh": 10, "density_thresh_torso": 0.01, "desired_resolution": 2048, "dt_gamma": 0.00390625, "eval_max_batches": 100, "exp_name": "woman/may_head", "eye_blink_dim": 2, "far": 0.9, "finetune_lips": true, "finetune_lips_start_iter": 200000, "geo_feat_dim": 128, "grid_interpolation_type": "linear", "grid_size": 128, "grid_type": "tiledgrid", "gui_fovy": 21.24, "gui_h": 512, "gui_max_spp": 1, "gui_radius": 3.35, "gui_w": 512, "hidden_dim_ambient": 128, "hidden_dim_color": 128, "hidden_dim_sigma": 128, "individual_embedding_dim": 4, "individual_embedding_num": 13000, "infer": false, "infer_audio_source_name": "", "infer_bg_img_fname": "", "infer_c2w_name": "", "infer_cond_name": "", "infer_lm3d_clamp_std": 1.5, "infer_lm3d_lle_percent": 0.25, "infer_lm3d_smooth_sigma": 0.0, "infer_out_video_name": "", "infer_scale_factor": 1.0, "infer_smo_std": 0.0, "infer_smooth_camera_path": true, "infer_smooth_camera_path_kernel_size": 7, "init_method": "tcp", "lambda_ambient": null, "lambda_dual_fm": 0.0, "lambda_lap_ambient_loss": 0.0, "lambda_lpips_loss": 0.001, "lambda_weights_entropy": 0.0001, "load_ckpt": "", "load_imgs_to_memory": false, "log2_hashmap_size": 16, "lpips_mode": "vgg19_v2", "lpips_start_iters": 200000, "lr": 0.0005, "lr_lambda_ambient": 0.01, "max_ray_batch": 4096, "max_steps": 16, "max_updates": 250000, "min_near": 0.05, "n_rays": 65536, "near": 0.3, "nerf_keypoint_mode": "lm68", "not_save_modules": [ "criterion_lpips", "dual_disc" ], "num_ckpt_keep": 1, "num_layers_ambient": 3, "num_layers_color": 2, "num_layers_sigma": 3, "num_sanity_val_steps": 2, "num_steps": 16, "num_valid_plots": 5, "optimizer_adam_beta1": 0.9, "optimizer_adam_beta2": 0.999, "polygon_face_mask": true, "print_nan_grads": false, "processed_data_dir": "data/processed/videos", "raw_data_dir": "data/raw/videos", "resume_from_checkpoint": 0, "save_best": true, "save_codes": [ "tasks", "modules", "egs" ], "save_gt": true, "scheduler": "exponential", "seed": 9999, "smo_win_size": 3, "smooth_lips": false, "sr_start_iters": 0, "start_rank": 0, "target_ambient_loss": 1e-08, "task_cls": "tasks.radnerfs.radnerf_sr.RADNeRFTask", "tb_log_interval": 100, "torso_head_aware": false, "torso_individual_embedding_dim": 8, "torso_shrink": 0.8, "update_extra_interval": 16, "upsample_steps": 0, "use_window_cond": true, "val_check_interval": 2000, "valid_infer_interval": 10000, "valid_monitor_key": "val_loss", "valid_monitor_mode": "min", "validate": false, "video_id": "woman", "warmup_updates": 0, "weight_decay": 0, "with_att": true, "with_sr": true, "work_dir": "checkpoints/woman/may_head", "world_size": -1, "zero_dummy": true } 05/29 03:37:08 PM GPU available: True, GPU used: [0], world_size: 1, multi-machine training: False /data/workpace/GeneFacePlusPlus/tasks/radnerfs/dataset_utils.py:266: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requiresgrad(True), rather than torch.tensor(sourceTensor). self.lm68s = torch.tensor(self.lm2ds[:, index_lm68_from_lm478, :]) val: Smooth head trajectory (rotation and translation) with a window size of 7 | Copied codes to checkpoints/woman/may_head/codes/20240529153717. | cond_prenet Trainable Parameters: 0.050M | blink_embedding Trainable Parameters: 0.000M | blink_encoder Trainable Parameters: 0.001M | cond_att_net Trainable Parameters: 0.004M | position_embedder Trainable Parameters: 1.807M | ambient_net Trainable Parameters: 0.029M | ambient_embedder Trainable Parameters: 1.807M | sigma_net Trainable Parameters: 0.041M | direction_embedder Trainable Parameters: 0.000M | color_net Trainable Parameters: 0.019M | dropout Trainable Parameters: 0.000M | sr_net Trainable Parameters: 0.271M Sanity Val: 0%| | 0/2 [00:00<?, ?step/s]/data/workpace/GeneFacePlusPlus/tasks/radnerfs/dataset_utils.py:427: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requiresgrad(True), rather than torch.tensor(sourceTensor). sample['lm68'] = torch.tensor(self.lm68s[idx].reshape([68*2]))
训练时一直卡在这个地方不动，请问有大佬知道问题原因吗。