resume_checkpoint继续训练失败，已经将pretrain_weights设置为None。还是会报错

[smallhaozigithub]

aistudio上跑的paddlex。版本paddlex-2.1.0

model.train( 
    num_epochs=300,#训练轮数
    train_dataset=train_dataset, #训练数据
    eval_dataset=eval_dataset, #验证数据
    train_batch_size=128, #batchsize
    save_interval_epochs=5, #保存轮数
    log_interval_steps=20, #log轮数
    save_dir='output/ppyolo_resnet50',#保存目录 
    # pretrain_weights='COCO', #预训练权重
    resume_checkpoint='./output/ppyolo_resnet50/epoch_45/',#恢复训练
    pretrain_weights=None, #恢复训练
    learning_rate=0.000125, #学习率
    warmup_steps=500, #预热部署
    lr_decay_epochs=[150, 250],  # step学习率衰减, 
    use_vdl=True)

报错内容

AssertionError                            Traceback (most recent call last)
/tmp/ipykernel_11128/1974317225.py in <module>
     33     warmup_steps=500, #预热部署
     34     lr_decay_epochs=[150, 250],  # step学习率衰减,
---> 35     use_vdl=True)

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/detector.py in train(self, num_epochs, train_dataset, train_batch_size, eval_dataset, optimizer, save_interval_epochs, log_interval_steps, save_dir, pretrain_weights, learning_rate, warmup_steps, warmup_start_lr, lr_decay_epochs, lr_decay_gamma, metric, use_ema, early_stop, early_stop_patience, use_vdl, resume_checkpoint)
    332             early_stop=early_stop,
    333             early_stop_patience=early_stop_patience,
--> 334             use_vdl=use_vdl)
    335 
    336     def quant_aware_train(self,

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/base.py in train_loop(self, num_epochs, train_dataset, train_batch_size, eval_dataset, save_interval_epochs, log_interval_steps, save_dir, ema, early_stop, early_stop_patience, use_vdl)
    338                 loss = outputs['loss']
    339                 loss.backward()
--> 340                 self.optimizer.step()
    341                 self.optimizer.clear_grad()
    342                 lr = self.optimizer.get_lr()

<decorator-gen-338> in step(self)

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dygraph/base.py in __impl__(func, *args, **kwargs)
    294         def __impl__(func, *args, **kwargs):
    295             with _switch_tracer_mode_guard_(is_train=False):
--> 296                 return func(*args, **kwargs)
    297 
    298         return __impl__(func)

<decorator-gen-336> in step(self)

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/wrapped_decorator.py in __impl__(func, *args, **kwargs)
     23     def __impl__(func, *args, **kwargs):
     24         wrapped_func = decorator_func(func)
---> 25         return wrapped_func(*args, **kwargs)
     26 
     27     return __impl__

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/framework.py in __impl__(*args, **kwargs)
    227         assert in_dygraph_mode(
    228         ), "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." % func.__name__
--> 229         return func(*args, **kwargs)
    230 
    231     return __impl__

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/optimizer/optimizer.py in step(self)
   1134 
   1135             self._apply_optimize(
-> 1136                 loss=None, startup_program=None, params_grads=params_grads)
   1137 
   1138         else:

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/optimizer/optimizer.py in _apply_optimize(self, loss, startup_program, params_grads)
    889                     params_grads['params'] = self.append_regularization_ops(
    890                         params_grads['params'], self.regularization)
--> 891                 optimize_ops = self._create_optimization_pass(params_grads)
    892         else:
    893             program = loss.block.program

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/optimizer/optimizer.py in _create_optimization_pass(self, parameters_and_grads)
    675             self._create_accumulators(
    676                 target_block,
--> 677                 [p[0] for p in parameters_and_grads if not p[0].stop_gradient])
    678 
    679         else:

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/optimizer/momentum.py in _create_accumulators(self, block, parameters)
    263                     "Consider using multi_precision=True option of the Momentum optimizer."
    264                 )
--> 265             self._add_accumulator(self._velocity_acc_str, p)
    266 
    267     def _create_regularization_of_grad(self, param, grad, regularization=None):

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/optimizer/optimizer.py in _add_accumulator(self, name, param, dtype, fill_value, shape, type, device)
    590             if len(self._accumulators_holder) > 0:
    591                 assert var_name in self._accumulators_holder, \
--> 592                         "Optimizer set error, {} should in state dict".format( var_name )
    593                 var.set_value(self._accumulators_holder[var_name])
    594 

AssertionError: Optimizer set error, conv2d_0.w_0_velocity_0 should in state dict

[will-jl944]

具体使用的是什么模型以及backbone？

该恢复训练找不到优化器参数的问题已经在PaddleX 2.1.0中修复了，详见https://github.com/PaddlePaddle/PaddleX/pull/1197 。我们使用PaddleX 2.1.0，PPYOLO + ResNet50_vd_dcn没有复现出该问题。

可以使用pip show paddlex确认一下使用的PaddleX版本。

[smallhaozigithub]

呀。。。用的就是PaddleX 2.1.0，PPYOLO + ResNet50_vd_dcn 呀。我确认下看看。

[will-jl944]

也可以在import paddex之后print(paddlex.__version__)看一下。

[smallhaozigithub]

Name: paddlex
Version: 2.1.0
Summary: PaddlePaddle End-to-End Development Toolkit
Home-page: https://github.com/PaddlePaddle/PaddleX
Author: paddlex
Author-email: paddlex@baidu.com
License: Apache 2.0
Location: /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages
Requires: chardet, colorama, flask-cors, lap, motmetrics, opencv-python, openpyxl, paddleslim, pycocotools, pyyaml, scikit-learn, scipy, shapely, tqdm, visualdl
Required-by: 

[smallhaozigithub]

paddlex是2.1.0版本

[smallhaozigithub]

我在aistudio,和本地跑都是同样的。

[smallhaozigithub]

都是一样的报错。

[will-jl944]

方便提供能复现出这个报错的代码吗？

[smallhaozigithub]

稍等会哈。

[smallhaozigithub]

https://aistudio.baidu.com/aistudio/projectdetail/2907510?contributionType=1&shared=1

[will-jl944]

我们使用你的代码，以Python脚本的形式运行是没有问题的，在AI Studio中重启环境之后再恢复训练也是没问题的。只在不重启环境直接重新运行恢复训练的单元格的情况下复现出了这个报错。

此外，PPYOLO+ResNet50_vd_dcn模型本身就是没有报错信息中的conv2d_79这个卷积层的（最后一层卷积编号为78）。因此，推测可能是因为AI Studio中有未清理的缓存导致的。

可以尝试一下在恢复训练前手动重启环境，看是否还会出现报错。

[smallhaozigithub]

我在aistudio重启环境。把项目关闭，重新开启项目。都试过了，也还是这个情况 报错。 这就很尴尬了。。

[smallhaozigithub]

很奇怪，我本地测试了一下，把jupyter notebook 重启了，可以正常恢复训练。aistudio上不行。 我再研究下看看。

[bingnoi]

同样的问题 同样的2.1 重启之后还是有一样的报错...

[15254042250]

重启完内核，训练的多启动几次。

[Jacky-zrm]

我用paddlex的pdx.det.FasterRCNN训练自己的数据集，前面数据处理都没问题，但最后

num_classes = len(train_dataset.labels) + 1
model = pdx.det.FasterRCNN(num_classes=num_classes)
model.train(
    num_epochs=12,
    train_dataset=train_dataset,
    train_batch_size=2,
    eval_dataset=eval_dataset,
    learning_rate=0.0025,
    lr_decay_epochs=[4, 8],
    save_interval_epochs=1,
    save_dir='output/faster_rcnn_r50_fpn',
    pretrain_weights='IMAGENET',
    use_vdl=True)

报错：

!!! The CPU_NUM is not specified, you should set CPU_NUM in the environment variable list.
CPU_NUM indicates that how many CPUPlace are used in the current task.
And if this parameter are set as N (equal to the number of physical CPU core) the program may be faster.

export CPU_NUM=64 # for example, set CPU_NUM as number of physical CPU core which is 64.

!!! The default number of CPU_NUM=1.
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/layers/math_op_patch.py:416: DeprecationWarning: /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/nets/detection/fpn.py:172
The behavior of expression A + B has been unified with elementwise_add(X, Y, axis=-1) from Paddle 2.0. If your code works well in the older versions but crashes in this version, try to use elementwise_add(X, Y, axis=0) instead of A + B. This transitional warning will be dropped in the future.
  category=DeprecationWarning)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/tmp/ipykernel_336/206520623.py in <module>
     11     save_dir='output/faster_rcnn_r50_fpn',
     12     pretrain_weights='IMAGENET',
---> 13     use_vdl=True)

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/faster_rcnn.py in train(self, num_epochs, train_dataset, train_batch_size, eval_dataset, save_interval_epochs, log_interval_steps, save_dir, pretrain_weights, optimizer, learning_rate, warmup_steps, warmup_start_lr, lr_decay_epochs, lr_decay_gamma, metric, use_vdl, early_stop, early_stop_patience, resume_checkpoint, sensitivities_file, eval_metric_loss)
    347         self.optimizer = optimizer
    348         # 构建训练、验证、测试网络
--> 349         self.build_program()
    350         fuse_bn = True
    351         if self.with_fpn and self.backbone in [

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/base.py in build_program(self)
    103         paddlex.model_built = True
    104         # 构建训练网络
--> 105         self.train_inputs, self.train_outputs = self.build_net(mode='train')
    106         self.train_prog = fluid.default_main_program()
    107         startup_prog = fluid.default_startup_program()

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/faster_rcnn.py in build_net(self, mode)
    222         inputs = model.generate_inputs()
    223         if mode == 'train':
--> 224             model_out = model.build_net(inputs)
    225             loss = model_out['loss']
    226             self.optimizer.minimize(loss)

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/nets/detection/faster_rcnn.py in build_net(self, inputs)
    224             body_feats, spatial_scale = self.fpn.get_output(body_feats)
    225 
--> 226         rois = self.rpn_head.get_proposals(body_feats, im_info, mode=self.mode)
    227 
    228         if self.mode == 'train':

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/nets/detection/rpn_head.py in get_proposals(self, fpn_feats, im_info, mode)
    608             fpn_feat = fpn_feats[fpn_feat_name]
    609             rois_fpn, roi_probs_fpn = self._get_single_proposals(
--> 610                 fpn_feat, im_info, lvl, mode)
    611             self.fpn_rpn_list.append((self.rpn_cls_score, self.rpn_bbox_pred))
    612             rois_list.append(rois_fpn)

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/nets/detection/rpn_head.py in _get_single_proposals(self, body_feat, im_info, feat_lvl, mode)
    571                 nms_thresh=self.train_nms_thresh,
    572                 min_size=self.train_min_size,
--> 573                 eta=self.train_eta)
    574         else:
    575             rpn_rois_fpn, rpn_roi_prob_fpn = fluid.layers.generate_proposals(

ValueError: too many values to unpack (expected 2)

这是为什么？