ccssu
commented
1 year ago Pruning Group
pytorch
pt_all_groups
```sh
--------------------------------
Pruning Group
--------------------------------
[0] prune_out_channels on block2_1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block2_1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=10
[1] prune_out_channels on block2_1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block2_1.1 (BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), #idxs=10
--------------------------------
--------------------------------
Pruning Group
--------------------------------
[0] prune_out_channels on block1.3 (Conv2d(10, 30, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block1.3 (Conv2d(10, 30, kernel_size=(1, 1), stride=(1, 1))), #idxs=30
[1] prune_out_channels on block1.3 (Conv2d(10, 30, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block1.4 (BatchNorm2d(30, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), #idxs=30
[2] prune_out_channels on block1.4 (BatchNorm2d(30, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)) => prune_out_channels on _SplitOp_0([0, 10, 30]), #idxs=30
[3] prune_out_channels on _SplitOp_0([0, 10, 30]) => prune_in_channels on block2_1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=10
[4] prune_out_channels on _SplitOp_0([0, 10, 30]) => prune_in_channels on block2_2.0 (Conv2d(20, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=20
--------------------------------
--------------------------------
Pruning Group
--------------------------------
[0] prune_out_channels on block1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=10
[1] prune_out_channels on block1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block1.1 (BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), #idxs=10
[2] prune_out_channels on block1.1 (BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)) => prune_out_channels on _ElementWiseOp_1(GeluBackward0), #idxs=10
[3] prune_out_channels on _ElementWiseOp_1(GeluBackward0) => prune_in_channels on block1.3 (Conv2d(10, 30, kernel_size=(1, 1), stride=(1, 1))), #idxs=10
--------------------------------
--------------------------------
Pruning Group
--------------------------------
[0] prune_out_channels on block2_2.0 (Conv2d(20, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block2_2.0 (Conv2d(20, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=10
[1] prune_out_channels on block2_2.0 (Conv2d(20, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block2_2.1 (BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), #idxs=10
--------------------------------
```
graph TB
A[Input] -->B["block1.0: Conv2d + BN"]
B --> D["_ElementWiseOp_1(GeluBackward0)"]
D --> E["block1.3: Conv2d "]
E --> F["block1.4: BatchNorm2d"]
F --> G["_SplitOp_0([0, 10, 30])"]
G --> H["block2_1: Conv2d + BN "]
G --> I["block2_2: Conv2d + BN"]
H --> J[Output1]
I --> K[Output2]OneFlow
oneflow_all_groups
```sh -------------------------------- Pruning Group -------------------------------- [0] prune_out_channels on block2_1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block2_1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=10 [1] prune_out_channels on block2_1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block2_1.1 (BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), #idxs=10 -------------------------------- -------------------------------- Pruning Group -------------------------------- [0] prune_out_channels on block1.3 (Conv2d(10, 30, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block1.3 (Conv2d(10, 30, kernel_size=(1, 1), stride=(1, 1))), #idxs=30 [1] prune_out_channels on block1.3 (Conv2d(10, 30, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block1.4 (BatchNorm2d(30, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), #idxs=30 [2] prune_out_channels on block1.4 (BatchNorm2d(30, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)) => prune_out_channels on _SplitOp_1([0, 10]), #idxs=30 [3] prune_out_channels on block1.4 (BatchNorm2d(30, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)) => prune_out_channels on _SplitOp_6([0, 20]), #idxs=30 [4] prune_out_channels on _SplitOp_6([0, 20]) => prune_out_channels on _ElementWiseOp_5(conv2dBackward), #idxs=20 [5] prune_out_channels on _ElementWiseOp_5(conv2dBackward) => prune_in_channels on block2_2.0 (Conv2d(20, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=20 [6] prune_out_channels on _SplitOp_1([0, 10]) => prune_out_channels on _ElementWiseOp_0(conv2dBackward), #idxs=10 [7] prune_out_channels on _ElementWiseOp_0(conv2dBackward) => prune_in_channels on block2_1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=10 -------------------------------- -------------------------------- Pruning Group -------------------------------- [0] prune_out_channels on block1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=10 [1] prune_out_channels on block1.0 (Conv2d(10, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block1.1 (BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), #idxs=10 [2] prune_out_channels on block1.1 (BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)) => prune_out_channels on _ElementWiseOp_3(geluBackward), #idxs=10 [3] prune_out_channels on _ElementWiseOp_3(geluBackward) => prune_out_channels on _ElementWiseOp_2(conv2dBackward), #idxs=10 [4] prune_out_channels on _ElementWiseOp_2(conv2dBackward) => prune_in_channels on block1.3 (Conv2d(10, 30, kernel_size=(1, 1), stride=(1, 1))), #idxs=10 -------------------------------- -------------------------------- Pruning Group -------------------------------- [0] prune_out_channels on block2_2.0 (Conv2d(20, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block2_2.0 (Conv2d(20, 10, kernel_size=(1, 1), stride=(1, 1))), #idxs=10 [1] prune_out_channels on block2_2.0 (Conv2d(20, 10, kernel_size=(1, 1), stride=(1, 1))) => prune_out_channels on block2_2.1 (BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)), #idxs=10 -------------------------------- ```graph TB
A[Input] -->B["block1.0: Conv2d + BN"]
B --> D["_ElementWiseOp_3(geluBackward)"]
D --> E["_ElementWiseOp_2(conv2dBackward)"]
E --> F["block1.3: Conv2d"]
F --> G["block1.4: BatchNorm2d"]
G --> H["_SplitOp_1([0, 10])"]
G --> I["_SplitOp_6([0, 20])"]
H --> K["_ElementWiseOp_0(conv2dBackward)"]
K --> L["block2_1: Conv2d + BN"]
I --> M["_ElementWiseOp_5(conv2dBackward)"]
M --> N["block2_2: Conv2d + BN"]
L --> O[Output1]
N --> P[Output2]
目录
问题记录
解决进度
END目录
脚本使用 模型
class Net(nn.Module):
```py class Net(nn.Module): def __init__(self, in_dim): super().__init__() self.block1 = nn.Sequential( nn.Conv2d(in_dim, in_dim, 1), nn.BatchNorm2d(in_dim), nn.GELU(), nn.Conv2d(in_dim, in_dim*3, 1), nn.BatchNorm2d(in_dim*3) ) self.block2_1 = nn.Sequential( nn.Conv2d(in_dim, in_dim, 1), nn.BatchNorm2d(in_dim) ) self.block2_2 = nn.Sequential( nn.Conv2d(2*in_dim, in_dim, 1), nn.BatchNorm2d(in_dim) ) def forward(self, x): x = self.block1(x) num_ch = x.shape[1] c1, c2 = self.block2_1[0].in_channels, self.block2_2[0].in_channels x1, x3 = torch.split(x, [c1, c2], dim=1) x1 = self.block2_1(x1) #x2 = self.block2_1(x2) x3 = self.block2_2(x3) return x1, x3 ```报错信息
环境信息
Linux oneflow27-root 错误脚本 commit号 ： 0ce83f1 (HEAD -> master) HEAD@{0}: reset: moving to 0ce83f1 https://docs.oneflow.org/master/cookies/oneflow_torch.html
启动指令
eval $(python3 -m oneflow.mock_torch) && python test_split.pyfile 路径 OneFlow-Pruning/tests/test_split.py
```py import sys, os sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) import torch import torch_pruning as tp import torch.nn as nn class Net(nn.Module): def __init__(self, in_dim): super().__init__() self.block1 = nn.Sequential( nn.Conv2d(in_dim, in_dim, 1), nn.BatchNorm2d(in_dim), nn.GELU(), nn.Conv2d(in_dim, in_dim*3, 1), nn.BatchNorm2d(in_dim*3) ) self.block2_1 = nn.Sequential( nn.Conv2d(in_dim, in_dim, 1), nn.BatchNorm2d(in_dim) ) self.block2_2 = nn.Sequential( nn.Conv2d(2*in_dim, in_dim, 1), nn.BatchNorm2d(in_dim) ) def forward(self, x): x = self.block1(x) num_ch = x.shape[1] c1, c2 = self.block2_1[0].in_channels, self.block2_2[0].in_channels x1, x3 = torch.split(x, [c1, c2], dim=1) x1 = self.block2_1(x1) #x2 = self.block2_1(x2) x3 = self.block2_2(x3) return x1, x3 def test_pruner(): model = Net(10) print(model) # Global metrics example_inputs = torch.randn(1, 10, 7, 7) imp = tp.importance.RandomImportance() ignored_layers = [] # DO NOT prune the final classifier! for m in model.modules(): if isinstance(m, torch.nn.Linear) and m.out_features == 1000: ignored_layers.append(m) iterative_steps = 1 pruner = tp.pruner.MagnitudePruner( model, example_inputs, importance=imp, iterative_steps=iterative_steps, ch_sparsity=0.5, # remove 50% channels, ResNet18 = {64, 128, 256, 512} => ResNet18_Half = {32, 64, 128, 256} ignored_layers=ignored_layers, ) for g in pruner.DG.get_all_groups(): pass base_macs, base_nparams = tp.utils.count_ops_and_params(model, example_inputs) for i in range(iterative_steps): for g in pruner.step(interactive=True): print(g.details()) g.prune() print(model) macs, nparams = tp.utils.count_ops_and_params(model, example_inputs) print([o.shape for o in model(example_inputs)]) print( " Iter %d/%d, Params: %.2f => %.2f" % (i+1, iterative_steps, base_nparams, nparams) ) print( " Iter %d/%d, MACs: %.2f => %.2f" % (i+1, iterative_steps, base_macs, macs ) ) # finetune your model here # finetune(model) # ... if __name__=='__main__': test_pruner() ```通这个错误 发现 要保证这个mock torch兼容 https://github.com/VainF/Torch-Pruning 达到 0 改动。