Open Bryce1010 opened 4 years ago
collate_fn() https://discuss.pytorch.org/t/how-to-create-a-dataloader-with-variable-size-input/8278/2 By default, torch stacks the input image to from a tensor of size NCH*W, so every image in the batch must have the same height and width. In order to load a batch with variable size input image, we have to use our own collate_fn which is used to pack a batch of images.
For image classification, the input to collate_fn is a list of with size batch_size. Each element is a tuple where the first element is the input image(a torch.FloatTensor) and the second element is the image label which is simply an int. Because the samples in a batch have different size, we can store these samples in a list ans store the corresponding labels in torch.LongTensor. Then we put the image list and the label tensor into a list and return the result.
here is a very simple snippet to demonstrate how to write a custom collate_fn:
import torch
from torch.utils.data import DataLoader
from torchvision import transforms
import torchvision.datasets as datasets
import matplotlib.pyplot as plt
# a simple custom collate function, just to show the idea
def my_collate(batch):
data = [item[0] for item in batch]
target = [item[1] for item in batch]
target = torch.LongTensor(target)
return [data, target]
def show_image_batch(img_list, title=None):
num = len(img_list)
fig = plt.figure()
for i in range(num):
ax = fig.add_subplot(1, num, i+1)
ax.imshow(img_list[i].numpy().transpose([1,2,0]))
ax.set_title(title[i])
plt.show()
# do not do randomCrop to show that the custom collate_fn can handle images of different size
train_transforms = transforms.Compose([transforms.Scale(size = 224),
transforms.ToTensor(),
])
# change root to valid dir in your system, see ImageFolder documentation for more info
train_dataset = datasets.ImageFolder(root="/hd1/jdhao/toyset",
transform=train_transforms)
trainset = DataLoader(dataset=train_dataset,
batch_size=4,
shuffle=True,
collate_fn=my_collate, # use custom collate function here
pin_memory=True)
trainiter = iter(trainset)
imgs, labels = trainiter.next()
# print(type(imgs), type(labels))
show_image_batch(imgs, title=[train_dataset.classes[x] for x in labels])
a list of torch -> torch
torch.cat([x,x],0)
torch.stack(a)
tensor -> PIL Image
img=transforms.ToPILImage(mode='')(img).convert()
torch
torch.stack
(tensors, dim=0, out=None) → Tensor
将张量的序列沿一个新的维度连接起来
torch.``unsqueeze
(input, dim, out=None) → Tensor
torch.``squeeze
(input, dim=None, out=None) → Tensor
torch.``reshape
(input, shape) → Tensor
torch.``topk
(input, k, dim=None, largest=True, sorted=True, out=None) -> (Tensor, LongTensor)
>>> x = torch.arange(1., 6.)
>>> x
tensor([ 1., 2., 3., 4., 5.])
>>> torch.topk(x, 3)
torch.return_types.topk(values=tensor([5., 4., 3.]), indices=tensor([4, 3, 2]))
torch.nn.Module
add_module
(name, module)
apply
(fn)
children
()
cpu
()
cuda
(device=None)
eval
()
train
(mode=True)
float
()
half
()
forward
(*input)
load_state_dict
(state_dict, strict=True)
modules
()
named_children
()
named_modules
(memo=None, prefix='')
parameters
(recurse=True)
register_backward_hook
(hook)
hook(module, grad_input, grad_output) -> Tensor or None
register_forward_hook
(hook)
hook(module, input, output) -> None or modified output
requires_grad_
(requires_grad=True)
state_dict
(destination=None, prefix='', keep_vars=False)
to
(args, kwargs*)
to(device=None, dtype=None, non_blocking=False)
to(dtype, non_blocking=False)
to(tensor, non_blocking=False)
class Parameter(torch.Tensor):
"""
A kind of Tensor that is to be considered a module parameter.
Arguments:
data(Tensor): parameter tensor.
requires_grad(bool,optional)
"""
def __new__(cls, data=None, requires_grad=True):
def __deepcopy__(self, memo): # 重写deepcopy()
def __repr__(self):
所有网络模型的基类
add_module(name,module)
往当前模块增加子模块
- name (string) – name of the child module. The child module can be accessed from this module using the given name
- module (Module) – child module to be added to the module.
apply
(fn)
经常用来初始化函数
>>> def init_weights(m):
>>> print(m)
>>> if type(m) == nn.Linear:
>>> m.weight.data.fill_(1.0)
>>> print(m.weight)
>>> net = nn.Sequential(nn.Linear(2, 2), nn.Linear(2, 2))
>>> net.apply(init_weights)
buffers
(recurse=True)
>>> for buf in model.buffers():
>>> print(type(buf.data), buf.size())
<class 'torch.FloatTensor'> (20L,)
<class 'torch.FloatTensor'> (20L, 1L, 5L, 5L)
torch.optim
是实现了不同的优化算法的python包,除了支持大量经常使用的算法外,结构也很丰富,所以可以轻易地接入后续的开发升级。
optimizer的参数首先需要一个可迭代的参数对象,然后可以个性化的设置lr
,weight_decay
,等参数;
如果你在使用GPU运算,还需要使用optimizer.cuda()
将优化器放到显存中,保证优化的参数和优化器在统一的位置。
构建一个optimizer
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
optimizer = optim.Adam([var1, var2], lr=0.0001)
预先设置参数可选项
optimizer
还可以通过dict的key_word
个性化的设置优化器的参数;
optim.SGD([
{'params': model.base.parameters()},
{'params': model.classifier.parameters(), 'lr': 1e-3}
], lr=1e-2, momentum=0.9)
如何更新参数?optimizer.step()
optimizer
可以通过backward
之后使用step()函数来更新网络参数,可以有两种使用方法:
optimizer.step()
for input, target in dataset:
optimizer.zero_grad()
output = model(input)
loss = loss_fn(output, target)
loss.backward()
optimizer.step()
optimizer.step(closure)
for input, target in dataset:
def closure():
optimizer.zero_grad()
output = model(input)
loss = loss_fn(output, target)
loss.backward()
return loss
optimizer.step(closure)
torch.optim.Optimizer
(params, defaults)load_state_dict
(state_dict)
state_dict
()
step
(closure)
zero_grad
()
torch.optim.SGD
(params, lr=, momentum=0, dampening=0, weight_decay=0, nesterov=False)torch.optim.lr_scheduler
torch.optim.lr_scheduler.ReduceLROnPlateau
(optimizer, mode='min', factor=0.1, patience=10, verbose=False, threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-08)
>>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
>>> scheduler = ReduceLROnPlateau(optimizer, 'min')
>>> for epoch in range(10):
>>> train(...)
>>> val_loss = validate(...)
>>> # Note that step should be called after validate()
>>> scheduler.step(val_loss)
rep:
model_name = 'nasnetalarge' # could be fbresnet152 or inceptionresnetv2
model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')
model.eval()
model.input_size
model.input_space
model.input_range
model.mean
model.std
model.features
Method which is used to extract the features from the image.
print(input_224.size()) # (1,3,224,224)
output = model.features(input_224)
print(output.size()) # (1,2048,1,1)
# print(input_448.size()) # (1,3,448,448)
output = model.features(input_448)
# print(output.size()) # (1,2048,7,7)
model.logits
Method which is used to classify the features from the image.
output = model.features(input_224)
print(output.size()) # (1,2048, 1, 1)
output = model.logits(output)
print(output.size()) # (1,1000)
model.forward
Method used to call model.features
and model.logits
. It can be overwritten as desired.
# Without model.__call__
output = model.forward(input_224)
print(output.size()) # (1,1000)
# With model.__call__
output = model(input_224)
print(output.size()) # (1,1000)
model.last_linear
Attribut of type nn.Linear
. This module is the last one to be called during the forward pass.
nn.Linear
for fine tuning.pretrained.utils.Identity
for features extraction.print(input_224.size()) # (1,3,224,224)
output = model.features(input_224)
print(output.size()) # (1,2048,1,1)
output = model.logits(output)
print(output.size()) # (1,1000)
# fine tuning
dim_feats = model.last_linear.in_features # =2048
nb_classes = 4
model.last_linear = nn.Linear(dim_feats, nb_classes)
output = model(input_224)
print(output.size()) # (1,4)
# features extraction
model.last_linear = pretrained.utils.Identity()
output = model(input_224)
print(output.size()) # (1,2048)
DatasetMixin provides the __getitem__()
operator. The default implementation uses get_example()
to extract each example, and combines the results into a list. This mixin makes it easy to implement a new dataset that does not support efficient slicing.
__getitem__(index)
Returns an example or a sequence of examples.
It implements the standard Python indexing and one-dimensional integer array indexing. It uses the get_example()
method by default, but it may be overridden by the implementation to, for example, improve the slicing performance.
>>> import numpy
>>> from chainer import dataset
>>> class SimpleDataset(dataset.DatasetMixin):
... def __init__(self, values):
... self.values = values
... def __len__(self):
... return len(self.values)
... def get_example(self, i):
... return self.values[i]
...
>>> ds = SimpleDataset([0, 1, 2, 3, 4, 5])
>>> ds[1] # Access by int
1
>>> ds[1:3] # Access by slice
[1, 2]
>>> ds[[4, 0]] # Access by one-dimensional integer list
[4, 0]
>>> index = numpy.arange(3)
>>> ds[index] # Access by one-dimensional integer numpy.ndarray
[0, 1, 2]
__len__()
Return the number of datapoint.
get_example(i)
Returns the i-th example.
Implementations should override it. It should raise IndexError
if the index is invalid.
[Learning and testing neural networks on PyTorch using Ignite]
通过Engine循环提供数据和执行进程函数;
例如执行一个监督学习任务:
训练过程
def update_model(trainer,batch):
model.train()
optimizer.zero_grad()
x,y=prepare_batch.next(batch)
y_pred=model(x)
loss=loss_fn(y_pred,y)
loss.backward()
optimizer.step()
return loss.item()
trainer=Engine(update_model)
trainer.run(data,max_epoch=100)
测试过程
from ignite.engine import Engine
total_loss = []
def compute_metrics(_, batch):
x, y = batch
model.eval()
with torch.no_grad():
y_pred = model(x)
loss = criterion(y_pred, y)
total_loss.append(loss.item())
return loss.item()
evaluator = Engine(compute_metrics)
evaluator.run(data, max_epochs=1)
print(f”Loss: {torch.tensor(total_loss).mean()}”)
Ignite的Events & Handlers设计结构如下所示:
fire_event(Events.STARTED)
while epoch < max_epochs:
fire_event(Events.EPOCH_STARTED)
# run once on data
for batch in data:
fire_event(Events.ITERATION_STARTED)
output = process_function(batch)
fire_event(Events.ITERATION_COMPLETED)
fire_event(Events.EPOCH_COMPLETED)
fire_event(Events.COMPLETED)
如何调用Events & Handlers?
train_loader = …
model = …
optimizer = …
criterion = ...
lr_scheduler = …
def process_function(engine, batch):
# … user function to update model weights
trainer = Engine(process_function)
@trainer.on(Events.STARTED)
def setup_logging_folder(_):
# create a folder for the run
# set up some run dependent variables
@trainer.on(Events.ITERATION_COMPLETED)
def update_lr(engine):
lr_scheduler.step()
trainer.run(train_loader, max_epochs=50)
# --------------------------
trainer=Engine(update_model)
trainer.add_event_hander(
Events.STARTED,
lambda engine:print("strating training")
)
# or
@trainer.on(Events.STARTED)
def on_training_started(engine):
print("Another message of start training")
# attach handler with args,kwargs
mydata=[1,2,3,4]
def on_train_ended(engine,data):
print("Training is ended.mydata={}".format(data))
trainer.add_event_handler(
Events.COMPLETED,on_training_ended,my_data
)
run the validation every 5 epochs,
store a checkpoint every 1000 iterations,
change a variable on 20th epoch,
log gradients on the first 10 iterations.
etc.
每五个epoch做一次validation
@trainer.on(Events.EPOCH_COMPLETED(every=5))
def run_validation(_):
# run validation
在第20个epoch改变训练变量的值
@trainer.on(Events.EPOCH_COMPLETED(every=5))
def run_validation(_):
# run validation
custom filtering method
def first_x_iters(_, event):
if event < 10:
return True
return False
@trainer.on(Events.ITERATION_COMPLETED(event_filter=first_x_iters))
def log_gradients(_):
# …
supports Tensorboard, Visdom, MLflow, Polyaxo
Ignite also provides a list of out-of-the-box metrics for various tasks: Precision, Recall, Accuracy, Confusion Matrix, IoU etc, ~20 regression metrics
from ignite.metrics import Accuracy
def compute_predictions(_, batch):
# …
return y_pred, y_true
evaluator = Engine(compute_predictions)
metric = Accuracy()
metric.attach(evaluator, "val_accuracy")
evaluator.run(val_loader)
> evaluator.state.metrics[“val_accuracy”] = 0.98765
precision = Precision(average=False)
recall = Recall(average=False)
F1_per_class = (precision * recall * 2 / (precision + recall))
F1_mean = F1_per_class.mean() # torch mean method
F1_mean.attach(engine, "F1")
Go here and here to see the full list of available metrics.
Save best validation
to_save{'model':classifier,'optimizer':optimizer,'scheduler':scheduler,'trainer':trainer}
def score_function(engine):
val_recall=engine.state.metrics['recall']
return val_recall
checkpoint_handler=Checkpoint(to_save,DiskSaver(outdir,create_dir=True),n_saved=2,filename_prefix='best',score_function=score_function,score_name='recall',global_step_transform=global_step_from_engine(trainer))
#trainer.add_event_handler(Events.EPOCH_COMPLETED,ModelSnapshotHandler((predictor,filepath=outdir/'model_{count:06}.pth',interval=5))
evaluator.add_event_handler(Events.COMPLETED,checkpoint_handler)
Early stopping
from ignite.handlers
import EarlyStopping
early_stopping = EarlyStopping(patience=10, score_function=score_function, trainer=trainer)
val_evaluator.add_event_handler(Events.EPOCH_COMPLETED, early_stopping)
TTA
资源