Traceback (most recent call last):
File "run.py", line 70, in
main(cmd_args)
File "run.py", line 52, in main
trainer.train(train_loader=train_loader)
File "/home/shaotianc/OpenTransformer/otrans/train.py", line 91, in train
train_loss = self.train_one_epoch(epoch, train_loader.loader)
File "/home/shaotianc/OpenTransformer/otrans/train.py", line 137, in train_oneepoch
loss = self.model(*batch)
File "/home/shaotianc/miniconda3/envs/py35/lib/python3.5/site-packages/torch/nn/modules/module.py", line 532, in call
result = self.forward(input, **kwargs)
File "/home/shaotianc/OpenTransformer/otrans/model/transformer.py", line 54, in forward
logits, = self.decoder(target_in, targets_length, enc_states, enc_mask)
File "/home/shaotianc/miniconda3/envs/py35/lib/python3.5/site-packages/torch/nn/modules/module.py", line 532, in call
result = self.forward(*input, *kwargs)
File "/home/shaotianc/OpenTransformer/otrans/decoder.py", line 39, in forward
dec_output = self.pos_encoding(dec_output)
File "/home/shaotianc/miniconda3/envs/py35/lib/python3.5/site-packages/torch/nn/modules/module.py", line 532, in call
result = self.forward(input, **kwargs)
File "/home/shaotianc/OpenTransformer/otrans/module.py", line 51, in forward
self.extend_pe(x)
File "/home/shaotianc/OpenTransformer/otrans/module.py", line 30, in extend_pe
self.pe = self.pe.to(dtype=x.dtype, device=x.device)
RuntimeError: CUDA error: device-side assert triggered
Traceback (most recent call last): File "run.py", line 70, in
main(cmd_args)
File "run.py", line 52, in main
trainer.train(train_loader=train_loader)
File "/home/shaotianc/OpenTransformer/otrans/train.py", line 91, in train
train_loss = self.train_one_epoch(epoch, train_loader.loader)
File "/home/shaotianc/OpenTransformer/otrans/train.py", line 137, in train_oneepoch
loss = self.model(*batch)
File "/home/shaotianc/miniconda3/envs/py35/lib/python3.5/site-packages/torch/nn/modules/module.py", line 532, in call
result = self.forward(input, **kwargs)
File "/home/shaotianc/OpenTransformer/otrans/model/transformer.py", line 54, in forward
logits, = self.decoder(target_in, targets_length, enc_states, enc_mask)
File "/home/shaotianc/miniconda3/envs/py35/lib/python3.5/site-packages/torch/nn/modules/module.py", line 532, in call
result = self.forward(*input, *kwargs)
File "/home/shaotianc/OpenTransformer/otrans/decoder.py", line 39, in forward
dec_output = self.pos_encoding(dec_output)
File "/home/shaotianc/miniconda3/envs/py35/lib/python3.5/site-packages/torch/nn/modules/module.py", line 532, in call
result = self.forward(input, **kwargs)
File "/home/shaotianc/OpenTransformer/otrans/module.py", line 51, in forward
self.extend_pe(x)
File "/home/shaotianc/OpenTransformer/otrans/module.py", line 30, in extend_pe
self.pe = self.pe.to(dtype=x.dtype, device=x.device)
RuntimeError: CUDA error: device-side assert triggered