Running spacy.prefer_gpu() caused supar parser GPU error for some inputs

[hardianlawi]

Code to reproduce the error

import spacy
from supar import Parser

spacy.prefer_gpu()

_input = [
    (
        (
            "https://www.kiro7.com/news/trending/jared-bell-drake-josh-faces-charges/EGMJACNBUVBZRJN5BSWCJMNZRU/",
            "https://www.kiro7.com/news/trending/jared-bell-drake-josh-faces-charges/egmjacnbuvbzrjn5bswcjmnzru/",
            "ADD",
        ),
    ),
    (
        ("Jared", "Jared", "NNP"),
        ("Bell", "Bell", "NNP"),
        ("from", "from", "IN"),
        ("‘", "'", "``"),
        ("Drake", "Drake", "NNP"),
        ("&", "&", "NNP"),
        ("amp", "amp", "NNP"),
        (";", ";", ":"),
        ("Josh", "Josh", "NNP"),
        ("’", "'", "``"),
        ("faces", "face", "VBZ"),
        ("charges", "charge", "NNS"),
        ("of", "of", "IN"),
        ("crimes", "crime", "NNS"),
        ("against", "against", "IN"),
        ("a", "a", "DT"),
        ("child", "child", "NN"),
    ),
]

model = Parser.load("biaffine-dep-roberta-en")
parsed = model.predict(_input, verbose=False)

Spacy version: 3.2.0 Supar version: 1.1.3

But some inputs don't get any errors for instance

_input = [
    (
        (
            "https://www.kaiserin-magazine.com/latest-news/ordnance-factory-board-to-be-split-into-7-entities-for-defence-manufacturing/",
            "https://www.kaiserin-magazine.com/latest-news/ordnance-factory-board-to-be-split-into-7-entities-for-defence-manufacturing/",
            "NN",
        ),
    ),
    (
        ("Ordnance", "Ordnance", "NNP"),
        ("Manufacturing", "Manufacturing", "NNP"),
        ("facility", "facility", "NN"),
        ("Board", "board", "NN"),
        ("To", "to", "TO"),
        ("Be", "be", "VB"),
        ("Cut", "Cut", "VBN"),
        ("up", "up", "RP"),
        ("Into", "into", "IN"),
        ("7", "7", "CD"),
        ("Entities", "entity", "NNS"),
        ("For", "for", "IN"),
        ("Defence", "defence", "NN"),
        ("Manufacturing", "manufacturing", "NN"),
    ),
]

parsed = model.predict(_input, verbose=False)

[yzhangcs]

@hardianlawi Hi, could you give me more details, I didn't encounter any error while running the first snippet :-(

[hardianlawi]

@yzhangcs Thanks for your reply. I got the error below when running the first snippet

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-1-6fcfb664c784> in <module>
     34
     35 model = Parser.load("biaffine-dep-roberta-en")
---> 36 parsed = model.predict(_input, verbose=False)

~/miniforge3/envs/pod-classification/lib/python3.8/site-packages/supar/parsers/dep.py in predict(self, data, pred, lang, buckets, batch_size, prob, tree, proj, verbose, **kwargs)
    122         """
    123
--> 124         return super().predict(**Config().update(locals()))
    125
    126     @classmethod

~/miniforge3/envs/pod-classification/lib/python3.8/site-packages/supar/parsers/parser.py in predict(self, data, pred, lang, buckets, batch_size, prob, **kwargs)
    135         logger.info("Making predictions on the dataset")
    136         start = datetime.now()
--> 137         preds = self._predict(dataset.loader)
    138         elapsed = datetime.now() - start
    139

~/miniforge3/envs/pod-classification/lib/python3.8/site-packages/torch/autograd/grad_mode.py in decorate_context(*args, **kwargs)
     26         def decorate_context(*args, **kwargs):
     27             with self.__class__():
---> 28                 return func(*args, **kwargs)
     29         return cast(F, decorate_context)
     30

~/miniforge3/envs/pod-classification/lib/python3.8/site-packages/supar/parsers/dep.py in _predict(self, loader)
    222             lens = mask.sum(1).tolist()
    223             s_arc, s_rel = self.model(words, feats)
--> 224             arc_preds, rel_preds = self.model.decode(s_arc, s_rel, mask, self.args.tree, self.args.proj)
    225             preds['arcs'].extend(arc_preds[mask].split(lens))
    226             preds['rels'].extend(rel_preds[mask].split(lens))

~/miniforge3/envs/pod-classification/lib/python3.8/site-packages/supar/models/dep.py in decode(self, s_arc, s_rel, mask, tree, proj)
    227         bad = [not CoNLL.istree(seq[1:i+1], proj) for i, seq in zip(lens.tolist(), arc_preds.tolist())]
    228         if tree and any(bad):
--> 229             arc_preds[bad] = (DependencyCRF if proj else MatrixTree)(s_arc[bad], mask[bad].sum(-1)).argmax
    230         rel_preds = s_rel.argmax(-1).gather(-1, arc_preds.unsqueeze(-1)).squeeze(-1)
    231

~/miniforge3/envs/pod-classification/lib/python3.8/site-packages/torch/distributions/utils.py in __get__(self, instance, obj_type)
    108             return _lazy_property_and_property(self.wrapped)
    109         with torch.enable_grad():
--> 110             value = self.wrapped(instance)
    111         setattr(instance, self.wrapped.__name__, value)
    112         return value

~/miniforge3/envs/pod-classification/lib/python3.8/site-packages/supar/structs/tree.py in argmax(self)
     60     def argmax(self):
     61         with torch.no_grad():
---> 62             return mst(self.scores, self.mask, self.multiroot)
     63
     64     def kmax(self, k):

~/miniforge3/envs/pod-classification/lib/python3.8/site-packages/supar/utils/alg.py in mst(scores, mask, multiroot)
    268     for i, length in enumerate(mask.sum(1).tolist()):
    269         s = scores[i][:length+1, :length+1]
--> 270         tree = chuliu_edmonds(s)
    271         roots = torch.where(tree[1:].eq(0))[0] + 1
    272         if not multiroot and len(roots) > 1:

~/miniforge3/envs/pod-classification/lib/python3.8/site-packages/supar/utils/alg.py in chuliu_edmonds(s)
    214     subtree = y < len(y)
    215     # add the nodes to the new tree
--> 216     tree[noncycle[subtree]] = noncycle[y[subtree]]
    217     # fix the subtree with heads coming from the cycle
    218     subtree = ~subtree

RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and CPU!

Did spacy.prefer_gpu() return True for you?

[yzhangcs]

@hardianlawi No, spacy.prefer_gpu() returns False. What is this function used for? You just need torch.cuda.set_device if you need parsing on GPUs.

[hardianlawi]

spacy.prefer_gpu() is used so that spacy models run on GPU. I ran that line because I am using spacy models aside from this parser but I was quite surprised that it could somehow affect this library. In order to reproduce it, I think you would need to have GPU and ensure spacy.prefer_gpu() returns True :')

[yzhangcs]

@hardianlawi Hi, I can't figure out why spacy.prefer_gpu always returns False, even though my GPU devices are actually available. However, I have checked the source code of spacy.prefer_gpu, and find it's indeed a wrapper for torch.cuda.set_deive. So I would expect you to use torch.cuda.set_device instead and check it again.

[hardianlawi]

You need to install spacy gpu for spacy.prefer_gpu to return True, but if that doesn't work, then I don't know what's wrong.

Nevertheless, I might have found the problem.

import torch
from supar import Parser

torch.cuda.set_device(0)
torch.set_default_tensor_type(torch.cuda.FloatTensor)  # <--- the root cause of the error above

_input = [
    (
        (
            "https://www.kiro7.com/news/trending/jared-bell-drake-josh-faces-charges/EGMJACNBUVBZRJN5BSWCJMNZRU/",
            "https://www.kiro7.com/news/trending/jared-bell-drake-josh-faces-charges/egmjacnbuvbzrjn5bswcjmnzru/",
            "ADD",
        ),
    ),
    (
        ("Jared", "Jared", "NNP"),
        ("Bell", "Bell", "NNP"),
        ("from", "from", "IN"),
        ("‘", "'", "``"),
        ("Drake", "Drake", "NNP"),
        ("&", "&", "NNP"),
        ("amp", "amp", "NNP"),
        (";", ";", ":"),
        ("Josh", "Josh", "NNP"),
        ("’", "'", "``"),
        ("faces", "face", "VBZ"),
        ("charges", "charge", "NNS"),
        ("of", "of", "IN"),
        ("crimes", "crime", "NNS"),
        ("against", "against", "IN"),
        ("a", "a", "DT"),
        ("child", "child", "NN"),
    ),
]

model = Parser.load("biaffine-dep-roberta-en")
parsed = model.predict(_input, verbose=False)

All in all, thanks for being helpful, and really appreciate the prompt reply!

[yzhangcs]

@hardianlawi This should be an intrinsic problem of torch.nn.utils.rnn.pack_padded_sequence. If you set torch.set_default_tensor_type(torch.cuda.FloatTensor), calling pack_padded_sequence(<tensors>, <lengths>) will always raise the error. Thank you for pointing this out!

[hardianlawi]

Got it. So this should be reported to https://github.com/pytorch/pytorch instead?

[yzhangcs]

@hardianlawi Yeah, but not sure if it has been reported. Close this issue as the bug has been located.

[hardianlawi]

Doesn't seem like it has been closed https://github.com/pytorch/pytorch/issues/16542 but thanks for the help!