search

openvinotoolkit / openvino

OpenVINO™ is an open-source toolkit for optimizing and deploying AI inference
https://docs.openvino.ai
Apache License 2.0
7.04k stars 2.22k forks source link

[Bug]: Errors happen when graph neural networks run with two inputs #20598

Closed hping666 closed 11 months ago

hping666 commented 11 months ago

OpenVINO Version

2023.1.0-12185-9e6b00e51cd-releases/2023/1

Operating System

Ubuntu 20.04 (LTS)

Device used for inference

CPU

Framework

PyTorch

Model used

Graph Attention Network

Issue description

Errors happen when graph neural networks run with two inputs

Error: Traceback (most recent call last): File "/home/u106118/openvino/GNN/GAT_openvino.py", line 90, in result = compiled_model({input_layer_1.any_name: features, input_layer_2.any_name:adj})[output_layer] File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/ie_api.py", line 384, in call return self._infer_request.infer( File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/ie_api.py", line 143, in infer return OVDict(super().infer(_data_dispatch( File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/utils/data_helpers/data_dispatcher.py", line 354, in _data_dispatch return create_shared(inputs, request) if is_shared else create_copied(inputs, request) File "/home/common/miniconda3/envs/openvino/lib/python3.10/functools.py", line 889, in wrapper return dispatch(args[0].class)(*args, *kw) File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/utils/data_helpers/datadispatcher.py", line 182, in return {k: value_to_tensor(v, request=request, is_shared=True, key=k) for k, v in request._inputs_data.items()} File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/utils/data_helpers/data_dispatcher.py", line 182, in return {k: value_to_tensor(v, request=request, is_shared=True, key=k) for k, v in request._inputs_data.items()} File "/home/common/miniconda3/envs/openvino/lib/python3.10/functools.py", line 889, in wrapper return dispatch(args[0].class)(args, **kw) File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/utils/data_helpers/datadispatcher.py", line 82, in return Tensor(value, shared_memory=is_shared) RuntimeError: Exception from src/bindings/python/src/pyopenvino/core/common.cpp:220: SHARED MEMORY MODE FOR THIS TENSOR IS NOT APPLICABLE! Passed numpy array must be C contiguous.

Step-by-step reproduction

import torch import torch.nn as nn import torch.nn.functional as F import scipy.sparse as sp import openvino as ov from pathlib import Path import warnings

class GraphAttentionLayer(nn.Module): """ graph attention layer """

def __init__(self, in_features, out_features, dropout, alpha, concat=True):
    super(GraphAttentionLayer, self).__init__()
    self.in_features = in_features
    self.out_features = out_features
    self.dropout = dropout
    self.alpha = alpha
    self.concat = concat

    self.W = nn.Parameter(torch.zeros(size=(in_features, out_features)))
    nn.init.xavier_uniform_(self.W.data)
    # nn.init.xavier_uniform(self.W.data, gain=1.414)
    self.a = nn.Parameter(torch.zeros(size=(2 * out_features, 1)))
    nn.init.xavier_uniform_(self.a.data)

    self.leakyrelu = nn.LeakyReLU(self.alpha)

def forward(self, input, adj):

    h = torch.mm(input, self.W)
    N = h.size()[0]
    # print(h.repeat(1, N).view(N*N, -1).size())
    # paired node feature concate
    a_input = torch.cat([h.repeat(1, N).view(N * N, -1), h.repeat(N, 1)], dim=1).view(
        N, -1, 2 * self.out_features
    )
    # print(a_input.size())
    e = self.leakyrelu(torch.matmul(a_input, self.a).squeeze(2))

    zero_vec = -9e15 * torch.ones_like(e)
    attention = torch.where(adj > 0, e, zero_vec)
    attention = F.softmax(attention, dim=1)
    attention = F.dropout(attention, self.dropout, training=self.training)
    h_prime = torch.matmul(attention, h)

    if self.concat:
        return F.elu(h_prime)
    else:
        return h_prime

def __repr__(self):
    return (
        self.__class__.__name__
        + " ("
        + str(self.in_features)
        + " -> "
        + str(self.out_features)
        + ")"
    )

class GAT(nn.Module): def init(self, nfeat, nhid, nclass, dropout, alpha, nheads): """Dense version of GAT.""" super(GAT, self).init() self.dropout = dropout

    self.attentions = [
        GraphAttentionLayer(nfeat, nhid, dropout=dropout, alpha=alpha, concat=True)
        for _ in range(nheads)
    ]
    for i, attention in enumerate(self.attentions):
        self.add_module("attention_{}".format(i), attention)

    self.out_att = GraphAttentionLayer(
        nhid * nheads, nclass, dropout=dropout, alpha=alpha, concat=False
    )

def forward(self, x, adj):

    x = F.dropout(x, self.dropout, training=self.training)
    x = torch.cat([att(x, adj) for att in self.attentions], dim=1)
    x = F.dropout(x, self.dropout, training=self.training)
    x = F.elu(self.out_att(x, adj))
    return F.log_softmax(x, dim=1)

def encode_onehot(labels): classes = set(labels) classes_dict = {c: np.identity(len(classes))[i, :] for i, c in enumerate(classes)} labels_onehot = np.array(list(map(classes_dict.get, labels)), dtype=np.int32) return labels_onehot

def load_data(path=r"./data/cora/", dataset="cora"): """Load citation network dataset (cora only for now)""" print("Loading {} dataset...".format(dataset))

idx_features_labels = np.genfromtxt("{}{}.content".format(path, dataset), dtype=np.dtype(str))
features = sp.csr_matrix(idx_features_labels[:, 1:-1], dtype=np.float32)
labels = encode_onehot(idx_features_labels[:, -1])

# build graph
idx = np.array(idx_features_labels[:, 0], dtype=np.int32)
idx_map = {j: i for i, j in enumerate(idx)}
edges_unordered = np.genfromtxt("{}{}.cites".format(path, dataset), dtype=np.int32)
edges = np.array(list(map(idx_map.get, edges_unordered.flatten())), dtype=np.int32).reshape(
    edges_unordered.shape
)
adj = sp.coo_matrix(
    (np.ones(edges.shape[0]), (edges[:, 0], edges[:, 1])),
    shape=(labels.shape[0], labels.shape[0]),
    dtype=np.float32,
)

# build symmetric adjacency matrix
adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj)

features = normalize_features(features)
adj = normalize_adj(adj + sp.eye(adj.shape[0]))

idx_train = range(140)
idx_val = range(200, 500)
idx_test = range(500, 1500)

adj = torch.FloatTensor(np.array(adj.todense()))
features = torch.FloatTensor(np.array(features.todense()))
labels = torch.LongTensor(np.where(labels)[1])

idx_train = torch.LongTensor(idx_train)
idx_val = torch.LongTensor(idx_val)
idx_test = torch.LongTensor(idx_test)

return adj, features, labels, idx_train, idx_val, idx_test

def normalize_adj(mx): """Row-normalize sparse matrix""" rowsum = np.array(mx.sum(1)) r_inv_sqrt = np.power(rowsum, -0.5).flatten() r_inv_sqrt[np.isinf(r_inv_sqrt)] = 0.0 r_mat_inv_sqrt = sp.diags(r_inv_sqrt) return mx.dot(r_mat_inv_sqrt).transpose().dot(r_mat_inv_sqrt)

def normalize_features(mx): """Row-normalize sparse matrix""" rowsum = np.array(mx.sum(1)) r_inv = np.power(rowsum, -1).flatten() r_inv[np.isinf(r_inv)] = 0.0 r_mat_inv = sp.diags(r_inv) mx = r_mat_inv.dot(mx) return mx

adj, features, labels, idx_train, idx_val, idx_test = load_data() net = GAT( nfeat=features.shape[1], nhid=8, nclass=int(labels.max()) + 1, dropout=0.6, nheads=8, alpha=0.2, )

print(net) onnx_path = Path('GAT.onnx') ir_path = onnx_path.with_suffix(".xml")

state_dict = torch.load('GAT.pt', map_location='cpu') # just use initial weight

net.load_state_dict(state_dict)

net.eval() print("Loaded GAT model")

input_names = ["input_0", "input_1"] with warnings.catch_warnings(): warnings.filterwarnings("ignore") if not onnx_path.exists(): dummy_input = features torch.onnx.export( net, (dummy_input, adj), onnx_path, input_names=input_names,

dynamic_axes={'input_0': [0]}

    )
    print(f"ONNX model exported to {onnx_path}.")
else:
    print(f"ONNX model {onnx_path} already exists.")

if not ir_path.exists(): print("Exporting ONNX model to IR... This may take a few minutes.") ov_model = ov.convert_model(onnx_path, input=[[2708, 1433], [2708, 2708]]) ov.save_model(ov_model, ir_path) else: print(f"IR model {ir_path} already exists.")

core = ov.Core() model_xml = "GAT.xml" model = core.read_model(model=model_xml) compiled_model = core.compile_model(model=model, device_name="CPU") input_layer_1 = compiled_model.input(0) input_layer_2 = compiled_model.input(1) output_layer = compiled_model.input(0) print(input_layer_1) print(input_layer_2) print(output_layer) result = compiled_model({input_layer_1.any_name: features, input_layer_2.any_name:adj})[output_layer]

Relevant log output

Traceback (most recent call last):
  File "/home/u106118/openvino/GNN/GAT_openvino.py", line 90, in <module>
    result = compiled_model({input_layer_1.any_name: features, input_layer_2.any_name:adj})[output_layer]
  File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/ie_api.py", line 384, in __call__
    return self._infer_request.infer(
  File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/ie_api.py", line 143, in infer
    return OVDict(super().infer(_data_dispatch(
  File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/utils/data_helpers/data_dispatcher.py", line 354, in _data_dispatch
    return create_shared(inputs, request) if is_shared else create_copied(inputs, request)
  File "/home/common/miniconda3/envs/openvino/lib/python3.10/functools.py", line 889, in wrapper
    return dispatch(args[0].__class__)(*args, **kw)
  File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/utils/data_helpers/data_dispatcher.py", line 182, in _
    return {k: value_to_tensor(v, request=request, is_shared=True, key=k) for k, v in request._inputs_data.items()}
  File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/utils/data_helpers/data_dispatcher.py", line 182, in <dictcomp>
    return {k: value_to_tensor(v, request=request, is_shared=True, key=k) for k, v in request._inputs_data.items()}
  File "/home/common/miniconda3/envs/openvino/lib/python3.10/functools.py", line 889, in wrapper
    return dispatch(args[0].__class__)(*args, **kw)
  File "/home/u106118/.local/lib/python3.10/site-packages/openvino/runtime/utils/data_helpers/data_dispatcher.py", line 82, in _
    return Tensor(value, shared_memory=is_shared)
RuntimeError: Exception from src/bindings/python/src/pyopenvino/core/common.cpp:220:
SHARED MEMORY MODE FOR THIS TENSOR IS NOT APPLICABLE! Passed numpy array must be C contiguous.

Issue submission checklist

hping666 commented 11 months ago

Sorry for your time, I have solved it.