search

mindspore-ai / mindspore

MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
https://gitee.com/mindspore/mindspore
Apache License 2.0
4.26k stars 705 forks source link

mindspore_lite通过 java 调用 npu 进行推理问题 #299

Closed 983183947 closed 1 month ago

983183947 commented 1 month ago

问题

我有一个 onnx 文件通过 mindspore_lite转换得到.ms文件，但是我想通过 java 进行推理，程序一直提示我 NPU 有问题。

文件

这个文件是 paddlepaddle 的方向分类模型。 paddle_class.zip https://github.com/PaddlePaddle/PaddleClas/blob/release/2.5.2/docs/zh_CN/models/PULC/PULC_textline_orientation.md

环境

image image

代码

/*
 * Copyright 2022 Huawei Technologies Co., Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.mindspore.lite.demo;

import com.mindspore.MSTensor;
import com.mindspore.Model;
import com.mindspore.config.*;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.util.Random;

public class Main {
    private static Model model;

    public static float[] generateArray(int len) {
        Random rand = new Random();
        float[] arr = new float[len];
        for (int i = 0; i < arr.length; i++) {
            arr[i] = rand.nextFloat();
        }
        return arr;
    }

    private static ByteBuffer floatArrayToByteBuffer(float[] floats) {
        if (floats == null) {
            return null;
        }
        ByteBuffer buffer = ByteBuffer.allocateDirect(floats.length * Float.BYTES);
        buffer.order(ByteOrder.LITTLE_ENDIAN);
        FloatBuffer floatBuffer = buffer.asFloatBuffer();
        floatBuffer.put(floats);
        return buffer;
    }

    private static boolean compile(String modelPath) {
        MSContext context = new MSContext();
        // use default param init context
//        context.init();
        context.init(2, CpuBindMode.MID_CPU);
//        context.addDeviceInfo(DeviceType.DT_NPU, true, 3);
//        context.addDeviceInfo(DeviceType.DT_CPU, true);
        boolean ret = context.addDeviceInfo(DeviceType.DT_NPU, true, 3);
        if (!ret) {
            System.err.println("Compile graph failed");
            context.free();
            return false;
        }
        // Create the MindSpore lite session.
        model = new Model();
        // Compile graph.
        ret = model.build(modelPath, ModelType.MT_MINDIR, context);
        if (!ret) {
            System.err.println("Compile graph failed");
            model.free();
            return false;
        }
        return true;
    }

    private static boolean run() {
        MSTensor inputTensor = model.getInputByTensorName("graph_input-173");
        if (inputTensor.getDataType() != DataType.kNumberTypeFloat32) {
            System.err.println("Input tensor data type is not float, the data type is " + inputTensor.getDataType());
            return false;
        }
        // Generator Random Data.
        int elementNums = inputTensor.elementsNum();
        float[] randomData = generateArray(elementNums);
        ByteBuffer inputData = floatArrayToByteBuffer(randomData);

        // Set Input Data.
        inputTensor.setData(inputData);
        // Run Inference.
        boolean ret = model.predict();
        if (!ret) {
            inputTensor.free();
            System.err.println("MindSpore Lite run failed.");
            return false;
        }

        // Get Output Tensor Data.
        MSTensor outTensor = model.getOutputByTensorName("Softmax-65");

        // Print out Tensor Data.
        StringBuilder msgSb = new StringBuilder();
        msgSb.append("out tensor shape: [");
        int[] shape = outTensor.getShape();
        for (int dim : shape) {
            msgSb.append(dim).append(",");
        }
        msgSb.append("]");
        if (outTensor.getDataType() != DataType.kNumberTypeFloat32) {
            inputTensor.free();
            outTensor.free();
            System.err.println("output tensor data type is not float, the data type is " + outTensor.getDataType());
            return false;
        }
        float[] result = outTensor.getFloatData();
        if (result == null) {
            inputTensor.free();
            outTensor.free();
            System.err.println("decodeBytes return null");
            return false;
        }
        msgSb.append(" and out data:");
        for (int i = 0; i < 50 && i < outTensor.elementsNum(); i++) {
            msgSb.append(" ").append(result[i]);
        }
        System.out.println(msgSb.toString());
        // In/Out Tensor must free
        inputTensor.free();
        outTensor.free();
        return true;
    }

    private static void freeBuffer() {
        model.free();
    }

    public static void main(String[] args) {
        System.out.println(Version.version());
        if (args.length < 1) {
            System.err.println("The model path parameter must be passed.");
            return;
        }
        String modelPath = args[0];
        boolean ret = compile(modelPath);
        if (!ret) {
            System.err.println("MindSpore Lite compile failed.");
            return;
        }

        ret = run();
        if (!ret) {
            System.err.println("MindSpore Lite run failed.");
            freeBuffer();
            return;
        }

        freeBuffer();
    }
}

日志

(MindSpore) [ma-user bin]$./java -jar ../../quick_start_java.jar ../../model.ms 
Aug 09, 2024 1:55:24 PM com.mindspore.config.Version <clinit>
INFO: Version init ...
Aug 09, 2024 1:55:24 PM com.mindspore.config.Version init
INFO: Version init load ...
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLibrary
INFO: tryLoadLibrary mindspore-lite-jni failed: java.lang.UnsatisfiedLinkError: /home/ma-user/work/lib/libmindspore-lite-jni.so: libmindspore_glog.so.0: cannot open shared object file: No such file or directory
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLibrary
INFO: tryLoadLibrary mindspore-lite-train-jni failed: java.lang.UnsatisfiedLinkError: no mindspore-lite-train-jni in java.library.path
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libmindspore_glog.so.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: extract 198672 bytes to /tmp/mindspore_lite_libs-1723211724966-0/libmindspore_glog.so
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary deleteFile
SEVERE: delete tmp file : /tmp/mindspore_lite_libs-1723211724966-0/libmindspore_glog.so success.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary deleteFile
SEVERE: delete tmp file : /tmp/mindspore_lite_libs-1723211724966-0 success.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libopencv_core.so.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libopencv_core.so not exist.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libopencv_imgproc.so.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libopencv_imgproc.so not exist.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libopencv_imgcodecs.so.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libopencv_imgcodecs.so not exist.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libmindspore_core.so.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libmindspore_core.so not exist.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libmslite_converter_plugin.so.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libmslite_converter_plugin.so not exist.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libmindspore_converter.so.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libmindspore_converter.so not exist.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libdnnl.so.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libdnnl.so not exist.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/liblite-unified-executor.so.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/liblite-unified-executor.so not exist.
Aug 09, 2024 1:55:24 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libmindspore-lite.so.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary loadLib
INFO: extract 7609224 bytes to /tmp/mindspore_lite_libs-1723211724966-0/libmindspore-lite.so
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary extractLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libmsplugin-ge-litert.so not exist.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary extractLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libruntime_convert_plugin.so not exist.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary extractLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libascend_kernel_plugin.so not exist.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary extractLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libascend_ge_plugin.so not exist.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary extractLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libtransformer-shared.so not exist.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary extractLib
WARNING: lib file: com/mindspore/lite/linux_aarch64/libtensorrt_plugin.so not exist.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary deleteFile
SEVERE: delete tmp file : /tmp/mindspore_lite_libs-1723211724966-0/libmindspore-lite.so success.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary deleteFile
SEVERE: delete tmp file : /tmp/mindspore_lite_libs-1723211724966-0 success.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary loadLib
INFO: start load libResourceName: com/mindspore/lite/linux_aarch64/libmindspore-lite-jni.so.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary loadLib
INFO: extract 264000 bytes to /tmp/mindspore_lite_libs-1723211724966-0/libmindspore-lite-jni.so
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary deleteFile
SEVERE: delete tmp file : /tmp/mindspore_lite_libs-1723211724966-0/libmindspore-lite-jni.so success.
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary deleteFile
SEVERE: delete tmp file : /tmp/mindspore_lite_libs-1723211724966-0 success.
MindSpore Lite 2.3.1
Aug 09, 2024 1:55:25 PM com.mindspore.config.MindsporeLite <clinit>
INFO: MindsporeLite init ...
Aug 09, 2024 1:55:25 PM com.mindspore.config.MindsporeLite init
INFO: MindsporeLite init load ...
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary load
INFO: Native lib has been loaded.
Aug 09, 2024 1:55:25 PM com.mindspore.config.MindsporeLite init
INFO: MindsporeLite init load ...
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary load
INFO: Native lib has been loaded.
Aug 09, 2024 1:55:25 PM com.mindspore.config.MindsporeLite init
INFO: MindsporeLite init load ...
Aug 09, 2024 1:55:25 PM com.mindspore.config.NativeLibrary load
INFO: Native lib has been loaded.
[ERROR] ME(59028,ffff9ba4c0e0,java):2024-08-09-21:55:25.063.976 [mindspore/lite/src/litert/inner_context.cc:193] IsValid] NPU is not supported.
[ERROR] ME(59028,ffff9ba4c0e0,java):2024-08-09-21:55:25.064.004 [mindspore/lite/src/litert/inner_context.cc:100] Init] Context is not valid
[ERROR] ME(59028,ffff9ba4c0e0,java):2024-08-09-21:55:25.064.017 [mindspore/lite/src/litert/lite_session.cc:953] InitContext] Init Context failed
[ERROR] ME(59028,ffff9ba4c0e0,java):2024-08-09-21:55:25.064.029 [mindspore/lite/src/litert/lite_session.cc:1168] Init] Init Context failed
[ERROR] ME(59028,ffff9ba4c0e0,java):2024-08-09-21:55:25.064.041 [mindspore/lite/src/litert/cxx_api/model/model_impl.cc:968] CreateLiteSession] init session failed
[ERROR] ME(59028,ffff9ba4c0e0,java):2024-08-09-21:55:25.064.057 [mindspore/lite/src/litert/cxx_api/model/model_impl.cc:230] Build] Allocate session failed.
ERROR [rc/main/native/model.cpp:208] Java_com_mindspore_Model_buildByPath] Error status -2 during build of model
Compile graph failed
MindSpore Lite compile failed.
longvoyage commented 1 month ago

日志看是相关so没找到，jar依赖添加了没? 是用maven编译的吗？ 而且用910，为什么接口里面用npu?应该是DT_ASCEND

context.addDeviceInfo接口