How can I use the generated pte file to process my own data and predict the results?

[tayloryoung-o]

auto train_loader = torch::data::make_data_loader( SWaTegLoader("/dataset/train.csv", 100, 10, "train"), batch_size=256, torch::data::DataLoaderOptions().workers(0).shuffle(true) );

Is this correct? Then how do we process the data with the model?

for (auto& batch : *train_loader) {
    auto input = batch.data.to(device), labels = batch.target.to(device);
    auto output = method->execute(input)

Is it correct to write code in libtorch way?

[digantdesai]

PTE is the model represented in a way ExecuTorch runtime can consume it. The runtime can consume data / input tensors at runtime and run "forward" on the device.

Modify this to pass your inputs to the method.

https://github.com/pytorch/executorch/blob/main/examples/portable/executor_runner/executor_runner.cpp#L154-L171

[tayloryoung-o]

I mean, before converting to pte file, my model input is processed by pytorch's dataloader

data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=shuffle, num_workers=0)

So in executor_runner.cpp, can I import libtorch to process data with pytorch's C++ API? Like below

include<torch/torch.h>

include

include <executorch/extension/data_loader/file_data_loader.h>

include <executorch/extension/evalue_util/print_evalue.h>

If this is possible, how can I use libtorch in executorch successfully?

[ChristophKarlHeck]

Hi, I solved it the following way. You can also have a look here: https://github.com/ChristophKarlHeck/mbed-torch-fusion-os/tree/main

#include "mbed.h"
#include <stdio.h>
#include <memory>
#include <vector>

#include <executorch/extension/data_loader/buffer_data_loader.h>
#include <executorch/extension/runner_util/inputs.h>
#include <executorch/runtime/executor/program.h>
#include <executorch/runtime/platform/log.h>
#include <executorch/runtime/platform/platform.h>
#include <executorch/runtime/platform/runtime.h>

#include "model_pte.h"

using namespace exec_aten;
using namespace std;
using torch::executor::Error;
using torch::executor::Result;

__attribute__((section(".sram.data"), aligned(16)))
uint8_t method_allocator_pool[4 * 1024U];

//void et_pal_init(void) {}

__ET_NORETURN void et_pal_abort(void) {
  __builtin_trap();
}

/**
 * Emit a log message via platform output (serial port, console, etc).
 */
void et_pal_emit_log_message(
    __ET_UNUSED et_timestamp_t timestamp,
    et_pal_log_level_t level,
    const char* filename,
    __ET_UNUSED const char* function,
    size_t line,
    const char* message,
    __ET_UNUSED size_t length) {
  fprintf(stderr, "%c executorch:%s:%zu] %s\n", level, filename, line, message);
}

int main()
{
    torch::executor::runtime_init();

    auto loader =
        torch::executor::util::BufferDataLoader(model_pte, sizeof(model_pte));
    //ET_LOG(Info, "Model PTE file loaded. Size: %lu bytes.", sizeof(model_pte));
    Result<torch::executor::Program> program =
        torch::executor::Program::load(&loader);
    if (!program.ok()) {
        ET_LOG(
            Info,
            "Program loading failed @ 0x%p: 0x%" PRIx32,
            model_pte,
            program.error());
    }

    ET_LOG(Info, "Model buffer loaded, has %lu methods", program->num_methods());

    const char* method_name = nullptr;
    {
        const auto method_name_result = program->get_method_name(0);
        ET_CHECK_MSG(method_name_result.ok(), "Program has no methods");
        method_name = *method_name_result;
    }
    ET_LOG(Info, "Running method %s", method_name);

    Result<torch::executor::MethodMeta> method_meta =
        program->method_meta(method_name);
    if (!method_meta.ok()) {
        ET_LOG(
            Info,
            "Failed to get method_meta for %s: 0x%x",
            method_name,
            (unsigned int)method_meta.error());
    }

    torch::executor::MemoryAllocator method_allocator{
        torch::executor::MemoryAllocator(
            sizeof(method_allocator_pool), method_allocator_pool)};

    std::vector<std::unique_ptr<uint8_t[]>> planned_buffers; // Owns the memory
    std::vector<torch::executor::Span<uint8_t>>
        planned_spans; // Passed to the allocator
    size_t num_memory_planned_buffers = method_meta->num_memory_planned_buffers();

    for (size_t id = 0; id < num_memory_planned_buffers; ++id) {
        size_t buffer_size =
            static_cast<size_t>(method_meta->memory_planned_buffer_size(id).get());
        ET_LOG(Info, "Setting up planned buffer %zu, size %zu.", id, buffer_size);

        planned_buffers.push_back(std::make_unique<uint8_t[]>(buffer_size));
        planned_spans.push_back({planned_buffers.back().get(), buffer_size});
    }

    torch::executor::HierarchicalAllocator planned_memory(
        {planned_spans.data(), planned_spans.size()});

    torch::executor::MemoryManager memory_manager(
        &method_allocator, &planned_memory);

    Result<torch::executor::Method> method =
        program->load_method(method_name, &memory_manager);
    if (!method.ok()) {
        ET_LOG(
            Info,
            "Loading of method %s failed with status 0x%" PRIx32,
            method_name,
            method.error());
    }
    ET_LOG(Info, "Method loaded.");

    ET_LOG(Info, "Preparing inputs...");
    auto inputs = torch::executor::util::prepare_input_tensors(*method);
    if (!inputs.ok()) {
        ET_LOG(
            Info,
            "Preparing inputs tensors for method %s failed with status 0x%" PRIx32,
            method_name,
            inputs.error());
    }

    // Meta API
    const char* meta_name = method_meta->name();
    size_t num_inputs = method_meta->num_inputs();
    auto input_tag = method_meta->input_tag(0);
    auto input_tensor_meta = method_meta->input_tensor_meta(0);
    size_t num_outputs = method_meta->num_outputs();
    auto output_tag = method_meta->output_tag(0);
    auto output_tensor_meta = method_meta->output_tensor_meta(0);
    // Meta API END

    //Get Input
    size_t input_size = method->inputs_size();
    const torch::executor::EValue input_original = method->get_input(0);
    Tensor tensor = input_original.payload.as_tensor;
    float* data = input_original.payload.as_tensor.mutable_data_ptr<float>();

    // Change input
    int j;
    for(j = 0; j < tensor.numel(); ++j){
        data[j] = 4.0; // new value
    }

    // Set input 
    method->set_input(input_original,0);
    const torch::executor::EValue input_new = method->get_input(0);

    // Check if input has changed
    for (int i = 0; i < input_size; ++i) {
        Tensor te = input_new.payload.as_tensor;
        for (int j = 0; j < te.numel(); ++j) { // numel returns the number of elements in the tensor
            if (te.scalar_type() == ScalarType::Int) {
                printf(
                    "Input[%d][%d]: %d\n",
                    i,
                    j,
                    te.const_data_ptr<int>()[j]);
            } else {
                printf(
                    "Input[%d][%d]: %f\n",
                    i,
                    j,
                    te.const_data_ptr<float>()[j]);
            }
        }
    }

    ET_LOG(Info, "Starting the model execution...");
    Error status = method->execute();
    if (status != Error::Ok) {
        ET_LOG(
            Info,
            "Execution of method %s failed with status 0x%" PRIx32,
            method_name,
            status);
    } else {
        ET_LOG(Info, "Model executed successfully.");
    }

    std::vector<torch::executor::EValue> outputs(method->outputs_size());
    ET_LOG(Info, "%zu outputs: ", outputs.size());
    status = method->get_outputs(outputs.data(), outputs.size());
    ET_CHECK(status == Error::Ok);
    for (int i = 0; i < outputs.size(); ++i) {
        Tensor t = outputs[i].toTensor();
        for (int j = 0; j < outputs[i].toTensor().numel(); ++j) {
        if (t.scalar_type() == ScalarType::Int) {
            printf(
                "Output[%d][%d]: %d\n",
                i,
                j,
                outputs[i].toTensor().const_data_ptr<int>()[j]);
        } else {
            printf(
                "Output[%d][%d]: %f\n",
                i,
                j,
                outputs[i].toTensor().const_data_ptr<float>()[j]);
        }
        }
    }

    // main() is expected to loop forever.
    // If main() actually returns the processor will halt
    return 0;
}