simple function for loading messages?

[waynenilsen]

I think this could be great for python (and other) FFI so I have been trying to figure out how to load &[u8] into an actual object. This is what I have so far but I think it's probably misguided.

#[no_mangle]
pub extern fn test_capnproto(thing: *const u8, length: size_t) {
    let array: &[u8] = unsafe{ slice::from_raw_parts(thing, length as usize) };
    let mut message = MallocMessageBuilder::new_default();
    let mut test_date = message.init_root::<date::Builder>();
    { // scope for builder 
        let b = test_date.borrow();
        for (i, elem) in array.iter().enumerate() {
            b.set_data_field(i, *elem);
        }
    }

    println!("{:?}", test_date);
}

Any tips? Perhaps something to do with read_message? I couldn't figure it out from the examples

[aij]

Could you say what it is you're trying to do?

If you're trying to read a serialized message, you probably want capnp::serialize::read_message or capnp::serialize_packed::read_message

I must admit I'm having a little trouble figuring out how to return the message once it's parsed, due to lifetime issues, but if you just want to print it that shouldn't be a problem.

[waynenilsen]

I'm trying to interface between rust and python with a nice common binary format. Basically I need something that takes &[u8] and returns the message object (if possible).

I couldn't quite figure out how to get this working with read_message.

[dwrensha]

Basically I need something that takes &[u8] and returns the message object (if possible).

Unfortunately, we don't yet have a function that can conveniently do this without copying the data. We had some discussion and made some progress on this on issue #25.

If you're willing to copy the data, then capnp::serialize::read_message() should work fine.

[dwrensha]

I couldn't quite figure out how to get this working with read_message.

Something like this should work:

fn foo(mut buffer: &[u8]) -> capnp::Result<capnp::OwnedSpaceMessageReader> {
   capnp::serialize::read_message(&mut buffer, capnp::ReaderOptions::new())
}

[aij]

Oh, I think you mean

capnp::serialize::read_message(&mut Cursor::new(&mut buffer), capnp::ReaderOptions::new())

since read_message requires a &mut Read.

Also, is there a way to return a date::Reader (to go with the OP's example) or at least some sort of less dynamically-typed MessageReader? The only way I've found so far to return a specific type of Reader , is to take a &mut MallocMessageBuilder and copy it into there.

Eg:

let message_reader = try!(capnp::serialize_packed::read_message(&mut io::Cursor::new(bytes), capnp::message::DEFAULT_READER_OPTIONS));
let reader : try!(date::Reader = message_reader.get_root());
try!(message_builder.set_root(reader));
Ok(try!(message.get_root::<date::Builder>()).as_reader())

[dwrensha]

Also, is there a way to return a date::Reader (to go with the OP's example) or at least some sort of less dynamically-typed MessageReader?

One way to go about this might be to implement a container type, like this

pub struct OwnedMessage<T> {
   phantom_data: ::std::marker::PhantomData<T>,
   message: ::capnp::OwnedSpaceMessageReader,
}

impl <'a, T> OwnedMessage <T> where T: ::capnp::traits::FromPointerReader<'a> {
   pub fn get(&'a self) -> ::capnp::Result<T> {
     use capnp::MessageReader;
     self.message.get_root()
   }
}

You could then return a OwnedMessage<date::Reader> from foo().

[waynenilsen]

Thanks for the suggestions! Oddly this compiles

fn foo(mut buffer: &[u8]) -> capnp::Result<capnp::OwnedSpaceMessageReader> {
   capnp::serialize::read_message(&mut buffer, capnp::ReaderOptions::new())
}

and this does not

fn foo(mut buffer: &[u8]) -> capnp::Result<capnp::OwnedSpaceMessageReader> {
   capnp::serialize::read_message(&mut std::io::Cursor::new(&mut buffer), capnp::ReaderOptions::new())
}

with message

cargo build --release (in directory: /home/wayne/projects/rust-py-test/src)
   Compiling py-test v0.1.0 (file:///home/wayne/projects/rust-py-test)
lib.rs:63:4: 63:34 error: the trait `std::io::Read` is not implemented for the type `std::io::cursor::Cursor<&mut &[u8]>` [E0277]
lib.rs:63    capnp::serialize::read_message(&mut std::io::Cursor::new(&mut buffer), capnp::ReaderOptions::new())
             ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

I haven't quite figured out this part

You could then return a OwnedMessage<date::Reader> from foo().

I put the code in that you mentioned but can't quite figure out what the body of foo is supposed to be.

pub struct OwnedMessage<T> {
   phantom_data: ::std::marker::PhantomData<T>,
   message: ::capnp::OwnedSpaceMessageReader,
}

impl <'a, T> OwnedMessage <T> where T: ::capnp::traits::FromPointerReader<'a> {
   pub fn get(&'a self) -> ::capnp::Result<T> {
     use capnp::MessageReader;
     self.message.get_root()
   }
}

//fn foo(mut buffer: &[u8]) -> capnp::Result<capnp::OwnedSpaceMessageReader> {
fn foo(mut buffer: &[u8]) -> OwnedMessage<date::Reader> {
    // what goes here?
    //capnp::serialize::read_message(&mut buffer, capnp::ReaderOptions::new())
}

in the end, i'm hoping to be able to write something like

let my_date = foo(buf);
do_something_with_a_date(&my_date);

[dwrensha]

Does this work?

pub struct OwnedMessage<T> {
   phantom_data: ::std::marker::PhantomData<T>,
   message: ::capnp::OwnedSpaceMessageReader,
}

impl <T> OwnedMessage <T> {
    pub fn new(message: ::capnp::OwnedSpaceMessageReader) -> OwnedMessage<T> {
        OwnedMessage { phantom_data: ::std::marker::PhantomData, message: message}
    }
}

impl <'a, T> OwnedMessage <T> where T: ::capnp::traits::FromPointerReader<'a> {
   pub fn get(&'a self) -> ::capnp::Result<T> {
     use capnp::MessageReader;
     self.message.get_root()
   }
}

fn foo(mut buffer: &[u8]) -> ::capnp::Result<OwnedMessage<date::Reader>> {
    let message = try!(::capnp::serialize::read_message(&mut buffer, ::capnp::ReaderOptions::new()));
    return Ok(OwnedMessage::new(message));
}

[dwrensha]

Hm. I think there could be some lifetime issues with what I just suggested. Things will probably work out better if we forget about trying to make OwnedMessage generic.

pub struct OwnedDateMessage {
   message: ::capnp::OwnedSpaceMessageReader,
}

impl OwnedDateMessage {
   pub fn get<'a>(&'a self) -> ::capnp::Result<date::Reader<'a>> {
     use capnp::MessageReader;
     self.message.get_root()
   }
}

[dwrensha]

(I think my original suggestion with the generic OwnedMessage<T> needs higher-kinded types to work properly.)

[aij]

This is what I'm using now and it compiles/runs:

pub struct OwnedMessage<T> {
   message: ::capnp::OwnedSpaceMessageReader,
   phantom_data: ::std::marker::PhantomData<T>,
}

impl <'a, T> OwnedMessage <T> where T: ::capnp::traits::FromPointerReader<'a> {
    pub fn new(mr: ::capnp::OwnedSpaceMessageReader) -> OwnedMessage<T> {
        OwnedMessage {
            message: mr,
            phantom_data: ::std::marker::PhantomData
        }
    }
   pub fn get(&'a self) -> ::capnp::Result<T> {
     use capnp::MessageReader;
     self.message.get_root()
   }
}

fn read_indir(&self, bref: cafs_capnp::reference::block_ref::Reader) -> Result<OwnedMessage<cafs_capnp::indirect_block::Reader>> {
    let indir_bytes = try!(self.read_blockref_vec(bref));
    let mut cursor = io::Cursor::new(indir_bytes);
    let message_reader = try!(capnp::serialize_packed::read_message(&mut cursor, capnp::message::DEFAULT_READER_OPTIONS));
    Ok(OwnedMessage::new(message_reader))
}

Then I'm using it as let indir = try!(self.read_indir(bref)); let reader = try!(indir.get());

[waynenilsen]

Your suggestion worked, thanks! Now I just need to figure out how to write out to a &[u8]

let x = foo(buf).unwrap();
let y = x.get().unwrap(); 
println!("{:?}", y.get_day()); // yayy this works
let new_date = (something about a date builder i think?)
let out_buf &[u8] = new_date.what_is_the_name_of_this_function()    

[waynenilsen]

Thanks for all of your help, my final program is

#[repr(C)]
pub struct bytes_output {
    values : *const u8,
    len : usize,
}

#[no_mangle]
pub extern fn capnp_ex(external_data: *const u8, data_len : *const size_t) -> bytes_output {
    let buf : &[u8] = unsafe{ slice::from_raw_parts(external_data, data_len as usize) };
    let x = foo(buf).unwrap();
    let y = x.get().unwrap(); 
    println!("capnproto year read! {:?}", y.get_year());
    let mut message = MallocMessageBuilder::new_default();
    {
        let mut out_dt = message.init_root::<date::Builder>();
        out_dt.set_year(y.get_year() + 1);
        out_dt.set_month(1);
        out_dt.set_day(1);
    }
    let mut out_buf : Vec<u8> = Vec::new();
    capnp::serialize::write_message( &mut out_buf, &message );
    bytes_output {
        values : out_buf.as_ptr(),
        len : out_buf.len(), 
    }
}

and the python that calls it

import cffi
import capnp

#setup ffi
ffi = cffi.FFI()
ffi.cdef('''
     typedef struct {
        char* values;
        size_t len;
    } bytes_output;

    bytes_output capnp_ex(char*, size_t);
''')
lib = ffi.dlopen("./target/release/libtest.so")

# setup capnp
capnp.remove_import_hook()
example = capnp.load('schema/example.capnp')

mydate = example.Date.new_message(year=2015, month=5, day=31)
mydatebytes = mydate.to_bytes()
myresp = lib.capnp_ex(ffi.new('char[]', mydatebytes), len(mydatebytes))

def out_iter(resp):
    for i in range(resp.len):
        yield resp.values[i]

print(myresp) 

with open('tmp.bin','wb') as outf:
    for byte in out_iter(myresp):
        outf.write(byte) # must be a better way to do this..
with open('tmp.bin', 'rb') as inf:
    myrespdate = example.Date.read( inf )
    print(myrespdate)

for some reason the python capnp implementation relies on files quite a bit and I couldn't find a way around using files.