Python binding to deal with generated stream objects

[uros-skopus]

I have created a pre-processed stream from a video using utils/video-viewer, and then test it by replaying it successfully with utils/led-image-viewer.

Since I am working with python bindings now I need to read and display the stream using my own modification of bindings/python/samples/image-viewer.py but I was not able to find appropriate python methods. Is it even supported with python bindings?

[hzeller]

You'd have to contribute some pull-request that adds that binding.

In the simplest case, you just reimplement the streaming classes (StreamWriter, StreamIO...) in Python, then you only need the c-Python binding for FrameCanvas::Serialize and Deserialize.

[uros-skopus]

@hzeller Thanks for quick response as usual! I re-implemented streaming part, but I am not familiar at all with c-Python bindings so implementation of that one goes really slow, and up to now without any success. Is there any open/closed issue here you can point me to get some guidelines? How time consuming you really find it?

[uros-skopus]

I have managed to put together even the c-binding part, but got Segmentation error trying to Deserialize stream in the end (c-binding was successfully built). The code looks like:

core.pyx extended with:

cdef class FrameCanvas(Canvas):
...     
    def Serialize(self, const char **data, size_t *len):
        (<cppinc.FrameCanvas*>self.__getCanvas()).Serialize()

    def Deserialize(self, const char *data, size_t len):
        (<cppinc.FrameCanvas*>self.__getCanvas()).Deserialize()

cppinc.pxd extended with:

    cdef cppclass FrameCanvas(Canvas):
        ...
    void Serialize(const char **, size_t *)
    bool Deserialize(const char *, size_t)

core.cpp extended with:

/* 
 * "rgbmatrix/core.pyx":78
 * def Deserialize(self):(<cppinc.FrameCanvas*>self.__getCanvas()).Deserialize()             # <<<<<<<<<<<<<<
 *     
 */
/* Python wrapper */
static bool __pyx_pf_9rgbmatrix_4core_11FrameCanvas_15Deserialize(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
static bool __pyx_pf_9rgbmatrix_4core_11FrameCanvas_15Deserialize(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
  PyObject *__pyx_v_data;
  PyObject *__pyx_v_len;
  bool __pyx_r;
  __Pyx_RefNannyDeclarations
  __Pyx_RefNannySetupContext("Deserialize (wrapper)", 0);
  {
    static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_data,&__pyx_n_s_len,0};
    PyObject* values[2] = {0,0};
    if (unlikely(__pyx_kwds)) {
      Py_ssize_t kw_args;
      const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
      switch (pos_args) {
        case  2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
        CYTHON_FALLTHROUGH;
        case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
        CYTHON_FALLTHROUGH;
        case  0: break;
        default: goto __pyx_L5_argtuple_error;
      }
      kw_args = PyDict_Size(__pyx_kwds);
      switch (pos_args) {
        case  0:
        if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_data)) != 0)) kw_args--;
        else goto __pyx_L5_argtuple_error;
        CYTHON_FALLTHROUGH;
        case  1:
        if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_len)) != 0)) kw_args--;
        else {
          __Pyx_RaiseArgtupleInvalid("Deserialize", 1, 2, 2, 1); __PYX_ERR(0, 78, __pyx_L3_error)
        }
        CYTHON_FALLTHROUGH;
      }
      if (unlikely(kw_args > 0)) {
        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "Deserialize") < 0)) __PYX_ERR(0, 78, __pyx_L3_error)
      }
    } else if (PyTuple_GET_SIZE(__pyx_args) != 2) {
      goto __pyx_L5_argtuple_error;
    } else {
      values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
      values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
    }

    __pyx_v_data = values[0];
    __pyx_v_len = values[1];
  }
  goto __pyx_L4_argument_unpacking_done;
  __pyx_L5_argtuple_error:;
  __Pyx_RaiseArgtupleInvalid("Deserialize", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 78, __pyx_L3_error)
  __pyx_L3_error:;
  __Pyx_AddTraceback("rgbmatrix.core.FrameCanvas.Deserialize", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __Pyx_RefNannyFinishContext();
  return 0;
  __pyx_L4_argument_unpacking_done:;
  __pyx_r = __pyx_pf_9rgbmatrix_4core_11FrameCanvas_14Deserialize(((struct __pyx_obj_9rgbmatrix_4core_FrameCanvas *)__pyx_v_self), __pyx_v_data, __pyx_v_len);

  /* function exit code */
  __Pyx_RefNannyFinishContext();

  printf("Response %d!\n", __pyx_r);

  return __pyx_r;
}

static bool __pyx_pf_9rgbmatrix_4core_11FrameCanvas_14Deserialize(struct __pyx_obj_9rgbmatrix_4core_FrameCanvas *__pyx_v_self, PyObject *__pyx_v_data, PyObject *__pyx_v_len) {
  bool __pyx_r;
  __Pyx_RefNannyDeclarations
  rgb_matrix::Canvas *__pyx_t_1;
  const char *__pyx_t_2;
  size_t __pyx_t_3;
  __Pyx_RefNannySetupContext("Deserialize", 0);

  /* "rgbmatrix/core.pyx":70
 * 
 *     def Deserialize(self, const char *data, size_t len):
 *       (<cppinc.FrameCanvas*>self.__getCanvas()).Deserialize()            # <<<<<<<<<<<<<<
 * 
 *     
 */

  __pyx_t_1 = ((struct __pyx_vtabstruct_9rgbmatrix_4core_FrameCanvas *)__pyx_v_self->__pyx_base.__pyx_vtab)->__pyx___getCanvas(__pyx_v_self); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 84, __pyx_L1_error)
  __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_v_data); if (unlikely((__pyx_t_2 == (const char *)-1) && PyErr_Occurred())) __PYX_ERR(0, 78, __pyx_L1_error)
  __pyx_t_3 = __Pyx_PyInt_As_size_t(__pyx_v_len); if (unlikely((__pyx_t_3 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 78, __pyx_L1_error)
  __pyx_r = ((rgb_matrix::FrameCanvas *)__pyx_t_1)->Deserialize(__pyx_t_2, __pyx_t_3);

  /* 
 * 
 *     def Deserialize(self, const char *data, size_t len):
 *       (<cppinc.FrameCanvas*>self.__getCanvas()).Deserialize()            # <<<<<<<<<<<<<<
 * 
 */

    /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_AddTraceback("rgbmatrix.core.FrameCanvas.Deserialize", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

Printing the response (printf("Response %d!\n", __pyx_r);) I could see it was done properly (it is printing value 1), but at the end I got Segmentation fault. I don't know what I am missing here neither how to debug it more detailed.

The way I used it:

   try:
           fd = open(image_path, "rb")
   except IOError:
           print ("Could not open file! Image path is wrong")

   file_info = FileInfo()
   file_info.content_stream = FileStreamIO(fd)
   reader = StreamReader(file_info.content_stream)

   if reader.get_next(self._canvas, None):
          file_info.is_multi_frame = reader.get_next(self._canvas, None)
          reader.rewind()
    else:
          err_msg = "Can't read as image or compatible stream"
          file_info.content_stream = None
          file_info = None

Class StreamReader:

class StreamReader:
    # internal STATES:
    STREAM_AT_BEGIN = 0
    STREAM_READING = 1

    def __init__(self, io):
        self._io = io
        self._state = StreamReader.STREAM_AT_BEGIN
        self._buffer = None
        self._buf_size = 0
        self._io.rewind()

    def rewind(self):
        self._io.rewind()
        self._state = StreamReader.STREAM_AT_BEGIN

    def get_next(self, frame_canvas, hold_time_us):
        if self._state == StreamReader.STREAM_AT_BEGIN and (not self.read_file_header(frame_canvas)):
            return False
        if self._state != StreamReader.STREAM_READING:
            return False

        h = struct.pack("IIIIQQ", FrameHeader.kFrameMagicValue, 0, 0, 0, 0, 0)
        read_buffer = self._io.read(struct.calcsize("IIIIQQ"))
        h = struct.unpack("IIIIQQ", read_buffer)

        # TODO: we might allow for this to be a kFileMagicValue, to allow people
        # to just concatenate streams. In that case, we just would need to read
        # ahead past this header (both headers are designed to be same size)
        if h[0] != FrameHeader.kFrameMagicValue:
            self._state = StreamReader.STREAM_ERROR
            return False

        # In the future, we might allow larger buffers (audio?), but never smaller.
        if h[1] < self._buf_size:
            return False

        if hold_time_us:
            hold_time_us = h.hold_time_us

        self._buffer = self._io.read(self._buf_size)
        result = frame_canvas.Deserialize(self._buffer, self._buf_size)
        if result == 1:
            return True
        return False

    def read_file_header(self, frame_canvas):
        header = struct.pack("IIIIQQ", FileHeader.kFileMagicValue, 0, 0, 0, 0, 0)
        read_buffer = self._io.read(struct.calcsize("IIIIQQ"))
        header = struct.unpack("IIIIQQ", read_buffer)

        if header[0] != FileHeader.kFileMagicValue:
            self._state = StreamReader.STREAM_ERROR
            return False

        if header[2] != frame_canvas.width or header[3] != frame_canvas.height:
            print("This stream is for %dx%d, can't play on %dx%d. "
                            "Please use the same settings for record/replay\n",
                    header[2], header[3], frame_canvas.width, frame_canvas.height)
            self._state = StreamReader.STREAM_ERROR
            return False

        self._state = StreamReader.STREAM_READING
        self._buf_size = header[1]
        print('Buffer size is: ', self._buf_size)
        if not self._buffer:
            self._buffer = np.chararray(header[1])
        return True

Class FileStreamIO:

class FileStreamIO:

    def __init__(self, fd):
        self._fd = fd

    def close(self):
        os.close(self._fd)

    def rewind(self):
        self._fd.seek(0, 0)

    def read(self, count):
        ret = self._fd.read(count)
        return ret

    def append(self, count):
        return os.append(self._fd, count)

@hzeller Any suggestions please?

[hzeller]

Segmentation fault is memory access issue; given that you are dealing with raw memory pointers, this could happen of course.

Is the call to Deserialize() giving you the trouble ? The stream you are reading, does it pass all the other tests, like the magic values and stuff ? (I suspect so if Deserialize() generates trouble. How did you allocate the FrameCanvas you call Deserialize on, you don't include that part of your code. It should come from a CreateFrameCanvas() call on the rgb-matrix.

(BTW, I'd first start with just creating the c-binding for the Deserialize(). The Serialize() is returning a raw c-pointer which then should be used to copy things into, so probably not something you'd want to do from Python wihtout some additional padding calls. But Deserialize is the most useful for you.)

[uros-skopus]

Yes, you are right for each of your comments:

1. I started only with Deserialize
2. Invocation of Deserialize is giving me the problem
3. I have successfully extracted the Header data and checked it, see:

header = struct.pack("IIIIQQ", FileHeader.kFileMagicValue, 0, 0, 0, 0, 0)
read_buffer = self._io.read(struct.calcsize("IIIIQQ"))
header = struct.unpack("IIIIQQ", read_buffer)
if header[0] != FileHeader.kFileMagicValue:
    self._state = StreamReader.STREAM_ERROR
    return False
if header[2] != frame_canvas.width or header[3] != frame_canvas.height:
    print("This stream is for %dx%d, can't play on %dx%d. "
             "Please use the same settings for record/replay\n",
              header[2], header[3], frame_canvas.width, frame_canvas.height)
    self._state = StreamReader.STREAM_ERROR
    return False

4. Allocation of frame canvas is done like you said: self._canvas = self._matrix.CreateFrameCanvas()

The weird thing is when I return 0 instead of 1 (just like the error appeared) there is no Segmentation fault, but of course stream is not displayed. Message I got is: Issue: <built-in method Deserialize of rgbmatrix.core.FrameCanvas object at 0x6bba6b70> returned NULL without setting an error. The code snippet with a single line added (__pyx_r = 0;):

...
__pyx_r = ((rgb_matrix::FrameCanvas *)__pyx_t_1)->Deserialize(__pyx_t_2, __pyx_t_3);
__pyx_r = 0; (ADDED THIS LINE TO RETURN 0)
 /* function exit code */
goto __pyx_L0;
__pyx_L1_error:;
__Pyx_AddTraceback("rgbmatrix.core.FrameCanvas.Deserialize", __pyx_clineno, __pyx_lineno, __pyx_filename);
__pyx_r = 0;
__pyx_L0:;
__Pyx_RefNannyFinishContext();
return __pyx_r; 
...

[uros-skopus]

I have done some additional testing.

1. As stated above - FileHeader struct is successfully extracted and checked. buf_size is 180224. I have used python bytearray to represent buffers.

2. Then a first FrameHeader is read successfully and checked with:

       if frame[0] != FrameHeader.kFrameMagicValue:
           self._state = StreamReader.STREAM_ERROR
           return False
   
       # In the future, we might allow larger buffers (audio?), but never smaller.
       if frame[1] < self._buf_size:
           return False

3. Then the first frame of size (180224) is successfully read. The value of the last byte was 12 (int).

4. Printouts of action sequences and current stream position: File size: 25956896 # CORRECT -- READING OF FILE HEADER StreamUtils.full_read.number_of_read_bytes: 32 Buffer size is: 180224 Stream reader buffer initialized as byte array with the size of (bytes): 180224 -- READING OF FRAME HEADER Current position 1: 32 # current position of IO stream object from StreamReader StreamUtils.full_read.number_of_read_bytes: 32 -- READING OF THE FIRST FRAME Current position 2: 64 StreamUtils.full_read.number_of_read_bytes: 180224 Current position 3: 180288 -- BEFORE DESERIALIZE METHOD INVOCATION Last buffer character before deserialize: 12 Buffer size BEFORE deserialize: 180224

5. Printouts from the cython core.cpp (within Deserialization method) Size of bytearray 180224! Last character 12 Segmentation fault

The method:

static bool __pyx_pf_9rgbmatrix_4core_11FrameCanvas_14Deserialize(struct __pyx_obj_9rgbmatrix_4core_FrameCanvas *__pyx_v_self, PyObject *__pyx_v_data, PyObject *__pyx_v_len) {
  bool __pyx_r;
  __Pyx_RefNannyDeclarations
  rgb_matrix::Canvas *__pyx_t_1;
  const char *__pyx_t_2;
  size_t __pyx_t_3;
  __Pyx_RefNannySetupContext("Deserialize", 0);

 /* "rgbmatrix/core.pyx":70
 * 
 *     def Deserialize(self, const char *data, size_t len):
 *       (<cppinc.FrameCanvas*>self.__getCanvas()).Deserialize()            # <<<<<<<<<<<<<<
 * 
 *     
 */

  __pyx_t_1 = ((struct __pyx_vtabstruct_9rgbmatrix_4core_FrameCanvas *)__pyx_v_self->__pyx_base.__pyx_vtab)->__pyx___getCanvas(__pyx_v_self); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 84, __pyx_L1_error)
  __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_v_data); if (unlikely((__pyx_t_2 == (const char *)-1) && PyErr_Occurred())) __PYX_ERR(0, 78, __pyx_L1_error)
  __pyx_t_3 = __Pyx_PyInt_As_size_t(__pyx_v_len); if (unlikely((__pyx_t_3 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 78, __pyx_L1_error)
  __pyx_r = ((rgb_matrix::FrameCanvas *)__pyx_t_1)->Deserialize(__pyx_t_2, __pyx_t_3);

  printf("Size of bytearray %d!\n", __pyx_t_3);
  printf("Last character %d\n", __pyx_t_2[__pyx_t_3 - 1]);
  printf("Return %d!\n", __pyx_r);

 /* 
 * 
 *     def Deserialize(self, const char *data, size_t len):
 *       (<cppinc.FrameCanvas*>self.__getCanvas()).Deserialize(data, len)            # <<<<<<<<<<<<<<
 * 
 */
    /* function exit code */
  goto __pyx_L0;
  __pyx_L1_error:;
  __Pyx_AddTraceback("rgbmatrix.core.FrameCanvas.Deserialize", __pyx_clineno, __pyx_lineno, __pyx_filename);
  __pyx_r = 0;
  __pyx_L0:;
  __Pyx_RefNannyFinishContext();
  return __pyx_r;
}

So the correct size and bytearray were sent to C code, but still a segmentation error occurs.

Yes, segmentation fault is a synonym for memory issue, but I cannot locate it here. Both bytearray and size are properly forwarded up to the C code invocation of Deserialized method. Am I missing something within core.pyx file? core.pyx FrameCanvas part was just extended as:

cdef class FrameCanvas(Canvas):
    def __dealloc__(self):
        if <void*>self.__canvas != NULL:
            self.__canvas = NULL

    cdef cppinc.Canvas* __getCanvas(self) except *:
        if <void*>self.__canvas != NULL:
            return self.__canvas
        raise Exception("Canvas was destroyed or not initialized, you cannot use this object anymore")

    def Fill(self, uint8_t red, uint8_t green, uint8_t blue):
        (<cppinc.FrameCanvas*>self.__getCanvas()).Fill(red, green, blue)

    def Clear(self):
        (<cppinc.FrameCanvas*>self.__getCanvas()).Clear()

    def SetPixel(self, int x, int y, uint8_t red, uint8_t green, uint8_t blue):
        (<cppinc.FrameCanvas*>self.__getCanvas()).SetPixel(x, y, red, green, blue)

    def Serialize(self, const char **data, size_t *len):
        (<cppinc.FrameCanvas*>self.__getCanvas()).Serialize(data, len)

    def Deserialize(self, const char *data, size_t len):
        (<cppinc.FrameCanvas*>self.__getCanvas()).Deserialize(data, len)

@hzeller Any ideas? Am I still missing something? I am pretty stuck...

[uros-skopus]

@Saij I've found some very useful hints from you regarding similar issues. Can you please check this one, maybe you can help?