How can i convert avframe to ImageBitmap?(Compose for Desktop)

[habibg1232191]

Here is an example code:

@Composable
fun App() {
    val videoUrl by remember { mutableStateOf("<videoUrl>") }
    var imageBitmap by remember { mutableStateOf<ImageBitmap?>(null) }

    LaunchedEffect(Unit) {
        val pFormatContext = avformat_alloc_context()

        avformat_open_input(pFormatContext, videoUrl, null, null)
        avformat_find_stream_info(pFormatContext, null as PointerPointer<*>?)

        var pCodecParameters = AVCodecParameters(null)
        for (i in 0 until pFormatContext.nb_streams()) {
            val pLocalCodecParameters = pFormatContext.streams(i).codecpar()

            if(pLocalCodecParameters.codec_type() == avutil.AVMEDIA_TYPE_VIDEO) {
                pCodecParameters = pLocalCodecParameters
            }
        }

        val pCodec = avcodec_find_decoder(pCodecParameters.codec_id())
        val pCodecContext = avcodec_alloc_context3(pCodec)
        avcodec_parameters_to_context(pCodecContext, pCodecParameters)
        avcodec_open2(pCodecContext, pCodec, null as PointerPointer<*>?)

        val pPacket = av_packet_alloc()
        val pFrame = avutil.av_frame_alloc()

        while(av_read_frame(pFormatContext, pPacket) >= 0) {
            avcodec_send_packet(pCodecContext, pPacket)
            avcodec_receive_frame(pCodecContext, pFrame)

            val buff = pPacket.data().position(0).capacity(pPacket.size().toLong()).asByteBuffer()
            val arr = ByteArray(buff.capacity())
            buff.get(arr)
            imageBitmap = Image.makeFromEncoded(arr).toComposeImageBitmap()
            delay(500)
        }
    }

    MaterialTheme {
        imageBitmap?.let {
            Image(
                bitmap = it,
                contentDescription = "Video"
            )
        }
    }
}

But it doesn't work

[saudet]

For a working example of reading data out of AVFrame, please take a look at FFmpegFrameGrabber: https://github.com/bytedeco/javacv/blob/master/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java

[habibg1232191]

I tried as you said, but the weight still does not work. Skia can't encode:

Loader.load(avutil::class.java)
        Loader.load(swresample::class.java)
        Loader.load(avcodec::class.java)
        Loader.load(avformat::class.java)
        Loader.load(swscale::class.java)

        // Register all formats and codecs
        avcodec.av_jni_set_java_vm(Loader.getJavaVM(), null)
        avcodec.avcodec_register_all()
        avformat.av_register_all()
        avformat.avformat_network_init()

        Loader.load(avdevice::class.java)
        avdevice.avdevice_register_all()

        val pFormatContext = avformat.avformat_alloc_context()

        avformat.avformat_open_input(pFormatContext, videoUrl, null, null)
        avformat.avformat_find_stream_info(pFormatContext, null as PointerPointer<*>?)

        var pCodecParameters = AVCodecParameters(null)
        var videoStream = -1
        for (i in 0 until pFormatContext.nb_streams()) {
            val pLocalCodecParameters = pFormatContext.streams(i).codecpar()

            if(pLocalCodecParameters.codec_type() == avutil.AVMEDIA_TYPE_VIDEO) {
                pCodecParameters = pLocalCodecParameters
                videoStream = i
            }
        }

        val pCodec = avcodec.avcodec_find_decoder(pCodecParameters.codec_id())
        val pCodecContext = avcodec.avcodec_alloc_context3(pCodec)
        avcodec.avcodec_parameters_to_context(pCodecContext, pCodecParameters)
        avcodec.avcodec_open2(pCodecContext, pCodec, null as PointerPointer<*>?)

        val pPacket = avcodec.av_packet_alloc()
        val pFrame = avutil.av_frame_alloc()
        val pFrameRgb = avutil.av_frame_alloc()

        var imgConvertCtx = SwsContext(null)

        imgConvertCtx = swscale.sws_getCachedContext(
            imgConvertCtx,
            pCodecContext.width(), pCodecContext.height(), pCodecContext.pix_fmt(),
            pCodecContext.width(), pCodecContext.height(), pCodecContext.pix_fmt(),
            swscale.SWS_BILINEAR, null, null, null as DoublePointer?
        )

        while(avformat.av_read_frame(pFormatContext, pPacket) >= 0) {
            avcodec.avcodec_send_packet(pCodecContext, pPacket)
            avcodec.avcodec_receive_frame(pCodecContext, pFrame)

            if(pPacket.stream_index() == videoStream) {
                val fmt = pCodecContext.pix_fmt()
                val height = pCodecContext.height()
                val width = pCodecContext.width()

                // work around bug in swscale: https://trac.ffmpeg.org/ticket/1031
                val align = 32
                var stride = width
                var i = 1
                while (i <= align) {
                    stride = width + (i - 1) and (i - 1).inv()
                    avutil.av_image_fill_linesizes(pFrameRgb.linesize(), fmt, stride)
                    if (pFrameRgb.linesize(0) and align - 1 == 0) {
                        break
                    }
                    i += i
                }

                val size = avutil.av_image_get_buffer_size(fmt, stride, height, 1)
                val image_ptr = arrayOf(BytePointer(avutil.av_malloc(size.toLong())).capacity(size.toLong()))
                val image_buf = arrayOf<Buffer>(image_ptr[0].asByteBuffer())

                pFrameRgb.format(fmt)
                pFrameRgb.width(width)
                pFrameRgb.height(height)

                swscale.sws_scale(
                    imgConvertCtx, PointerPointer<AVFrame?>(pFrame), pFrame.linesize(), 0,
                    pCodecContext.height(), PointerPointer<AVFrame?>(pFrameRgb), pFrameRgb.linesize()
                )

                val buff = image_ptr[0].asByteBuffer()
                if(buff != null) {
                    val arr = ByteArray(buff.capacity())
                    buff.get(arr)
                    imageBitmap = Image.makeFromEncoded(arr).toComposeImageBitmap()
                    println("Success, Buff: $image_buf")
                    delay(500)
                }
            }
        }

First time I'm writing something like this. Could you help me?

[Mett-Barr]

Maybe this is a bit late, and it's converting from a Frame to an ImageBitmap, not using an AVFrame as the data source. But it worked for me, and I hope it helps you.

fun extractPixelsFromFrame(frame: Frame): ByteArray {
    val width: Int
    val height: Int
    val byteBuffer: ByteBuffer
    try {
        width = frame.imageWidth
        height = frame.imageHeight
        byteBuffer = frame.image[0] as? ByteBuffer ?: return ByteArray(0)
    } catch (e: Exception) {
        e.printStackTrace()
        return ByteArray(0)
    }

    byteBuffer.rewind()

    val pixelData = ByteArray(width * height * 4) // 4 bytes per pixel (RGBA)
    var index = 0

    while (byteBuffer.hasRemaining()) {
        // Assuming the frame is in BGR format, we convert to RGBA
        val b = byteBuffer.get().toInt() and 0xFF
        val g = byteBuffer.get().toInt() and 0xFF
        val r = byteBuffer.get().toInt() and 0xFF

        pixelData[index++] = r.toByte()
        pixelData[index++] = g.toByte()
        pixelData[index++] = b.toByte()
        pixelData[index++] = 255.toByte() // Full opacity
    }

    return pixelData
}

fun createBitmapFromPixels(width: Int, height: Int, pixelData: ByteArray): Bitmap {
    val bitmap = Bitmap()
    val imageInfo = ImageInfo(width, height, ColorType.RGBA_8888, ColorAlphaType.PREMUL)

    val rowBytes = width * 4 // 4 bytes per pixel for RGBA

    if (!bitmap.installPixels(imageInfo, pixelData, rowBytes)) {
        throw IllegalArgumentException("Failed to install pixels in the bitmap.")
    }

    return bitmap
}

fun frameToImageBitmap(frame: Frame): ImageBitmap {
    val byteArray = extractPixelsFromFrame(frame)
    val bitmap = createBitmapFromPixels(frame.imageWidth, frame.imageHeight, byteArray)
    return bitmap.asComposeImageBitmap()
}