Using FfmpegCaptureCommand will cause Memory leak in Linux build

[se7enXF]

I use FfplayCommand and FfmpegCaptureCommand as follow:

using FfmpegUnity;
using System;
using UnityEngine;
using System.IO;
using OwnRequestStruct;

public class FFmpegManager : MonoBehaviour
{
    GameObject captureObject;
    FfmpegCaptureCommand capture;
    FfplayCommand videoPlayer;
    FfmpegCommand videoConverter;

    private void Awake()
    {

        // 初始化Ffmpeg for unity，用于音视频录制
        captureObject = new GameObject("FFmpeg");

        // 使用Ffmpeg for unity播放视频
        videoPlayer = captureObject.AddComponent<FfplayCommand>();
        videoPlayer.ExecuteOnStart = false;
        videoPlayer.DefaultPath = FfmpegPath.DefaultPath.NONE;
        var videoTexture = gameObject.AddComponent<FfmpegPlayerVideoTexture>();
        videoTexture.VideoTexture = GetComponent<InitBaseScense>().imageShow.texture;
        videoPlayer.VideoTexture = videoTexture;
        var audioSourceVideo = captureObject.AddComponent<AudioSource>();
        videoPlayer.AudioSourceComponent = audioSourceVideo;

        // 使用Ffmpeg for unity调整音轨时间
        videoConverter = captureObject.AddComponent<FfmpegCommand>();
        videoConverter.ExecuteOnStart = false;
    }

    #region FFmpeg相关函数
    /// <summary>
    /// 开始录制，在这里初始化录制脚本，避免修改分辨率以后画面错位问题。
    /// 以“.mp4”结尾录制为文件，否则推流，流地址为rtmp://ip:1935/live/{savePath}
    /// </summary>
    /// <param name="savePath"></param>
    public void StartRecord(string savePath, RequestStruct req)
    {
        // 设置相关参数
        capture = captureObject.AddComponent<FfmpegCaptureCommand>();   // 视频录制脚本
        capture.ExecuteOnStart = false;
        capture.req = req;
        capture.callback = GetComponent<Callback>();
        var video = capture.CaptureSources[0];
        video.Type = FfmpegCaptureCommand.CaptureSource.SourceType.Video_Camera;
        video.SourceCamera = Camera.main;
        video.Width = Screen.width;
        video.Height = Screen.height;
        video.FrameRate = 30;

        // 录制为文件或推流
        string recordArgs;
        string msg;

        var ext = Path.GetExtension(savePath);
        if (ext == "")
        {
            string rtmp = $"rtmp://127.0.0.1:1935/live/{savePath}";
            recordArgs = $"-c:v libopenh264 -c:a aac -map 0:v:0 -map 1:a:0 -g 30 -vsync 1 -async 96000 -f flv {rtmp}";
            msg = $"Vidoe will stream to: {rtmp}";
        }
        else
        {
            if (ext == ".webm") recordArgs = $"-c:v libvpx-vp9 -c:a libvorbis -map 0:v:0 -map 1:a:0 -crf 30 -b:v 0 -deadline realtime -cpu-used 8 {savePath}";
            else if (ext == ".mp4") recordArgs = $"-c:v libopenh264 -c:a aac -map 0:v:0 -map 1:a:0 {savePath}";
            else throw new NotImplementedException($"Recording video to {ext} is unsupport");
            msg = $"Vidoe will save to: {savePath}";
        }

        // 开始FFmpeg录制线程
        capture.CaptureOptions = recordArgs;
        capture.StartFfmpeg();
        Debug.Log($" [ - FFmpeg - ] Start recording. Res: {video.Width} x {video.Height}. {msg}");
    }

    public void StopRecord()
    {
        // 停止FFmpeg线程
        if (capture.IsRunning)
        {
            capture.StopFfmpeg();
            Destroy(capture);
            Destroy(Camera.main.GetComponent<FfmpegCaptureCamera>());
            Debug.Log(" [ - FFmpeg - ] Stop recording.");
        }
    }

    public enum VideoExt
    {
        mp4,
        avi,
        flv,
        m4v,
        mkv,
        mov,
        webm,
        wmv
    }

    /// <summary>
    /// 播放视频文件，经过测试，支持的后缀为：mp4,avi,flv,m4v,mkv,mov,webm,wmv
    /// </summary>
    /// <param name="videoPath"></param>
    /// <param name="volume"></param>
    /// <exception cref="Exception"></exception>
    public void PlayVideoFile(string videoPath, float volume=1.0f)
    {
        if (!File.Exists(videoPath))
            throw new Exception("Video file not exist: " + videoPath);
        GetComponent<AudioPlayer>().StopPlay(); // 关闭音频
        if (videoPlayer.IsRunning)
        {
            videoPlayer.Dispose();
        }
        videoPlayer.InputPath = videoPath;
        videoPlayer.Volume = volume;
        videoPlayer.Play();
        GetComponent<InitBaseScense>().canvasShow.enabled = true;
        Debug.Log($" [ - FFmpeg - ] Start playing: {videoPath}. volume={volume}");
    }

    public void StopPlay()
    {
        if (videoPlayer.IsRunning)
        {
            videoPlayer.Dispose();
            GetComponent<InitBaseScense>().canvasShow.enabled = false;
            Debug.Log(" [ - FFmpeg - ] Stop playing");
        }
    }

    private void Update()
    {
        if (!videoPlayer.IsRunning)
            return;

        // 视频播放完毕时IsRunning=true,Paused=true
        if (videoPlayer.Paused)
        {
            StopPlay();
        }
    }

    public void Convert(string inputPath, string outputPath, float audioDelaySec)
    {
        videoConverter.Options = string.Join(
          "\n",
          "-y",
          $"-i {inputPath}",
          $"-itsoffset {audioDelaySec}",
          $"-i {inputPath}",
          "-map 0:v",
          "-map 1:a",
          "-c copy",
          outputPath
          );
        videoConverter.ExecuteFfmpeg();
        Debug.Log($"Convert {inputPath} to {outputPath} by delaying audio {audioDelaySec} s");
    }
#endregion
}

In windows editor, if you call PlayVideoFile() and StopPlay() or StartRecord() and StopRecord() many times, there is no memory leak.
In linux build, invoking 10 times StartRecord(), PlayVideoFile(), StopPlay() and StopPlay() will result in about 1GB of memory leakage. Here is the error code. Onec the issue occurres, the errors will loop to print many times :

OutOfMemoryException: Out of memory
  at (wrapper managed-to-native) System.Object.__icall_wrapper_ves_icall_array_new_specific(intptr,int)
  at System.Collections.Generic.List`1[T].set_Capacity (System.Int32 value) [0x00021] in <0bfb382d99114c52bcae2561abca6423>:0
  at System.Collections.Generic.List`1[T].EnsureCapacity (System.Int32 min) [0x00036] in <0bfb382d99114c52bcae2561abca6423>:0
  at System.Collections.Generic.List`1[T].InsertRange (System.Int32 index, System.Collections.Generic.IEnumerable`1[T] collection) [0x00032] in <0bfb382d99114c52bcae2561abca6423>:0
  at System.Collections.Generic.List`1[T].AddRange (System.Collections.Generic.IEnumerable`1[T] collection) [0x00000] in <0bfb382d99114c52bcae2561abca6423>:0
  at FfmpegUnity.FfmpegCaptureCommand.OnAudioFilterWriteToCaptureAudio (System.Single[] data, System.Int32 channels, System.Int32 streamId) [0x000b6] in <9a6ccb49780e4cc8976535d3f6ffa12c>:0
  at FfmpegUnity.FfmpegCaptureAudio.OnAudioFilterRead (System.Single[] data, System.Int32 channels) [0x0000e] in <9a6ccb49780e4cc8976535d3f6ffa12c>:0

[NON906]

It's probably because ffmpeg started before it finished processing and it's still there. Make sure it is stopped with FfmpegCommand.IsRunning or FfmpegCommand.IsFinished before trying to start it.

[se7enXF]

@NON906

We tested v2.1, v2.7, v2.9 of FFmpeg for Unity. The test result is that only the v2.1 does not have Memory leak. The current situation is that higher versions not only do not release memory after recording, but also occupy a high CPU usage, forever. V2.1 will release memory and cpu after recording.

[NON906]

How did you check? I want to check it here as well.

[se7enXF]

@NON906

Using my script and create two button on UI, one for StartRecord and another for StopRecord. Just run Linux build on Ubuntu20.04, using top to monitor memory and CPU. You can first click StartRecord, wait for seconds, then click StopRecord. Loop the action and watch the changes in memory usage.

[NON906]

I checked %CPU and %MEM with the top command, but they did not remain after stopping.

[se7enXF]

@NON906

Which version? I need to check again.

[NON906]

Unity 2021.3.16f1 FFmpeg for Unity 2.9 Ubuntu 20.04.6 LTS

Confirmed with Editor and IL2CPP build.

[se7enXF]

@NON906 I tested v2.7 and v2.9 with IL2CPP build. v2.7 has no memory leak but v2.9 has minor memory leaks. What's more, I have other plugins only support Mono build in my project, so IL2CPP is not working for me. I am now working on combining v2.1 with #2 to avoid segmentation fault. If you can solve the memory leak problem in Mono build, that would be very grateful!