Output video is horizontally flipped on emulator

[troy-lamerton]

On emulator (Bluestacks) the output video is flipped: https://www.twitch.tv/videos/533743233

In the emulator the app looks correct.

Since OpenGlView renders everything, I try openGlView.setCameraFlip(true, false). Now video is correct: https://www.twitch.tv/videos/533748141 But also the app is made incorrect! It has flipped horizontally too. So now it looks flipped for the user, but good for the rtmp viewers.

So how do I flip only the output video? I have been looking here: https://github.com/pedroSG94/rtmp-rtsp-stream-client-java/blob/3c096e74c2852c4e952d8a02d3acd9dbbeba5af7/rtplibrary/src/main/java/com/pedro/rtplibrary/view/OpenGlView.java#L141 Will it need changes to opengl matrix, or is there another solution?

[pedroSG94]

Yes, you are right you need modify that line. You need know when you are drawing preview or output video for it. Preview: https://github.com/pedroSG94/rtmp-rtsp-stream-client-java/blob/master/rtplibrary/src/main/java/com/pedro/rtplibrary/view/OpenGlView.java#L128 Output video: https://github.com/pedroSG94/rtmp-rtsp-stream-client-java/blob/master/rtplibrary/src/main/java/com/pedro/rtplibrary/view/OpenGlView.java#L138 Now you should create 2 matrix to modify it. Try follow this post: https://github.com/pedroSG94/rtmp-rtsp-stream-client-java/issues/452

This is the classes that you need modify (You can see the differencies with my original classes): https://github.com/Radcat501/rtmp-rtsp-stream-client-java/blob/master/encoder/src/main/java/com/pedro/encoder/input/gl/render/ManagerRender.java#L68 https://github.com/Radcat501/rtmp-rtsp-stream-client-java/blob/master/encoder/src/main/java/com/pedro/encoder/input/gl/render/ScreenRender.java

Let me know if you have problems to implement it.

[troy-lamerton]

add setFlip method to ScreenRender https://github.com/thesingularitygroup/rtmp-rtsp-stream-client-java/commit/a261232ffdef9b31e1e928a5576a1e25e09bd721

add method setOutputFlip to OpenGlView which will flip only the output video https://github.com/thesingularitygroup/rtmp-rtsp-stream-client-java/commit/36c9765c50c7cb2b600a33a4639592be41a8ffd3

Just adding this code made the screen totally black. I didn't make my code call setOutputFlip yet. Do the repeated calls to managerRender.setScreenFlip screw up the matrix? Matrix code was copied from CameraRender.java.

[pedroSG94]

Replace: https://github.com/pedroSG94/rtmp-rtsp-stream-client-java/blob/master/encoder/src/main/java/com/pedro/encoder/input/gl/render/ScreenRender.java To:

package com.pedro.encoder.input.gl.render;

import android.content.Context;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.os.Build;

import androidx.annotation.RequiresApi;
import com.pedro.encoder.R;
import com.pedro.encoder.utils.gl.GlUtil;
import com.pedro.encoder.utils.gl.PreviewSizeCalculator;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.util.Arrays;

/**
 * Created by pedro on 29/01/18.
 */

@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
public class ScreenRender {

  //rotation matrix
  private final float[] squareVertexData = {
      // X, Y, Z, U, V
      -1f, -1f, 0f, 0f, 0f, //bottom left
      1f, -1f, 0f, 1f, 0f, //bottom right
      -1f, 1f, 0f, 0f, 1f, //top left
      1f, 1f, 0f, 1f, 1f, //top right
  };

  private final float[] rotatedVertexData = Arrays.copyOf(squareVertexData, squareVertexData.length);

  private FloatBuffer squareVertex;

  private float[] MVPMatrix = new float[16];
  private float[] STMatrix = new float[16];
  private boolean AAEnabled = false;  //FXAA enable/disable

  private int texId;

  private int program = -1;
  private int uMVPMatrixHandle = -1;
  private int uSTMatrixHandle = -1;
  private int aPositionHandle = -1;
  private int aTextureHandle = -1;
  private int uSamplerHandle = -1;
  private int uResolutionHandle = -1;
  private int uAAEnabledHandle = -1;

  private int streamWidth;
  private int streamHeight;

  public ScreenRender() {
    // Initialization
    squareVertex =
        ByteBuffer.allocateDirect(squareVertexData.length * BaseRenderOffScreen.FLOAT_SIZE_BYTES)
            .order(ByteOrder.nativeOrder())
            .asFloatBuffer();
    squareVertex.put(squareVertexData).position(0);

    Matrix.setIdentityM(squareVertexData, 0);
    Matrix.rotateM(squareVertexData, 0, 0, 0f, 0f, -1f);

    // Copy squareVertexData and rotate it on 90 degrees
    Matrix.setIdentityM(rotatedVertexData, 0);
    Matrix.rotateM(rotatedVertexData, 0, 90, 0f, 0f, -1f);

    Matrix.setIdentityM(MVPMatrix, 0);
    Matrix.setIdentityM(STMatrix, 0);
  }

  public void initGl(Context context) {
    GlUtil.checkGlError("initGl start");
    String vertexShader = GlUtil.getStringFromRaw(context, R.raw.simple_vertex);
    String fragmentShader = GlUtil.getStringFromRaw(context, R.raw.fxaa);

    program = GlUtil.createProgram(vertexShader, fragmentShader);
    aPositionHandle = GLES20.glGetAttribLocation(program, "aPosition");
    aTextureHandle = GLES20.glGetAttribLocation(program, "aTextureCoord");
    uMVPMatrixHandle = GLES20.glGetUniformLocation(program, "uMVPMatrix");
    uSTMatrixHandle = GLES20.glGetUniformLocation(program, "uSTMatrix");
    uSamplerHandle = GLES20.glGetUniformLocation(program, "uSampler");
    uResolutionHandle = GLES20.glGetUniformLocation(program, "uResolution");
    uAAEnabledHandle = GLES20.glGetUniformLocation(program, "uAAEnabled");
    GlUtil.checkGlError("initGl end");
  }

  public void draw(int width, int height, boolean keepAspectRatio) {
    GlUtil.checkGlError("drawScreen start");

    PreviewSizeCalculator.calculateViewPort(keepAspectRatio, width, height, streamWidth,
        streamHeight);

    GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
    GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);

    GLES20.glUseProgram(program);

    squareVertex.position(BaseRenderOffScreen.SQUARE_VERTEX_DATA_POS_OFFSET);
    GLES20.glVertexAttribPointer(aPositionHandle, 3, GLES20.GL_FLOAT, false,
        BaseRenderOffScreen.SQUARE_VERTEX_DATA_STRIDE_BYTES, squareVertex);
    GLES20.glEnableVertexAttribArray(aPositionHandle);

    squareVertex.position(BaseRenderOffScreen.SQUARE_VERTEX_DATA_UV_OFFSET);
    GLES20.glVertexAttribPointer(aTextureHandle, 2, GLES20.GL_FLOAT, false,
        BaseRenderOffScreen.SQUARE_VERTEX_DATA_STRIDE_BYTES, squareVertex);
    GLES20.glEnableVertexAttribArray(aTextureHandle);

    GLES20.glUniformMatrix4fv(uMVPMatrixHandle, 1, false, MVPMatrix, 0);
    GLES20.glUniformMatrix4fv(uSTMatrixHandle, 1, false, STMatrix, 0);
    GLES20.glUniform2f(uResolutionHandle, width, height);
    GLES20.glUniform1f(uAAEnabledHandle, AAEnabled ? 1f : 0f);

    GLES20.glUniform1i(uSamplerHandle, 5);
    GLES20.glActiveTexture(GLES20.GL_TEXTURE5);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId);
    //draw
    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);

    GlUtil.checkGlError("drawScreen end");
  }

  public void setRotation(int rotation) {
    if(rotation == 90) {
      updateWithMatrix(rotatedVertexData);
    }
    else {
      updateWithMatrix(squareVertexData);
    }
  }

  private void updateWithMatrix(float[] matrixData) {
    Matrix.setIdentityM(MVPMatrix, 0);
    Matrix.multiplyMM(MVPMatrix, 0, matrixData, 0, MVPMatrix, 0);
  }

  public void release() {
    GLES20.glDeleteProgram(program);
  }

  public void setTexId(int texId) {
    this.texId = texId;
  }

  public void setAAEnabled(boolean AAEnabled) {
    this.AAEnabled = AAEnabled;
  }

  public boolean isAAEnabled() {
    return AAEnabled;
  }

  public void setStreamSize(int streamWidth, int streamHeight) {
    this.streamWidth = streamWidth;
    this.streamHeight = streamHeight;
  }
}

Replace: https://github.com/pedroSG94/rtmp-rtsp-stream-client-java/blob/master/encoder/src/main/java/com/pedro/encoder/input/gl/render/ManagerRender.java To:

package com.pedro.encoder.input.gl.render;

import android.content.Context;
import android.graphics.SurfaceTexture;
import android.os.Build;
import android.view.Surface;
import androidx.annotation.RequiresApi;
import com.pedro.encoder.input.gl.render.filters.BaseFilterRender;
import com.pedro.encoder.input.gl.render.filters.NoFilterRender;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by pedro on 27/01/18.
 */

@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
public class ManagerRender {

  //Increase it to render more than 1 filter and set filter by position.
  // You must modify it before create your rtmp or rtsp object.
  public static int numFilters = 1;

  private CameraRender cameraRender;
  private List<BaseFilterRender> baseFilterRender = new ArrayList<>(numFilters);
  private ScreenRender screenRender;

  private int width;
  private int height;
  private int previewWidth;
  private int previewHeight;
  private Context context;

  public ManagerRender() {
    cameraRender = new CameraRender();
    for (int i = 0; i < numFilters; i++) baseFilterRender.add(new NoFilterRender());
    screenRender = new ScreenRender();
  }

  public void initGl(Context context, int encoderWidth, int encoderHeight, int previewWidth,
      int previewHeight) {
    this.context = context;
    this.width = encoderWidth;
    this.height = encoderHeight;
    this.previewWidth = previewWidth;
    this.previewHeight = previewHeight;
    cameraRender.initGl(width, height, context, previewWidth, previewHeight);
    for (int i = 0; i < numFilters; i++) {
      int textId = i == 0 ? cameraRender.getTexId() : baseFilterRender.get(i - 1).getTexId();
      baseFilterRender.get(i).setPreviousTexId(textId);
      baseFilterRender.get(i).initGl(width, height, context, previewWidth, previewHeight);
      baseFilterRender.get(i).initFBOLink();
    }
    screenRender.setStreamSize(encoderWidth, encoderHeight);
    screenRender.setTexId(baseFilterRender.get(numFilters - 1).getTexId());
    screenRender.initGl(context);
  }

  public void drawOffScreen() {
    cameraRender.draw();
    for (BaseFilterRender baseFilterRender : baseFilterRender) baseFilterRender.draw();
  }

  public void drawScreen(int width, int height, boolean keepAspectRatio) {
    screenRender.draw(width, height, keepAspectRatio);
  }

  public void drawScreen(int width, int height, boolean keepAspectRatio, boolean isPreview) {
    if(isPreview) {
      screenRender.setRotation(0);
    } else {
      screenRender.setRotation(180);
    }
    screenRender.draw(width, height, keepAspectRatio);
  }

  public void release() {
    cameraRender.release();
    for (int i = 0; i < this.baseFilterRender.size(); i++) {
      this.baseFilterRender.get(i).release();
      this.baseFilterRender.set(i, new NoFilterRender());
    }
    screenRender.release();
  }

  public void enableAA(boolean AAEnabled) {
    screenRender.setAAEnabled(AAEnabled);
  }

  public boolean isAAEnabled() {
    return screenRender.isAAEnabled();
  }

  public void updateFrame() {
    cameraRender.updateTexImage();
  }

  public SurfaceTexture getSurfaceTexture() {
    return cameraRender.getSurfaceTexture();
  }

  public Surface getSurface() {
    return cameraRender.getSurface();
  }

  public void setFilter(int position, BaseFilterRender baseFilterRender) {
    final int id = this.baseFilterRender.get(position).getPreviousTexId();
    final RenderHandler renderHandler = this.baseFilterRender.get(position).getRenderHandler();
    this.baseFilterRender.get(position).release();
    this.baseFilterRender.set(position, baseFilterRender);
    this.baseFilterRender.get(position).setPreviousTexId(id);
    this.baseFilterRender.get(position).initGl(width, height, context, previewWidth, previewHeight);
    this.baseFilterRender.get(position).setRenderHandler(renderHandler);
  }

  public void setCameraRotation(int rotation) {
    cameraRender.setRotation(rotation);
  }

  public void setCameraFlip(boolean isFlipHorizontal, boolean isFlipVertical) {
    cameraRender.setFlip(isFlipHorizontal, isFlipVertical);
  }

  public void setPreviewSize(int previewWidth, int previewHeight) {
    for (int i = 0; i < this.baseFilterRender.size(); i++) {
      this.baseFilterRender.get(i).setPreviewSize(previewWidth, previewHeight);
    }
  }
}

Replace: https://github.com/pedroSG94/rtmp-rtsp-stream-client-java/blob/master/rtplibrary/src/main/java/com/pedro/rtplibrary/view/OpenGlView.java To:

package com.pedro.rtplibrary.view;

import android.content.Context;
import android.content.res.TypedArray;
import android.graphics.SurfaceTexture;
import android.os.Build;
import android.util.AttributeSet;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceHolder;
import androidx.annotation.RequiresApi;
import com.pedro.encoder.input.gl.SurfaceManager;
import com.pedro.encoder.input.gl.render.ManagerRender;
import com.pedro.encoder.input.gl.render.filters.BaseFilterRender;
import com.pedro.encoder.utils.gl.GlUtil;
import com.pedro.rtplibrary.R;

/**
 * Created by pedro on 9/09/17.
 */

@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
public class OpenGlView extends OpenGlViewBase {

  private ManagerRender managerRender = null;
  private boolean loadAA = false;

  private boolean AAEnabled = false;
  private boolean keepAspectRatio = false;
  private boolean isFlipHorizontal = false, isFlipVertical = false;

  public OpenGlView(Context context) {
    super(context);
  }

  public OpenGlView(Context context, AttributeSet attrs) {
    super(context, attrs);
    TypedArray typedArray = context.obtainStyledAttributes(attrs, R.styleable.OpenGlView);
    try {
      keepAspectRatio = typedArray.getBoolean(R.styleable.OpenGlView_keepAspectRatio, false);
      AAEnabled = typedArray.getBoolean(R.styleable.OpenGlView_AAEnabled, false);
      ManagerRender.numFilters = typedArray.getInt(R.styleable.OpenGlView_numFilters, 1);
      isFlipHorizontal = typedArray.getBoolean(R.styleable.OpenGlView_isFlipHorizontal, false);
      isFlipVertical = typedArray.getBoolean(R.styleable.OpenGlView_isFlipVertical, false);
    } finally {
      typedArray.recycle();
    }
  }

  @Override
  public void init() {
    if (!initialized) managerRender = new ManagerRender();
    managerRender.setCameraFlip(isFlipHorizontal, isFlipVertical);
    initialized = true;
  }

  @Override
  public SurfaceTexture getSurfaceTexture() {
    return managerRender.getSurfaceTexture();
  }

  @Override
  public Surface getSurface() {
    return managerRender.getSurface();
  }

  @Override
  public void setFilter(int filterPosition, BaseFilterRender baseFilterRender) {
    filterQueue.add(new Filter(filterPosition, baseFilterRender));
  }

  @Override
  public void setFilter(BaseFilterRender baseFilterRender) {
    setFilter(0, baseFilterRender);
  }

  @Override
  public void enableAA(boolean AAEnabled) {
    this.AAEnabled = AAEnabled;
    loadAA = true;
  }

  @Override
  public void setRotation(int rotation) {
    managerRender.setCameraRotation(rotation);
  }

  public boolean isKeepAspectRatio() {
    return keepAspectRatio;
  }

  public void setKeepAspectRatio(boolean keepAspectRatio) {
    this.keepAspectRatio = keepAspectRatio;
  }

  public void setCameraFlip(boolean isFlipHorizontal, boolean isFlipVertical) {
    managerRender.setCameraFlip(isFlipHorizontal, isFlipVertical);
  }

  @Override
  public boolean isAAEnabled() {
    return managerRender != null && managerRender.isAAEnabled();
  }

  @Override
  public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
    Log.i(TAG, "size: " + width + "x" + height);
    this.previewWidth = width;
    this.previewHeight = height;
    if (managerRender != null) managerRender.setPreviewSize(previewWidth, previewHeight);
  }

  @Override
  public void run() {
    releaseSurfaceManager();
    surfaceManager = new SurfaceManager(getHolder().getSurface());
    surfaceManager.makeCurrent();
    managerRender.initGl(getContext(), encoderWidth, encoderHeight, previewWidth, previewHeight);
    managerRender.getSurfaceTexture().setOnFrameAvailableListener(this);
    semaphore.release();
    try {
      while (running) {
        if (frameAvailable) {
          frameAvailable = false;
          surfaceManager.makeCurrent();
          managerRender.updateFrame();
          managerRender.drawOffScreen();
          managerRender.drawScreen(previewWidth, previewHeight, keepAspectRatio, true);
          surfaceManager.swapBuffer();
          if (takePhotoCallback != null) {
            takePhotoCallback.onTakePhoto(
                GlUtil.getBitmap(previewWidth, previewHeight, encoderWidth, encoderHeight));
            takePhotoCallback = null;
          }
          synchronized (sync) {
            if (surfaceManagerEncoder != null  && !fpsLimiter.limitFPS()) {
              surfaceManagerEncoder.makeCurrent();
              managerRender.drawScreen(encoderWidth, encoderHeight, false, false);
              surfaceManagerEncoder.swapBuffer();
            }
          }
          if (!filterQueue.isEmpty()) {
            Filter filter = filterQueue.take();
            managerRender.setFilter(filter.getPosition(), filter.getBaseFilterRender());
          } else if (loadAA) {
            managerRender.enableAA(AAEnabled);
            loadAA = false;
          }
        }
      }
    } catch (InterruptedException ignore) {
      Thread.currentThread().interrupt();
    } finally {
      managerRender.release();
      releaseSurfaceManager();
    }
  }
}