App video does not display

[BeccaClipper]

I'm working with a team on an app we adopted, it contains a video component that is intended to launch the twilio video chat on load. The purpose is that a user can share audio and video with another user viewing from a web application. When the component loads the app user will be sharing audio with the option to toggle on video.

We've found we sometimes don't have video included in our video chat when the user toggles it on, it only shows a black screen or video is very delayed. Other times the component will crash or freeze on load. I've included a truncated version of the component below:

import { images } from 'assets';
import { SafeAreaContainer } from 'components';
import { LogErrorToDb } from 'containers/app/actions';
import {
  makeSelectAppLocation,
  makeSelectTwilioAccessToken,
} from 'containers/app/selectors';
import { IncidentEscort, Organization } from 'incident-code-core';
import React, { FC, useCallback, useEffect, useRef } from 'react';
import { useIntl } from 'react-intl';
import { Location } from 'react-native-background-geolocation';
import KeepAwake from 'react-native-keep-awake';
import { ScreenProps } from 'react-native-screens';
import { TwilioVideo } from 'react-native-twilio-video-webrtc';
import { NavigationStackScreenProps } from 'react-navigation-stack';
import { useAction, useSelector } from 'utils/hooks';

import { isTokenExpired, refreshTwilioAccessToken } from 'utils/token';

import messages from './messages';
import RecordAudio from './RecordAudio';
import { makeSelectRecordType } from './selectors';
import {
  AudioStream,
  CameraFlipButton,
  Container,
  Logo,
  OrganizationName,
  RecordContainer,
  RecordDot,
  SharingText,
  TwilioVideoPreview,
  BottomControls,
  ToggleContainer,
  ToggleImageContainer,
  ToggleImage,
  ToggleText,
} from './styles';
import { RecordType } from './types';
import { toggleRecordType } from './actions';

interface Params {
  incidentEscort: IncidentEscort;
  organization: Organization;
  fromPassive?: boolean;
}
interface Props extends NavigationStackScreenProps<Params, ScreenProps> {}

const EscortScreen: FC<Props> = ({ navigation: { getParam } }) => {
  const incidentEscort = getParam('incidentEscort');
  const organization = getParam('organization');

  const locationRef = useRef<Location>();

  const { formatMessage } = useIntl();

  const logError = useAction(LogErrorToDb);
  const toggleRecordTypeAction = useAction(toggleRecordType);

  const recordType = useSelector(makeSelectRecordType());
  const accessToken = useSelector(makeSelectTwilioAccessToken());
  const location = useSelector(makeSelectAppLocation());

  const twilioVideoRef = useRef<TwilioVideo>(null);

  useEffect(() => {
    KeepAwake.activate();
    return function componentDidUnMount() {
      KeepAwake.deactivate();
    };
  }, [incidentEscort]);

  useEffect(() => {
    if (isTokenExpired(accessToken)) {
      refreshTwilioAccessToken().then(newToken => {
        twilioVideoRef.current?.connect({
          roomName: incidentEscort.twilio?.roomId,
          accessToken: newToken.access_token || '',
        });
      });
    } else {
      twilioVideoRef.current?.connect({
        roomName: incidentEscort.twilio?.roomId,
        accessToken: accessToken.access_token || '',
      });
    }
    return function onComponentDidUnMount() {
      twilioVideoRef.current?.disconnect();
    };
  }, [accessToken, incidentEscort, twilioVideoRef.current]);

  useEffect(() => {
    locationRef.current = location;
  }, [location]);

  const onFlipButtonClick = useCallback(() => {
    twilioVideoRef.current?.flipCamera();
  }, []);

  const handleRoomDidConnect = useCallback(() => {
    twilioVideoRef.current?.flipCamera();
    twilioVideoRef.current?.setLocalVideoEnabled(false);
  }, []);

  const handleRoomDisconnect = useCallback(payload => {
    logError({
      source: 'Record Screen: OnRoomDidDisconnect',
      message: payload.error,
      raw: JSON.stringify(payload),
    });
  }, []);

  const handleRoomConnectionFail = useCallback(payload => {
    logError({
      source: 'Record Screen: OnRoomDidFailToConnect',
      message: payload.error,
      raw: JSON.stringify(payload),
    });
  }, []);

  return (
    <>
      <SafeAreaContainer>
        <Container>
          <RecordContainer>
            <TwilioVideoPreview enabled={recordType === RecordType.Video} />
            {recordType === RecordType.Audio && (
              <RecordAudio location={location} /> //Shows a map overlay when the user isn't sharing video
            )}
            <Logo />
            <OrganizationName>
              {formatMessage(messages.escort, {
                organizationName: organization.name,
              })}
            </OrganizationName>
            <SharingText>
              {formatMessage(
                recordType === RecordType.Audio
                  ? messages.sharingAudio
                  : messages.shareVideo,
              )}
            </SharingText>
            {recordType === RecordType.Audio && <AudioStream />}
            {recordType === RecordType.Video && <RecordDot />}
            {recordType === RecordType.Video && (
              <CameraFlipButton
                onPress={onFlipButtonClick}
                image={images.icFlip()}
              />
            )}
          </RecordContainer>
          <BottomControls>
            <ToggleContainer
              recordType={recordType}
              onPress={() => toggleRecordTypeAction(incidentEscort)}
            >
              <ToggleImageContainer>
                <ToggleImage
                  source={
                    recordType === RecordType.Audio
                      ? images.icMicrophone()
                      : images.icCamera()
                  }
                />
              </ToggleImageContainer>
              <ToggleText>
                {recordType === RecordType.Audio
                  ? formatMessage(messages.shareVideo)
                  : formatMessage(messages.audioOnly)}
              </ToggleText>
            </ToggleContainer>
          </BottomControls>
        </Container>
        <TwilioVideo
          ref={twilioVideoRef}
          onRoomDidConnect={handleRoomDidConnect}
          onRoomDidDisconnect={handleRoomDisconnect}
          onRoomDidFailToConnect={handleRoomConnectionFail}
        />
      </SafeAreaContainer>
    </>
  );
};

export default EscortScreen;

We've also noticed when adding all hooks and logging to the TwilioVideo component we get the following pattern during the connection process when we do have a successful connection: RoomDidConnect ParticipantDidConnect RoomDidDisconnect RoomDidConnect ParticipantDidConnect

Steps to reproduce

1. Initiate record screen component
2. Toggle video on, only black screen displays

Expected behaviour

When toggling video on, video displays on screen in the app and is visible to the other participant

Actual behaviour

When video is toggled on the app user only sees a black screen, video is not available to the other participant. On occasion, component will crash or freeze on load.

Environment

• Node.js version: v12.14.1
• React Native version: 0.61.5
• React Native platform + platform version: Android version 10

react-native-twilio-video-webrtc

Version: "master"

[slycoder]

Thanks so much for the detailed information. Just to confirm the app has requested the required android permissions before mounting this component (I've seen crashes in twilio if audio or video is not granted in advance).

The other thing that would be helpful is if you had the stack traces from the crashes. One thing that comes to mind is that there might be some race condition between enabling on the preview, calling setLocalVideoEnabled, and calling flipCamera. You might want to try, just for testing purposes, removing some of these calls (e.g. remove flipCamera calls, setting enabled to always be true, etc.) and see if that helps alleviate the issue. If it does it'll help us track down the issue and potentially patch things.

[BeccaClipper]

Thanks @slycoder for taking a look at my issue.

I can confirm that we are requesting and require all permissions be accepted before this component is mounted.

I think you may be on to something with the race condition, when I tried removing some of those calls I was able to get more stable performance. After adding them back I was able to trigger a crash and grabbed the log to share here. Please see the log below:

` OS Version: Android 10 (sdk_gphone_x86-userdebug 10 QSR1.191030.002 5978551 dev-keys) Report Version: 104

Exception Type: Unknown (SIGABRT)

Application Specific Information: Abort

Thread 0 Crashed: 0 linux-gate.so 0xedf3dad9 __kernel_vsyscall 1 libc.so 0xecd97328 syscall 2 libc.so 0xecdb2651 abort 3 libc.so 0xece2144a __fortify_fatal 4 libc.so 0xece2096a HandleUsingDestroyedMutex 5 libc.so 0xece20812 pthread_mutex_lock 6 libjingle_peerconnection_so.so 0xb4aef877 + 3031365751 7 libjingle_peerconnection_so.so 0xb4734b4b + 3027454795 8 libjingle_peerconnection_so.so 0xb473528a Java_com_twilio_video_VideoCapturerDelegate_00024NativeObserver_nativeOnFrameCaptured 9 base.odex 0xc2dc9e18 + 3269238296

EOF `

[BeccaClipper]

We were able to work past this by moving our connection outside of this useEffect loop:

useEffect(() => { if (isTokenExpired(accessToken)) { refreshTwilioAccessToken().then(newToken => { twilioVideoRef.current?.connect({ roomName: incidentEscort.twilio?.roomId, accessToken: newToken.access_token || '', }); }); } else { twilioVideoRef.current?.connect({ roomName: incidentEscort.twilio?.roomId, accessToken: accessToken.access_token || '', }); } return function onComponentDidUnMount() { twilioVideoRef.current?.disconnect(); }; }, [accessToken, incidentEscort, twilioVideoRef.current]);

By adding TwilioVideoRef.current.connection to a useEffect ran only once on init we seem to be avoiding the crashes and have had consistent video connection.

Thank you for your help

[slycoder]

Thanks again for the detailed investigation. If you ever narrow down a particular race condition let me know and I can try to improve the library to address it. But your workaround is very clearly described and will no doubt be useful to others!

[BeccaClipper]

Unfortunately after what was seen to be initial success we're still experiencing intermittent crashes. This is the error log that we get in sentry:

OS Version: Android 10 (sdk_gphone_x86-userdebug 10 QSR1.191030.002 5978551 dev-keys)
Report Version: 104

Exception Type: Unknown (SIGSEGV)

Application Specific Information:
Segfault

Thread 0 Crashed:
0   libjingle_peerconnection_so.so  0xa85665bc          <unknown> + 2824234428
1   libjingle_peerconnection_so.so  0xa8559e63          Java_com_twilio_video_ConnectOptions_nativeCreate
2   libart.so                       0xe45c6f67          art_quick_generic_jni_trampoline
3   libart.so                       0xe45c07d2          art_quick_invoke_stub
4   libart.so                       0xe45cba69          art::ArtMethod::Invoke
5   libart.so                       0xe47b4502          art::interpreter::ArtInterpreterToCompiledCodeBridge
6   libart.so                       0xe47af3df          art::interpreter::DoCall<T>
7   libart.so                       0xe4b09e3d          MterpInvokeDirectRange
8   libart.so                       0xe45bac21          mterp_op_invoke_direct_range
9   libart.so                       0xe477ae0a          [clone .llvm.6689468213397061768]
10  libart.so                       0xe4781cc5          art::interpreter::EnterInterpreterFromEntryPoint
11  libart.so                       0xe4af1bd9          artQuickToInterpreterBridge
12  libart.so                       0xe45c703d          art_quick_to_interpreter_bridge
13  libart.so                       0xe45c07d2          art_quick_invoke_stub
14  libart.so                       0xe45cba69          art::ArtMethod::Invoke
15  libart.so                       0xe49dc513          art::(anonymous namespace)::InvokeWithArgArray
16  libart.so                       0xe49ddc7a          art::InvokeVirtualOrInterfaceWithVarArgs
17  libart.so                       0xe4886b9f          art::JNI::CallLongMethodV
18  libart.so                       0xe485ae91          art::(anonymous namespace)::CheckJNI::CallMethodV
19  libart.so                       0xe4847009          art::(anonymous namespace)::CheckJNI::CallLongMethodV
20  libjingle_peerconnection_so.so  0xa855b5af          <unknown> + 2824189359
21  libjingle_peerconnection_so.so  0xa857b6a5          <unknown> + 2824320677
22  libjingle_peerconnection_so.so  0xa857b375          <unknown> + 2824319861
23  libjingle_peerconnection_so.so  0xa895919b          <unknown> + 2828374427
24  libjingle_peerconnection_so.so  0xa8974275          <unknown> + 2828485237
25  libjingle_peerconnection_so.so  0xa8974200          <unknown> + 2828485120
26  libjingle_peerconnection_so.so  0xa897409c          <unknown> + 2828484764
27  libc.so                         0xe701b8e5          __pthread_start
28  libc.so                         0xe6fb06a7          __start_thread
29  <unknown>                       0x0                 <unknown>

EOF

It appears to be an issue in Android only.

[BeccaClipper]

I reached out to twilio support giving them the twilio-video-android version (5.5.0 from what I can see) and sharing the same crash log with them. They suggested updating to the current version, 5.10.0 as many of the updates have resolved issues with crashes since them.

Are there any work in progress at this point to update to version 5.10.0?

[slycoder]

I haven't personally tried that newer version, but you can just change the line here:

https://github.com/blackuy/react-native-twilio-video-webrtc/blob/ad03d0106cfa5bcc75157f8ddb31a85ba02e0640/android/build.gradle#L52

And give it a shot. Since there's no major version bump I'm hoping there are no api changes that have to be made.