search

kaltura / playkit-ios

PlayKit: Kaltura Player SDK for iOS
https://developer.kaltura.com/player/ios
GNU Affero General Public License v3.0
86 stars 37 forks source link

Pause, Seeking and Seeked events are not emitted when using Airplay mode with the Apple TV remote #446

Open nidhik opened 3 years ago

nidhik commented 3 years ago

Pause, Seeking and Seeked events are not emitted when using Airplay mode with the Apple TV remote

Steps to reproduce:

In our app we have a simple Kaltura Playkit player with a pause/play button, a slider and a MPVolumeView to airplay when available.

  1. Choose airplay to apple tv
  2. Start playing the video
  3. Pause and play again with the apple tv remote controller
  4. Seek with the apple tv remote controller

Result: The pause, seeking and seeked events are never emitted. When the player is paused on the Apple TV, the controllers on the device (iPhone) still look as if it is playing because it's not on paused state.

Expected result: Pause, play, seeking and seeked events are emitted even when interacting with the Apple TV remote controller in airplay mode. The controllers in the device always match the state in the Apple TV

Prerequisites

class PlayerViewController: UIViewController { var kalturaPlayer: Player? let kalturaPlayerContainer = PlayerView() let playButton = UIButton() let closeButton = UIButton() let playheadSlider = UISlider() let positionLabel = UILabel() let durationLabel = UILabel() let airplayButton = MPVolumeView(frame: CGRect(x: 0, y: 0, width: 44, height: 44))

// MUX
let playerName = "iOS KalturaPlayer"

private var playerState: PlayerState = .idle {
    didSet {
        // Update player button icon depending on the state
        switch playerState {
        case .idle:
            self.playButton.setImage(UIImage(systemName: "play"), for: .normal)
        case .playing:
            self.playButton.setImage(UIImage(systemName: "pause"), for: .normal)
        case .paused:
            self.playButton.setImage(UIImage(systemName: "play"), for: .normal)
        case .ended:
            self.playButton.setImage(UIImage(systemName: "arrow.clockwise"), for: .normal)
        }
    }
}

override func viewDidLoad() {
    super.viewDidLoad()

    self.setupLayout()

    // Load PlayKit player
    self.kalturaPlayer = PlayKitManager.shared.loadPlayer(pluginConfig: nil)
    self.setupKalturaPlayer()

    // Setup MUX
    self.setupMUX()
}

override func viewWillDisappear(_ animated: Bool) {
    super.viewWillDisappear(animated)

    MUXSDKStats.destroyPlayer(name: self.playerName)
    self.kalturaPlayer?.destroy()
}

func setupKalturaPlayer() {
    // Set PlayerView as the container for PlayKit Player variable
    self.kalturaPlayer?.view = self.kalturaPlayerContainer
    self.loadMediaKalturaPlayer()

    // Handle PlayKit events
    self.playerState = .idle
    let events = [
        PlayerEvent.pause,
        PlayerEvent.playing,
        PlayerEvent.ended,
        PlayerEvent.durationChanged
    ]

    // Update player state depending on the Playkit events
    self.kalturaPlayer?.addObserver(self, events: events) { [weak self] (event) in
        guard let self = self else { return }

        switch event {
        case is PlayerEvent.Playing:
            self.playerState = .playing
        case is PlayerEvent.Pause:
            self.playerState = .paused
        case is PlayerEvent.Ended:
            self.playerState = .ended
            // Test video change
            self.changeMediaKalturaPlayer()
        case is PlayerEvent.DurationChanged:
            // Observe PlayKit event durationChanged to update the maximum duration of the slider and duration label
            guard let duration = event.duration as? TimeInterval else {
                return
            }

            self.playheadSlider.maximumValue = Float(duration)
            self.durationLabel.text = duration.formattedTimeDisplay
        default:
            break
        }
    }

    // Checks media progress to update the player slider and the current position label
    _ = self.kalturaPlayer?.addPeriodicObserver(
        interval: 0.2,
        observeOn: DispatchQueue.main,
        using: { [weak self] currentPosition in
            self?.playheadSlider.value = Float(currentPosition)
            self?.positionLabel.text = currentPosition.formattedTimeDisplay
        }
    )
}

func loadMediaKalturaPlayer() {
    let mediaConfig = createKalturaMediaConfig(
        contentURL: "https://bitdash-a.akamaihd.net/content/sintel/hls/playlist.m3u8",
        entryId: "sintel"
    )

    // Prepare PlayKit player
    self.kalturaPlayer?.prepare(mediaConfig)
}

func changeMediaKalturaPlayer() {
    let mediaConfig = createKalturaMediaConfig(
        contentURL: "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8",
        entryId: "bipbop_16x9"
    )

    // Call MUX videoChange before stop, because playkit stop will replace current item for nil
    self.MUXVideoChange()

    // Resets The Player And Prepares for Change Media
    self.kalturaPlayer?.stop()

    // Prepare PlayKit player
    self.kalturaPlayer?.prepare(mediaConfig)

    // Wait for `canPlay` event to play
    self.kalturaPlayer?.addObserver(self, events: [PlayerEvent.canPlay]) { event in
        self.kalturaPlayer?.play()
    }
}

func createKalturaMediaConfig(contentURL: String, entryId: String) -> MediaConfig {
    // Create PlayKit media source
    let source = PKMediaSource(entryId, contentUrl: URL(string: contentURL), drmData: nil, mediaFormat: .hls)

    // Setup PlayKit media entry
    let mediaEntry = PKMediaEntry(entryId, sources: [source])

    // Create PlayKit media config
    return MediaConfig(mediaEntry: mediaEntry)
}

func setupMUX() {
    let playerData = MUXSDKCustomerPlayerData(environmentKey: "YOUR_ENV_KEY_HERE")
    playerData?.playerName = self.playerName

    let videoData = MUXSDKCustomerVideoData()
    videoData.videoTitle = "Title Video Kaltura"
    videoData.videoId = "sintel"
    videoData.videoSeries = "animation"

    let viewData = MUXSDKCustomerViewData()
    viewData.viewSessionId = "my session id"

    let customData = MUXSDKCustomData()
    customData.customData1 = "Kaltura test"
    customData.customData2 = "Custom Data 2"

    let viewerData = MUXSDKCustomerViewerData()
    viewerData.viewerApplicationName = "MUX Kaltura DemoApp"

    let customerData = MUXSDKCustomerData(
        customerPlayerData: playerData,
        videoData: videoData,
        viewData: viewData,
        customData: customData,
        viewerData: viewerData
    )

    guard let player = self.kalturaPlayer, let data = customerData else {
        return
    }

    MUXSDKStats.monitorPlayer(
        player: player,
        playerName: self.playerName,
        customerData: data
    )
}

func MUXVideoChange() {
    let playerData = MUXSDKCustomerPlayerData(environmentKey: "shqcbkagevf0r4jh9joir48kp")
    playerData?.playerName = self.playerName

    let videoData = MUXSDKCustomerVideoData()
    videoData.videoTitle = "Apple Video Kaltura"
    videoData.videoId = "apple"
    videoData.videoSeries = "conference"

    let viewData = MUXSDKCustomerViewData()
    viewData.viewSessionId = "my second session id"

    let customData = MUXSDKCustomData()
    customData.customData1 = "Kaltura test video change"

    let viewerData = MUXSDKCustomerViewerData()
    viewerData.viewerApplicationName = "MUX Kaltura DemoApp"

    guard let customerData = MUXSDKCustomerData(
        customerPlayerData: playerData,
        videoData: videoData,
        viewData: viewData,
        customData: customData,
        viewerData: viewerData
    ) else {
        return
    }

    MUXSDKStats.videoChangeForPlayer(name: self.playerName, customerData: customerData)
}

@objc func playButtonPressed() {
    guard let player = self.kalturaPlayer else {
        return
    }

    // Handle PlayKit events
    switch playerState {
    case .playing:
        player.pause()
    case .idle:
        player.play()
    case .paused:
        player.play()
    case .ended:
        player.seek(to: 0)
        player.play()
    }
}

@objc func closeButtonPressed() {
    self.navigationController?.popToRootViewController(animated: true)
}

@objc func playheadValueChanged() {
    guard let player = self.kalturaPlayer else {
        return
    }

    if self.playerState == .ended && self.playheadSlider.value < self.playheadSlider.maximumValue {
        self.playerState = .paused
    }

    player.currentTime = TimeInterval(self.playheadSlider.value)
}

}

extension PlayerViewController { enum PlayerState { case idle case playing case paused case ended } }

extension PlayerViewController { func setupLayout() { self.view.backgroundColor = .black self.view.addSubview(self.kalturaPlayerContainer)

    // Constraint PlayKit player container to safe area layout guide
    self.kalturaPlayerContainer.translatesAutoresizingMaskIntoConstraints = false
    let guide = self.view.safeAreaLayoutGuide
    NSLayoutConstraint.activate([
        self.kalturaPlayerContainer.topAnchor.constraint(equalTo: guide.topAnchor),
        self.kalturaPlayerContainer.bottomAnchor.constraint(equalTo: guide.bottomAnchor),
        self.kalturaPlayerContainer.leadingAnchor.constraint(equalTo: guide.leadingAnchor),
        self.kalturaPlayerContainer.trailingAnchor.constraint(equalTo: guide.trailingAnchor)
    ])

    let actionsContainer = UIStackView()
    actionsContainer.axis = .vertical
    actionsContainer.isLayoutMarginsRelativeArrangement = true
    actionsContainer.layoutMargins = UIEdgeInsets(top: 0, left: 8.0, bottom: 0, right: 8.0)
    actionsContainer.translatesAutoresizingMaskIntoConstraints = false
    self.kalturaPlayerContainer.addSubview(actionsContainer)
    NSLayoutConstraint.activate([
        actionsContainer.bottomAnchor.constraint(equalTo: self.kalturaPlayerContainer.bottomAnchor),
        actionsContainer.leadingAnchor.constraint(equalTo: self.kalturaPlayerContainer.leadingAnchor),
        actionsContainer.trailingAnchor.constraint(equalTo: self.kalturaPlayerContainer.trailingAnchor)
    ])

    // Add airplay button
    self.airplayButton.showsVolumeSlider = false
    NSLayoutConstraint.activate([
        self.airplayButton.widthAnchor.constraint(equalToConstant: 44.0),
        self.airplayButton.heightAnchor.constraint(equalToConstant: 44.0)
    ])

    let airplayRowStack = UIStackView()
    airplayRowStack.axis = .horizontal
    airplayRowStack.addArrangedSubview(UIView())
    airplayRowStack.addArrangedSubview(airplayButton)
    actionsContainer.addArrangedSubview(airplayRowStack)

    let actionsRowStack = UIStackView()
    actionsRowStack.axis = .horizontal
    actionsRowStack.spacing = 6.0
    actionsContainer.addArrangedSubview(actionsRowStack)
    NSLayoutConstraint.activate([
        actionsRowStack.heightAnchor.constraint(equalToConstant: 44.0)
    ])

    // Add play/pause button
    self.playButton.addTarget(self, action: #selector(self.playButtonPressed), for: .touchUpInside)
    self.playButton.contentEdgeInsets = UIEdgeInsets(top: 10, left: 4, bottom: 10, right: 4)
    self.playButton.contentHorizontalAlignment = .fill
    self.playButton.contentVerticalAlignment = .fill
    actionsRowStack.addArrangedSubview(self.playButton)
    NSLayoutConstraint.activate([
        self.playButton.widthAnchor.constraint(equalToConstant: 28.0)
    ])

    self.positionLabel.textColor = .lightText
    self.positionLabel.text = TimeInterval.zero.formattedTimeDisplay
    actionsRowStack.addArrangedSubview(self.positionLabel)

    self.playheadSlider.addTarget(self, action: #selector(self.playheadValueChanged), for: .valueChanged)
    actionsRowStack.addArrangedSubview(self.playheadSlider)

    self.durationLabel.textColor = .lightText
    self.durationLabel.text = TimeInterval.zero.formattedTimeDisplay
    actionsRowStack.addArrangedSubview(self.durationLabel)

    // Add close button
    self.closeButton.translatesAutoresizingMaskIntoConstraints = false
    self.closeButton.addTarget(self, action: #selector(self.closeButtonPressed), for: .touchUpInside)
    self.closeButton.setImage(UIImage(systemName: "xmark.square"), for: .normal)
    self.closeButton.contentVerticalAlignment = .fill
    self.closeButton.contentHorizontalAlignment = .fill
    self.kalturaPlayerContainer.addSubview(self.closeButton)
    NSLayoutConstraint.activate([
        self.closeButton.heightAnchor.constraint(equalToConstant: 32.0),
        self.closeButton.widthAnchor.constraint(equalToConstant: 32.0),
        self.closeButton.trailingAnchor.constraint(equalTo: self.kalturaPlayerContainer.trailingAnchor, constant: -24.0),
        self.closeButton.topAnchor.constraint(equalTo: self.kalturaPlayerContainer.topAnchor, constant: 24.0)
    ])
}

}


##### Expected behavior
It's a simple kaltura playkit player with the button to airplay. I reproduce in airplay mode, when I pause or seek with the apple tv remote, I get kaltura PlayEvents for Pause, Seeking and Seeked. The UI in the device is in sync with the apple tv player, for example:if it is pause it shows the play button and viceversa. 

##### Actual behavior
I reproduce in airplay mode, when I pause or seek with the apple tv remote, I don't get any of the following kaltura PlayEvents: Pause, Seeking and Seeked. The UI in the device is not in sync with the apple tv player, for example: if it is paused in apple tv it still shows the pause button on the device, even though it should be showing the play button instead.

##### Console output

2021-11-05 2:06:04.139 PM [Debug] [DefaultAssetHandler.swift:88] build(from:readyCallback:) > Creating clear AVURLAsset 2021-11-05 2:06:04.178 PM [Debug] [AVPlayerEngine.swift:147] startPosition > set startPosition: nan 2021-11-05 2:06:04.179 PM [Debug] [AVPlayerEngine.swift:76] asset > The asset status changed to: preparing 2021-11-05 2:06:04.179 PM [Debug] [NetworkUtils.swift:57] sendKavaAnalytics(forPartnerId:entryId:eventType:sessionId:) > Sending Kava Event type: 1 2021-11-05 2:06:04.180 PM [Debug] [NetworkUtils.swift:57] sendKavaAnalytics(forPartnerId:entryId:eventType:sessionId:) > Sending Kava Event type: 2 2021-11-05 2:06:04.181 PM [Debug] [PlayerController+TimeMonitor.swift:16] addPeriodicObserver(interval:observeOn:using:) > add periodic observer with interval: 0.2, on queue: Optional(<OS_dispatch_queue_main: com.apple.main-thread[0x105668c80] = { xref = -2147483648, ref = -2147483648, sref = 1, target = com.apple.root.default-qos.overcommit[0x105669100], width = 0x1, state = 0x001ffe9000000300, dirty, in-flight = 0, thread = 0x303 }>) 2021-11-05 2:06:04.181 PM [Debug] [PlayerController+TimeMonitor.swift:18] addPeriodicObserver(interval:observeOn:using:) > periodic observer added with token: 6503F1FF-68E4-4E8C-9011-DAD6B36BDE6E 2021-11-05 2:06:04.184 PM [Debug] [PlayerController+TimeMonitor.swift:16] addPeriodicObserver(interval:observeOn:using:) > add periodic observer with interval: 0.1, on queue: nil 2021-11-05 2:06:04.184 PM [Debug] [PlayerController+TimeMonitor.swift:18] addPeriodicObserver(interval:observeOn:using:) > periodic observer added with token: 1AE02AF5-F7F5-484C-A19A-1A94BF991B47 2021-11-05 2:06:04.287 PM [Debug] [AVPlayerEngine.swift:76] asset > The asset status changed to: prepared 2021-11-05 2:06:04.288 PM [Debug] [AVPlayerEngine+Observation.swift:224] handleRate() > player rate was changed, now: 0.0 2021-11-05 2:06:04.288 PM [Debug] [AVPlayerEngine.swift:387] postStateChange(newState:oldState:) > stateChanged:: new:Idle old:Idle 2021-11-05 2:06:04.288 PM [Error] [AVPlayerEngine+Observation.swift:176] observeValue(forKeyPath:of:change:context:) > unknown player item status 2021-11-05 2:06:04.289 PM [Debug] [AVPlayerEngine+Observation.swift:365] handleDurationChanged() > Duration in seconds: 888.0 2021-11-05 2:06:04.289 PM [Debug] [AVPlayerEngine.swift:387] postStateChange(newState:oldState:) > stateChanged:: new:Buffering old:Idle 2021-11-05 2:06:04.289 PM [Debug] [AVPlayerEngine.swift:387] postStateChange(newState:oldState:) > stateChanged:: new:Idle old:Buffering 2021-11-05 2:06:04.303 PM [Debug] [AVPlayerEngine+Observation.swift:236] handle(status:) > player is ready to play player items 2021-11-05 2:06:04.303 PM [Debug] [AVPlayerEngine+Observation.swift:239] handle(status:) > duration in seconds: 888.0 2021-11-05 2:06:04.402 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:06:04.430 PM [Debug] [AVPlayerEngine+Observation.swift:75] onAccessLogEntryNotification(notification:) > event log: event log: averageAudioBitrate - 378256.0 event log: averageVideoBitrate - 0.0 event log: indicatedAverageBitrate - -1.0 event log: indicatedBitrate - 6214307.0 event log: observedBitrate - nan event log: observedMaxBitrate - 0.0 event log: observedMinBitrate - -1.0 event log: switchBitrate - -1.0 event log: numberOfBytesTransferred - 94564 event log: numberOfStalls - 0 event log: URI - 'https://bitdash-a.akamaihd.net/content/sintel/hls/video/6000kbit.m3u8' event log: startupTime - -1.0 2021-11-05 2:06:04.543 PM [Warning] [AVPlayerEngine+Observation.swift:91] onErrorLogEntryNotification(notification:) > error description: Optional("Segment exceeds specified bandwidth for variant"), error domain: CoreMediaErrorDomain, error code: -12318 2021-11-05 2:06:04.555 PM [Debug] [AVPlayerEngine+Observation.swift:365] handleDurationChanged() > Duration in seconds: 888.0 2021-11-05 2:06:04.556 PM [Debug] [AVPlayerEngine.swift:387] postStateChange(newState:oldState:) > stateChanged:: new:Ready old:Idle 2021-11-05 2:06:04.562 PM [Debug] [TracksManager.swift:38] handleTracks(item:cea608CaptionsEnabled:block:) > audio tracks:: Optional([<PlayKit.Track: 0x28057a540>, <PlayKit.Track: 0x28057a600>]), text tracks:: Optional([<PlayKit.Track: 0x28057aa80>, <PlayKit.Track: 0x28057a7c0>, <PlayKit.Track: 0x28057a840>, <PlayKit.Track: 0x28057a900>, <PlayKit.Track: 0x28057aa00>]) 2021-11-05 2:06:04.562 PM [Debug] [AVPlayerEngine+Observation.swift:274] handle(playerItemStatus:) > duration in seconds: 888.0 2021-11-05 2:06:04.562 PM [Debug] [AVPlayerEngine.swift:387] postStateChange(newState:oldState:) > stateChanged:: new:Ready old:Ready 2021-11-05 2:06:04.954 PM [Debug] [NetworkUtils.swift:55] sendKavaAnalytics(forPartnerId:entryId:eventType:sessionId:) > Response: Status Code: 0 Error: Data: { time = "1636142765.009"; viewEventsEnabled = 1; } 2021-11-05 2:06:05.371 PM [Debug] [NetworkUtils.swift:55] sendKavaAnalytics(forPartnerId:entryId:eventType:sessionId:) > Response: Status Code: 0 Error: Data: { time = "1636142765.427"; viewEventsEnabled = 1; } 2021-11-05 2:06:07.092 PM [Debug] [AVPlayerEngine.swift:294] play() > Play player 2021-11-05 2:06:07.092 PM [Debug] [AVPlayerEngine+Observation.swift:224] handleRate() > player rate was changed, now: 1.0 2021-11-05 2:06:08.000 PM [Debug] [AVPlayerEngine+Observation.swift:75] onAccessLogEntryNotification(notification:) > event log: event log: averageAudioBitrate - 0.0 event log: averageVideoBitrate - 41360.0 event log: indicatedAverageBitrate - -1.0 event log: indicatedBitrate - 1558322.0 event log: observedBitrate - 32336999.91437678 event log: observedMaxBitrate - 94714957.2420789 event log: observedMinBitrate - 2296042.501051438 event log: switchBitrate - -1.0 event log: numberOfBytesTransferred - 10340 event log: numberOfStalls - 0 event log: URI - 'https://bitdash-a.akamaihd.net/content/sintel/hls/video/1500kbit.m3u8' event log: startupTime - 0.0 MUXSDK-INFO - Switch advertised bitrate from: 6214307.0 to: 1558322.0 2021-11-05 2:06:08.018 PM [Warning] [AVPlayerEngine+Observation.swift:91] onErrorLogEntryNotification(notification:) > error description: Optional("Segment exceeds specified bandwidth for variant"), error domain: CoreMediaErrorDomain, error code: -12318 2021-11-05 2:06:11.296 PM [Debug] [AVPlayerEngine.swift:387] postStateChange(newState:oldState:) > stateChanged:: new:Ready old:Ready 2021-11-05 2:06:24.329 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:06:33.309 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:07:03.311 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:07:05.396 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:07:07.314 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:07:09.319 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:07:11.312 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:07:13.312 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:07:17.036 PM [Debug] [AVPlayerEngine+Observation.swift:224] handleRate() > player rate was changed, now: 0.0 2021-11-05 2:07:17.312 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:07:34.309 PM [Debug] [AVPlayerEngine.swift:387] postStateChange(newState:oldState:) > stateChanged:: new:Buffering old:Ready 2021-11-05 2:07:34.646 PM [Debug] [AVPlayerEngine+Observation.swift:224] handleRate() > player rate was changed, now: 8.0 2021-11-05 2:07:35.607 PM [Debug] [AVPlayerEngine+Observation.swift:224] handleRate() > player rate was changed, now: 24.0 2021-11-05 2:07:40.318 PM [Debug] [AVPlayerEngine+Observation.swift:224] handleRate() > player rate was changed, now: 0.0 2021-11-05 2:07:42.236 PM [Debug] [AVPlayerEngine+Observation.swift:224] handleRate() > player rate was changed, now: 1.0 2021-11-05 2:07:42.236 PM [Debug] [AVPlayerEngine.swift:387] postStateChange(newState:oldState:) > stateChanged:: new:Ready old:Buffering 2021-11-05 2:07:55.237 PM [Debug] [AVPlayerEngine+Observation.swift:157] observeValue(forKeyPath:of:change:context:) > Buffer Full 2021-11-05 2:08:01.814 PM [Debug] [AVPlayerEngine+Observation.swift:224] handleRate() > player rate was changed, now: 0.0