Media3 with Google Assistant integration

[andreas-umbricht]

We wanted to integrate Google Assistant functionalities within our app with the new Media3 Library.

The goal is so that the user is able to say:

"Play Energy Basel" or "Play Energy Basel on/in Energy" or something similar to that.

Energy is the name of the app, in which there are about 20 radio channels. Sadly, the Google Assistant always tries to open another app called TuneIn, even though it's not even installed, and my MediaLibrarySession.Callback functions never get invoked by it.

The simplified code where I left out unnecessary code and reduced the available channels to only one possible MediaItem:

class RadioService: MediaLibraryService() {

    override fun onCreate() {
        super.onCreate()

        val sessionActivityPendingIntent =
            PendingIntent.getActivity(
                this,
                0,
                Intent(this, MainActivity::class.java),
                PendingIntent.FLAG_IMMUTABLE
            )

        mediaSession = MediaLibrarySession.Builder(
            this,
            exoPlayer,
            MediaLibrarySessionCallback()
        )
            .setSessionActivity(sessionActivityPendingIntent)
            .setBitmapLoader(CacheBitmapLoader(DataSourceBitmapLoader(this)))
            .build()
    }

    override fun onGetSession(controllerInfo: MediaSession.ControllerInfo): MediaLibrarySession? {
        return mediaSession
    }

    private inner class MediaLibrarySessionCallback: MediaLibrarySession.Callback {
        //All clients can connect
        override fun onConnect(
            session: MediaSession,
            controller: MediaSession.ControllerInfo
        ): MediaSession.ConnectionResult {
            return MediaSession.ConnectionResult.AcceptedResultBuilder(session)
                .setAvailablePlayerCommands(MediaSession.ConnectionResult.DEFAULT_PLAYER_COMMANDS)
                .build()
        }

        override fun onGetLibraryRoot(
            session: MediaLibrarySession,
            browser: MediaSession.ControllerInfo,
            params: LibraryParams?
        ): ListenableFuture<LibraryResult<MediaItem>> {
            return Futures.immediateFuture(
                LibraryResult.ofItem(
                    MediaItem.Builder()
                        .setUri("*****") <- Censored url for StackOverflow
                        .setMediaId("energy")
                        .setMediaMetadata(
                            MediaMetadata
                                .Builder()
                                .setIsBrowsable(false)
                                .setIsPlayable(false)
                                .build()
                        )
                        .build(),
                    params
                )
            )
        }

        override fun onGetItem(
            session: MediaLibrarySession,
            browser: MediaSession.ControllerInfo,
            mediaId: String
        ): ListenableFuture<LibraryResult<MediaItem>> {
            return Futures.immediateFuture(
                LibraryResult.ofItem(
                    allMediaItems[0],
                    null
                )
            )
        }

        override fun onGetChildren(
            session: MediaLibrarySession,
            browser: MediaSession.ControllerInfo,
            parentId: String,
            page: Int,
            pageSize: Int,
            params: LibraryParams?
        ): ListenableFuture<LibraryResult<ImmutableList<MediaItem>>> {
            return Futures.immediateFuture(
                LibraryResult.ofItemList(
                    allMediaItems,
                    null
                )
            )
        }

        override fun onSearch(
            session: MediaLibrarySession,
            browser: MediaSession.ControllerInfo,
            query: String,
            params: LibraryParams?
        ): ListenableFuture<LibraryResult<Void>> {
            return super.onSearch(session, browser, query, params)
        }

        override fun onGetSearchResult(
            session: MediaLibrarySession,
            browser: MediaSession.ControllerInfo,
            query: String,
            page: Int,
            pageSize: Int,
            params: LibraryParams?
        ): ListenableFuture<LibraryResult<ImmutableList<MediaItem>>> {
            return super.onGetSearchResult(session, browser, query, page, pageSize, params)
        }
    }

    val allMediaItems = listOf(
        MediaItem.Builder()
            .setMediaId("energy-basel")
            .setUri("****") <- Censored url for StackOverflow
            .setMimeType(MimeTypes.AUDIO_MPEG)
            .setLiveConfiguration(MediaItem.LiveConfiguration.Builder().build())
            .setMediaMetadata(
                MediaMetadata.Builder()
                    .setTitle("Energy Basel")
                    .setArtworkUri(Uri.parse("****")) <- Censored url for StackOverflow
                    .setIsPlayable(true)
                    .setIsBrowsable(false)
                    .build()
            )
            .build()
    )
}

<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools">

    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PLAYBACK" />
    <uses-permission android:name="android.permission.INTERNET" />

    <application
        android:allowBackup="true"
        android:dataExtractionRules="@xml/data_extraction_rules"
        android:fullBackupContent="@xml/backup_rules"
        android:icon="@mipmap/ic_launcher"
        android:label="@string/app_name"
        android:roundIcon="@mipmap/ic_launcher_round"
        android:supportsRtl="true"
        android:theme="@style/Theme.Energy"
        tools:targetApi="34">
        <activity
            android:name=".MainActivity"
            android:exported="true"
            android:label="@string/app_name"
            android:theme="@style/Theme.EnergyTest">
            <intent-filter>
                <action android:name="android.intent.action.MAIN" />
                <category android:name="android.intent.category.LAUNCHER" />
            </intent-filter>

            <intent-filter>
                <action android:name="android.media.action.MEDIA_PLAY_FROM_SEARCH" />
                <category android:name="android.intent.category.DEFAULT" />
            </intent-filter>
        </activity>

        <service
            android:name=".player.MusicService"
            android:enabled="true"
            android:exported="true"
            android:foregroundServiceType="mediaPlayback"
            tools:ignore="ExportedService">

            <intent-filter>
                <action android:name="androidx.media3.session.MediaLibraryService"/>
                <action android:name="android.media.browse.MediaBrowserService" />
            </intent-filter>
        </service>
    </application>

</manifest>

I have tried it with Media3 library version 1.1.1 and 1.2.0-beta01, but with the same behavior. Neither onSearch, onGetSearchResult, onGetChildren, onGetItem, onGetLibraryRoot or onConnect are called. So I think, there might be a problem with advertising my app to the Google Assistant.

Are some additional steps required to make this work?

I have read about BII's but there seems to be no BII that is related to media and according to the documentation, this should somehow work magically out of the box with Media3.

[NikSatyr]

we've faced this issue a while ago, and this problem was coming not from the library, but from the Google Assistant indexing. as far as I remember, in order for the Assistant to pick up your app, your app should be installed from the Play Store (which probably has some indexing trigger when uploading the app to the Play Console). this should work for the Alpha (and probably Closed Testing) track, so there's no need to push the build to production

in the meantime, we were relying on Media Controller Test app (https://developer.android.com/media/optimize/mct#testing_prepare_and_play) to check that the commands work at all, and after this primary testing we continued with testing the build from the Play Store

[andreas-umbricht]

Thanks for your interesting input.

I tried it by downloading the app from Google Play on the internal test track as well as the closed testing track but the issue still persists. On the mentioned Media Controller Test app, everything seems to work fine. Also the Media Browser works as desired.

I am quite sure that about 2 years ago, when the app was initially created, starting streams with the Google Assistant used to work somehow. Since then, Google introduced Google Media Actions ([https://developers.google.com/actions/media]). I could imagine that our desired behavior is now only available or overwritten by these Media Actions, which would be bad, because as it is written in the documentation "Currently, Google is working with a limited number of providers at a time...". We tried to contact the support to sign up as such a provider, but sadly had a negative response.

Does anybody else have some issue here about that? For me (Pixel 7, Android 14) Google Assistant does not even work for UAMP.

[NikSatyr]

that's weird but understandable as Google Assistant is basically a black box. however, I'd like to point out that we did not register as a Google partner of any sort nor used the mentioned Google Media Actions, but our app is perfectly able to receive & handle "play artist" voice commands. we do not include our app name in the command, as it is hard for the voice recognition to handle

in addition to installing the app from the Play Store, my other suggestion would be to ensure that you have your media session active prior to issuing any voice commands, e.g. manually starting the playback from within your app, and then issuing the voice command to play smth. this seems to work quite reliably for us even without specifying our app name in the command

if the info above does not help, I would also be curious get a deeper explanation from the media3 devs – I'm hoping they can shed some light here

[NielsMasdorp]

What has worked for me in Android Auto voice control, which I presume uses the same setup als Google Assistant is this:

override fun onSearch(
        session: MediaLibrarySession,
        browser: MediaSession.ControllerInfo,
        query: String,
        params: LibraryParams?
    ): ListenableFuture<LibraryResult<Void>> {
        return serviceScope.future {
            mediaSession.notifySearchResultChanged(
                /* browser */ browser,
                /* query */ query,
                /* itemCount */ contentTree.await().search(query = query).size,
                /* params */ params
            )
            LibraryResult.ofVoid()
        }
    }

override fun onGetSearchResult(
        session: MediaLibrarySession,
        browser: MediaSession.ControllerInfo,
        query: String,
        page: Int,
        pageSize: Int,
        params: LibraryParams?
    ): ListenableFuture<LibraryResult<ImmutableList<MediaItem>>> {
        return serviceScope.future {
            val results = contentTree.await().search(query = query)
            val fromIndex = max((page - 1) * pageSize, results.size - 1)
            val toIndex = max(fromIndex + pageSize, results.size)
            LibraryResult.ofItemList(
                /* items */ results.subList(fromIndex, toIndex),
                /* params */ params
            )
        }
    }

Which then results in a MediaItem without id and query in its requestMetadata which in turn can be used to find the "real" item or items.

override fun onSetMediaItems(
        mediaSession: MediaSession,
        controller: MediaSession.ControllerInfo,
        mediaItems: MutableList<MediaItem>,
        startIndex: Int,
        startPositionMs: Long
    ): ListenableFuture<MediaItemsWithStartPosition> {
        return serviceScope.future {
            val contentTree = contentTree.await()
            if (mediaItems.size == 1) {
                val singleItem = mediaItems[0]
                val itemId = singleItem.mediaId
                if (itemId.isBlank() && singleItem.requestMetadata.searchQuery != null) {
                    // Voice search -> return search results
                    val streams = contentTree
                        .search(query = singleItem.requestMetadata.searchQuery)
                        .toMutableList()
                    MediaItemsWithStartPosition(streams, startIndex, startPositionMs)
                } else {
                    // not relevant
                }
            } else {
                // not relevant
            }
        }
    }

My use case is the same, radio stations.