diff --git a/.changeset/cold-plants-push.md b/.changeset/cold-plants-push.md new file mode 100644 index 000000000..f21653596 --- /dev/null +++ b/.changeset/cold-plants-push.md @@ -0,0 +1,5 @@ +--- +"client-sdk-android": minor +--- + +Add `LocalAudioTrack.applyOptions`, which allows updating the audio track options on the fly. diff --git a/.changeset/cuddly-mugs-compare.md b/.changeset/cuddly-mugs-compare.md new file mode 100644 index 000000000..696ca1705 --- /dev/null +++ b/.changeset/cuddly-mugs-compare.md @@ -0,0 +1,5 @@ +--- +"client-sdk-android": patch +--- + +Fix custom LocalAudioTrackOptions not applying correctly diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6c1cec46a..6c3caf7b0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,5 @@ [versions] -webrtc = "144.7559.05" +webrtc = "144.7559.09" androidJainSipRi = "1.3.0-91" androidx-activity = "1.9.0" diff --git a/livekit-android-sdk/detekt-baseline-release.xml b/livekit-android-sdk/detekt-baseline-release.xml index 418e8ea50..02fa6d27a 100644 --- a/livekit-android-sdk/detekt-baseline-release.xml +++ b/livekit-android-sdk/detekt-baseline-release.xml @@ -30,7 +30,6 @@ CyclomaticComplexMethod:SignalClient.kt$SignalClient$private fun handleSignalResponseImpl(ws: WebSocket, response: LivekitRtc.SignalResponse) EmptyFunctionBlock:RTCEngine.kt$RTCEngine${ } HasPlatformType:DataChannelManager.kt$DataChannelManager$@get:FlowObservable var state by flowDelegate(dataChannel.state()) private set - IgnoredReturnValue:BaseStreamReceiver.kt$BaseStreamReceiver$catch { } IgnoredReturnValue:RpcServerManager.kt$RpcServerManager$publishRpcAck(callerIdentity, requestId) InstanceOfCheckForException:RpcServerManager.kt$RpcServerManager$e is RpcError LargeClass:LocalParticipant.kt$LocalParticipant : ParticipantOutgoingDataStreamManagerRpcManager @@ -42,7 +41,7 @@ LongParameterList:AudioBufferCallbackDispatcher.kt$AudioBufferCallback$(buffer: ByteBuffer, audioFormat: Int, channelCount: Int, sampleRate: Int, bytesRead: Int, captureTimeNs: Long) LongParameterList:KeyProvider.kt$BaseKeyProvider$( ratchetSalt: String = defaultRatchetSalt, uncryptedMagicBytes: String = defaultMagicBytes, ratchetWindowSize: Int = defaultRatchetWindowSize, override var enableSharedKey: Boolean = true, failureTolerance: Int = defaultFailureTolerance, keyRingSize: Int = defaultKeyRingSize, discardFrameWhenCryptorNotReady: Boolean = defaultDiscardFrameWhenCryptorNotReady, keyDerivationAlgorithm: FrameCryptorKeyDerivationAlgorithm = defaultKeyDerivationAlgorithm, ) LongParameterList:LiveKitOverrides.kt$AudioOptions$( /** * Override the default output [AudioType]. * * This affects the audio routing and how the audio is handled. Default is [AudioType.CallAudioType]. * * Note: if [audioHandler] is also passed, the values from [audioOutputType] will not be reflected in it, * and must be set yourself. */ val audioOutputType: AudioType? = null, /** * Override the default [AudioHandler]. * * Default is [AudioSwitchHandler]. * * Use [NoAudioHandler] to turn off automatic audio handling or * [AudioFocusHandler] to get simple audio focus handling. */ val audioHandler: AudioHandler? = null, /** * Override the default [AudioDeviceModule]. * * If a non-null value is passed, the library does not * take ownership of the object and will not release it upon [Room.release]. * It is the responsibility of the owner to call [AudioDeviceModule.release] when finished * with it to prevent memory leaks. */ val audioDeviceModule: AudioDeviceModule? = null, /** * Called after default setup to allow for customizations on the [JavaAudioDeviceModule]. * * Not used if [audioDeviceModule] is provided. * * Note: We require setting the [JavaAudioDeviceModule.Builder.setSamplesReadyCallback] to provide * support for [LocalAudioTrack.addSink]. If you wish to grab the audio samples * from the local microphone track, use [LocalAudioTrack.addSink] instead of setting your own * callback. */ val javaAudioDeviceModuleCustomizer: ((builder: JavaAudioDeviceModule.Builder) -> Unit)? = null, /** * On Android 11+, the audio mode will reset itself from [AudioManager.MODE_IN_COMMUNICATION] if * there is no audio playback or capture for 6 seconds (for example when joining a room with * no speakers and the local mic is muted.) This mode reset will cause unexpected * behavior when trying to change the volume, causing it to not properly change the volume. * * We use a workaround by playing a silent audio track to keep the communication mode from * resetting. * * Setting this flag to true will disable the workaround. * * This flag is a no-op when the audio mode is set to anything other than * [AudioManager.MODE_IN_COMMUNICATION]. */ val disableCommunicationModeWorkaround: Boolean = false, /** * Options for processing the mic and incoming audio. */ val audioProcessorOptions: AudioProcessorOptions? = null, /** * Devices may take some time initializing the audio stack for recording. * Prewarming allows starting up the underlying audio recording prior to publish, letting * the audio device be ready immediately when the track is fully published. * * If set to true, disables audio recording prewarming (and the related * [LocalAudioTrack.prewarm] function), and audio resources are only used while the * track is connected and published. Defaults to false. */ val disableAudioPrewarming: Boolean = false, ) - LongParameterList:LocalAudioTrack.kt$LocalAudioTrack$( @Assisted name: String, @Assisted mediaTrack: livekit.org.webrtc.AudioTrack, @Assisted private val options: LocalAudioTrackOptions, private val audioProcessingController: AudioProcessingController, @Named(InjectionNames.DISPATCHER_DEFAULT) private val dispatcher: CoroutineDispatcher, @Named(InjectionNames.LOCAL_AUDIO_RECORD_SAMPLES_DISPATCHER) private val audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher, @Named(InjectionNames.LOCAL_AUDIO_BUFFER_CALLBACK_DISPATCHER) private val audioBufferCallbackDispatcher: AudioBufferCallbackDispatcher, private val audioRecordPrewarmer: AudioRecordPrewarmer, rtcThreadToken: RTCThreadToken, ) + LongParameterList:LocalAudioTrack.kt$LocalAudioTrack$( @Assisted name: String, @Assisted mediaTrack: livekit.org.webrtc.AudioTrack, @Assisted options: LocalAudioTrackOptions, private val audioProcessingController: AudioProcessingController, @Named(InjectionNames.DISPATCHER_DEFAULT) private val dispatcher: CoroutineDispatcher, @Named(InjectionNames.LOCAL_AUDIO_RECORD_SAMPLES_DISPATCHER) private val audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher, @Named(InjectionNames.LOCAL_AUDIO_BUFFER_CALLBACK_DISPATCHER) private val audioBufferCallbackDispatcher: AudioBufferCallbackDispatcher, private val audioRecordPrewarmer: AudioRecordPrewarmer, rtcThreadToken: RTCThreadToken, ) LongParameterList:LocalParticipant.kt$LocalParticipant$( @Assisted internal var dynacast: Boolean, internal val engine: RTCEngine, private val peerConnectionFactory: PeerConnectionFactory, private val context: Context, private val eglBase: EglBase, private val screencastVideoTrackFactory: LocalScreencastVideoTrack.Factory, private val videoTrackFactory: LocalVideoTrack.Factory, private val audioTrackFactory: LocalAudioTrack.Factory, private val defaultsManager: DefaultsManager, @Named(InjectionNames.DISPATCHER_DEFAULT) coroutineDispatcher: CoroutineDispatcher, @Named(InjectionNames.SENDER) private val capabilitiesGetter: CapabilitiesGetter, private val outgoingDataStreamManager: OutgoingDataStreamManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, ) LongParameterList:LocalScreencastVideoTrack.kt$LocalScreencastVideoTrack$( @Assisted capturer: VideoCapturer, @Assisted source: VideoSource, @Assisted name: String, @Assisted options: LocalVideoTrackOptions, @Assisted rtcTrack: livekit.org.webrtc.VideoTrack, @Assisted mediaProjectionCallback: MediaProjectionCallback, peerConnectionFactory: PeerConnectionFactory, context: Context, eglBase: EglBase, defaultsManager: DefaultsManager, videoTrackFactory: LocalVideoTrack.Factory, rtcThreadToken: RTCThreadToken, ) LongParameterList:LocalScreencastVideoTrack.kt$LocalScreencastVideoTrack.Companion$( mediaProjectionPermissionResultData: Intent, peerConnectionFactory: PeerConnectionFactory, context: Context, name: String, options: LocalVideoTrackOptions, rootEglBase: EglBase, screencastVideoTrackFactory: Factory, videoProcessor: VideoProcessor?, onStop: (Track) -> Unit, ) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/audio/AudioRecordPrewarmer.kt b/livekit-android-sdk/src/main/java/io/livekit/android/audio/AudioRecordPrewarmer.kt index 98c43ab64..1f0e47062 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/audio/AudioRecordPrewarmer.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/audio/AudioRecordPrewarmer.kt @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit, Inc. + * Copyright 2025-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,15 @@ package io.livekit.android.audio +import io.livekit.android.room.track.LocalAudioTrackOptions +import livekit.org.webrtc.audio.AudioProcessingOptions import livekit.org.webrtc.audio.JavaAudioDeviceModule /** * @suppress */ interface AudioRecordPrewarmer { - fun prewarm() + fun prewarm(options: LocalAudioTrackOptions) fun stop() } @@ -30,7 +32,7 @@ interface AudioRecordPrewarmer { * @suppress */ class NoAudioRecordPrewarmer : AudioRecordPrewarmer { - override fun prewarm() { + override fun prewarm(options: LocalAudioTrackOptions) { // nothing to do. } @@ -43,8 +45,15 @@ class NoAudioRecordPrewarmer : AudioRecordPrewarmer { * @suppress */ class JavaAudioRecordPrewarmer(private val audioDeviceModule: JavaAudioDeviceModule) : AudioRecordPrewarmer { - override fun prewarm() { - audioDeviceModule.prewarmRecording() + override fun prewarm(options: LocalAudioTrackOptions) { + audioDeviceModule.prewarmRecording( + AudioProcessingOptions( + options.echoCancellation, + options.noiseSuppression, + options.autoGainControl, + options.highPassFilter, + ), + ) } override fun stop() { diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrack.kt index de39af071..38b69b231 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrack.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrack.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2025 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,7 +41,6 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.combine -import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.stateIn import livekit.LivekitModels.AudioTrackFeature import livekit.org.webrtc.AudioTrackSink @@ -50,6 +49,7 @@ import livekit.org.webrtc.PeerConnectionFactory import livekit.org.webrtc.RtpSender import livekit.org.webrtc.RtpTransceiver import livekit.org.webrtc.audio.AudioDeviceModule +import livekit.org.webrtc.audio.AudioProcessingOptionsResult import livekit.org.webrtc.audio.JavaAudioDeviceModule import java.util.UUID import javax.inject.Named @@ -64,7 +64,7 @@ class LocalAudioTrack constructor( @Assisted name: String, @Assisted mediaTrack: livekit.org.webrtc.AudioTrack, - @Assisted private val options: LocalAudioTrackOptions, + @Assisted options: LocalAudioTrackOptions, private val audioProcessingController: AudioProcessingController, @Named(InjectionNames.DISPATCHER_DEFAULT) private val dispatcher: CoroutineDispatcher, @@ -86,11 +86,22 @@ constructor( private val trackSinks = mutableSetOf() + /** + * The current capture processing options for this track. + * + * Changes can be observed by using [io.livekit.android.util.flow] + */ + @FlowObservable + @get:FlowObservable + var options: LocalAudioTrackOptions by flowDelegate(options) + /** * Prewarms the audio stack if needed by starting the recording regardless of whether it's being published. + * + * Platform AEC/NS are configured from [LocalAudioTrackOptions] before the audio session starts. */ fun prewarm() { - audioRecordPrewarmer.prewarm() + audioRecordPrewarmer.prewarm(options) } fun stopPrewarm() { @@ -129,6 +140,37 @@ constructor( audioBufferCallbackDispatcher.bufferCallback = callback } + /** + * Updates the capture processing options on this track. + * + * Note: [LocalAudioTrackOptions.typingNoiseDetection] is only applied at track creation time. + * + * Example: + * ``` + * val track = localParticipant.getTrackPublication(Track.Source.MICROPHONE)?.track as? LocalAudioTrack + * track?.applyOptions(LocalAudioTrackOptions(echoCancellation = false)) + * ``` + */ + fun applyOptions(options: LocalAudioTrackOptions): Result { + val result = withRTCTrack(null as AudioProcessingOptionsResult?) { + (this as livekit.org.webrtc.AudioTrack).setAudioProcessingOptions(options.toAudioProcessingOptions()) + } ?: return Result.failure( + TrackException.InvalidTrackStateException("Cannot apply options to a disposed track"), + ) + + if (!result.isSuccess) { + return Result.failure( + TrackException.MediaException( + result.message?.takeIf { it.isNotEmpty() } + ?: "Failed to apply audio processing options (${result.code})", + ), + ) + } + + this.options = options + return Result.success(Unit) + } + /** * Changes can be observed by using [io.livekit.android.util.flow] */ @@ -136,23 +178,20 @@ constructor( @get:FlowObservable val features by flowDelegate( stateFlow = combine( + ::options.flow, audioProcessingController::capturePostProcessor.flow, audioProcessingController::bypassCapturePostProcessing.flow, - ) { processor, bypass -> - processor to bypass - } - .map { - val features = getConstantFeatures() - val (processor, bypass) = it - if (!bypass && processor?.getName() == "krisp_noise_cancellation") { - features.add(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION) - } - return@map features + ) { opts, processor, bypass -> + val features = getConstantFeatures(opts) + if (!bypass && processor?.getName() == "krisp_noise_cancellation") { + features.add(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION) } + features + } .stateIn(delegateScope, SharingStarted.Eagerly, emptySet()), ) - private fun getConstantFeatures(): MutableSet { + private fun getConstantFeatures(options: LocalAudioTrackOptions): MutableSet { val features = mutableSetOf() if (options.echoCancellation) { diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrackOptions.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrackOptions.kt index 00bcfd2bf..25be2db8b 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrackOptions.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrackOptions.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package io.livekit.android.room.track +import livekit.org.webrtc.audio.AudioProcessingOptions + data class LocalAudioTrackOptions( val noiseSuppression: Boolean = true, val echoCancellation: Boolean = true, @@ -23,3 +25,12 @@ data class LocalAudioTrackOptions( val highPassFilter: Boolean = true, val typingNoiseDetection: Boolean = true, ) + +internal fun LocalAudioTrackOptions.toAudioProcessingOptions(): AudioProcessingOptions { + return AudioProcessingOptions( + echoCancellation, + noiseSuppression, + autoGainControl, + highPassFilter, + ) +} diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockAudioStreamTrack.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockAudioStreamTrack.kt index 52ea301db..61f4cad90 100644 --- a/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockAudioStreamTrack.kt +++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockAudioStreamTrack.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,8 @@ package io.livekit.android.test.mock import livekit.org.webrtc.AudioTrack +import livekit.org.webrtc.audio.AudioProcessingOptions +import livekit.org.webrtc.audio.AudioProcessingOptionsResult class MockAudioStreamTrack( val id: String = "id", @@ -26,6 +28,13 @@ class MockAudioStreamTrack( ) : AudioTrack(1L) { var disposed = false + var lastAudioProcessingOptions: AudioProcessingOptions? = null + var audioProcessingOptionsResult: AudioProcessingOptionsResult = AudioProcessingOptionsResult.stored() + + override fun setAudioProcessingOptions(options: AudioProcessingOptions): AudioProcessingOptionsResult { + lastAudioProcessingOptions = options + return audioProcessingOptionsResult + } override fun id(): String = id @@ -49,6 +58,8 @@ class MockAudioStreamTrack( disposed = true } + override fun isDisposed(): Boolean = disposed + override fun setVolume(volume: Double) { } } diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/participant/LocalParticipantMockE2ETest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/participant/LocalParticipantMockE2ETest.kt index 0048872d6..cb1bef245 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/participant/LocalParticipantMockE2ETest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/participant/LocalParticipantMockE2ETest.kt @@ -21,7 +21,6 @@ import android.app.Application import android.content.Context import androidx.test.core.app.ApplicationProvider import com.google.protobuf.ByteString -import io.livekit.android.audio.AudioProcessorInterface import io.livekit.android.events.ParticipantEvent import io.livekit.android.events.RoomEvent import io.livekit.android.room.DefaultsManager @@ -39,7 +38,6 @@ import io.livekit.android.test.MockE2ETest import io.livekit.android.test.assert.assertIsClassList import io.livekit.android.test.coroutines.toListUntilSignal import io.livekit.android.test.events.EventCollector -import io.livekit.android.test.mock.MockAudioProcessingController import io.livekit.android.test.mock.MockDataChannel import io.livekit.android.test.mock.MockEglBase import io.livekit.android.test.mock.MockRTCThreadToken @@ -61,7 +59,6 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.test.StandardTestDispatcher import kotlinx.coroutines.test.advanceUntilIdle import livekit.LivekitModels -import livekit.LivekitModels.AudioTrackFeature import livekit.LivekitModels.DataPacket import livekit.LivekitRtc import livekit.LivekitRtc.SubscribedCodec @@ -79,7 +76,6 @@ import org.mockito.Mockito.mock import org.mockito.kotlin.argThat import org.robolectric.RobolectricTestRunner import org.robolectric.Shadows -import java.nio.ByteBuffer import kotlin.time.Duration.Companion.seconds @ExperimentalCoroutinesApi @@ -679,116 +675,6 @@ class LocalParticipantMockE2ETest : MockE2ETest() { assertEquals(preference, transceiver.sender.parameters.degradationPreference) } - @Test - fun sendsInitialAudioTrackFeatures() = runTest { - connect() - - wsFactory.ws.clearRequests() - room.localParticipant.publishAudioTrack( - track = createMockLocalAudioTrack(), - ) - - advanceUntilIdle() - assertEquals(2, wsFactory.ws.sentRequests.size) - - // Verify the update audio track request gets the proper publish options set. - val requestString = wsFactory.ws.sentRequests[1].toPBByteString() - val sentRequest = LivekitRtc.SignalRequest.newBuilder() - .mergeFrom(requestString) - .build() - - assertTrue(sentRequest.hasUpdateAudioTrack()) - val features = sentRequest.updateAudioTrack.featuresList - assertEquals(3, features.size) - assertTrue(features.contains(AudioTrackFeature.TF_ECHO_CANCELLATION)) - assertTrue(features.contains(AudioTrackFeature.TF_NOISE_SUPPRESSION)) - assertTrue(features.contains(AudioTrackFeature.TF_AUTO_GAIN_CONTROL)) - } - - @Test - fun sendsUpdatedAudioTrackFeatures() = runTest { - connect() - - val audioProcessingController = MockAudioProcessingController() - room.localParticipant.publishAudioTrack( - track = createMockLocalAudioTrack(audioProcessingController = audioProcessingController), - ) - - advanceUntilIdle() - wsFactory.ws.clearRequests() - - audioProcessingController.capturePostProcessor = object : AudioProcessorInterface { - override fun isEnabled(): Boolean = true - - override fun getName(): String = "krisp_noise_cancellation" - - override fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int) {} - - override fun resetAudioProcessing(newRate: Int) {} - - override fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer) {} - } - assertEquals(1, wsFactory.ws.sentRequests.size) - - // Verify the update audio track request gets the proper publish options set. - val requestString = wsFactory.ws.sentRequests[0].toPBByteString() - val sentRequest = LivekitRtc.SignalRequest.newBuilder() - .mergeFrom(requestString) - .build() - - assertTrue(sentRequest.hasUpdateAudioTrack()) - val features = sentRequest.updateAudioTrack.featuresList - assertEquals(4, features.size) - assertTrue(features.contains(AudioTrackFeature.TF_ECHO_CANCELLATION)) - assertTrue(features.contains(AudioTrackFeature.TF_NOISE_SUPPRESSION)) - assertTrue(features.contains(AudioTrackFeature.TF_AUTO_GAIN_CONTROL)) - assertTrue(features.contains(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION)) - } - - @Test - fun bypassUpdatesAudioFeatures() = runTest { - connect() - - val audioProcessingController = MockAudioProcessingController() - room.localParticipant.publishAudioTrack( - track = createMockLocalAudioTrack(audioProcessingController = audioProcessingController), - ) - - advanceUntilIdle() - wsFactory.ws.clearRequests() - - audioProcessingController.capturePostProcessor = object : AudioProcessorInterface { - override fun isEnabled(): Boolean = true - - override fun getName(): String = "krisp_noise_cancellation" - - override fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int) {} - - override fun resetAudioProcessing(newRate: Int) {} - - override fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer) {} - } - assertEquals(1, wsFactory.ws.sentRequests.size) - - wsFactory.ws.clearRequests() - - audioProcessingController.bypassCapturePostProcessing = true - assertEquals(1, wsFactory.ws.sentRequests.size) - // Verify the update audio track request gets the proper publish options set. - val requestString = wsFactory.ws.sentRequests[0].toPBByteString() - val sentRequest = LivekitRtc.SignalRequest.newBuilder() - .mergeFrom(requestString) - .build() - - assertTrue(sentRequest.hasUpdateAudioTrack()) - val features = sentRequest.updateAudioTrack.featuresList - assertEquals(3, features.size) - assertTrue(features.contains(AudioTrackFeature.TF_ECHO_CANCELLATION)) - assertTrue(features.contains(AudioTrackFeature.TF_NOISE_SUPPRESSION)) - assertTrue(features.contains(AudioTrackFeature.TF_AUTO_GAIN_CONTROL)) - assertFalse(features.contains(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION)) - } - @Test fun lackOfPublishPermissionReturnsFalse() = runTest { val noCanPublishJoin = with(TestData.JOIN.toBuilder()) { diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/track/LocalAudioTrackMockE2ETest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/track/LocalAudioTrackMockE2ETest.kt new file mode 100644 index 000000000..6118c34dc --- /dev/null +++ b/livekit-android-test/src/test/java/io/livekit/android/room/track/LocalAudioTrackMockE2ETest.kt @@ -0,0 +1,430 @@ +/* + * Copyright 2023-2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track + +import io.livekit.android.audio.AudioProcessorInterface +import io.livekit.android.test.MockE2ETest +import io.livekit.android.test.assert.assertIsClass +import io.livekit.android.test.coroutines.toListUntilSignal +import io.livekit.android.test.mock.MockAudioProcessingController +import io.livekit.android.test.mock.MockAudioStreamTrack +import io.livekit.android.test.mock.room.track.createMockLocalAudioTrack +import io.livekit.android.test.util.toPBByteString +import io.livekit.android.util.flow +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.test.advanceUntilIdle +import livekit.LivekitModels.AudioTrackFeature +import livekit.LivekitRtc +import livekit.org.webrtc.audio.AudioProcessingOptionsResult +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.nio.ByteBuffer + +@ExperimentalCoroutinesApi +@RunWith(RobolectricTestRunner::class) +class LocalAudioTrackMockE2ETest : MockE2ETest() { + + @Test + fun defaultFeatures() = runTest { + val track = createMockLocalAudioTrack() + advanceUntilIdle() + + assertConstantFeatures(track.features) + } + + @Test + fun featuresReflectInitialOptions() = runTest { + val noEcho = createMockLocalAudioTrack( + options = LocalAudioTrackOptions(echoCancellation = false), + ) + advanceUntilIdle() + assertEquals( + setOf( + AudioTrackFeature.TF_NOISE_SUPPRESSION, + AudioTrackFeature.TF_AUTO_GAIN_CONTROL, + ), + noEcho.features, + ) + + val noNoiseSuppression = createMockLocalAudioTrack( + options = LocalAudioTrackOptions(noiseSuppression = false), + ) + advanceUntilIdle() + assertEquals( + setOf( + AudioTrackFeature.TF_ECHO_CANCELLATION, + AudioTrackFeature.TF_AUTO_GAIN_CONTROL, + ), + noNoiseSuppression.features, + ) + + val noAutoGainControl = createMockLocalAudioTrack( + options = LocalAudioTrackOptions(autoGainControl = false), + ) + advanceUntilIdle() + assertEquals( + setOf( + AudioTrackFeature.TF_ECHO_CANCELLATION, + AudioTrackFeature.TF_NOISE_SUPPRESSION, + ), + noAutoGainControl.features, + ) + } + + @Test + fun featuresWithAllProcessingDisabled() = runTest { + val track = createMockLocalAudioTrack( + options = LocalAudioTrackOptions( + echoCancellation = false, + noiseSuppression = false, + autoGainControl = false, + ), + ) + advanceUntilIdle() + + assertTrue(track.features.isEmpty()) + } + + @Test + fun featuresIncludeEnhancedNoiseCancellationForKrisp() = runTest { + val audioProcessingController = MockAudioProcessingController() + val track = createMockLocalAudioTrack(audioProcessingController = audioProcessingController) + advanceUntilIdle() + assertConstantFeatures(track.features) + + audioProcessingController.capturePostProcessor = krispProcessor() + advanceUntilIdle() + + assertEquals(4, track.features.size) + assertTrue(track.features.contains(AudioTrackFeature.TF_ECHO_CANCELLATION)) + assertTrue(track.features.contains(AudioTrackFeature.TF_NOISE_SUPPRESSION)) + assertTrue(track.features.contains(AudioTrackFeature.TF_AUTO_GAIN_CONTROL)) + assertTrue(track.features.contains(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION)) + } + + @Test + fun featuresExcludeEnhancedNoiseCancellationForOtherProcessor() = runTest { + val audioProcessingController = MockAudioProcessingController() + val track = createMockLocalAudioTrack(audioProcessingController = audioProcessingController) + advanceUntilIdle() + + audioProcessingController.capturePostProcessor = otherProcessor() + advanceUntilIdle() + + assertConstantFeatures(track.features) + assertFalse(track.features.contains(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION)) + } + + @Test + fun featuresFlowEmitsOnApplyOptions() = runTest { + val track = createMockLocalAudioTrack() + val signal = MutableStateFlow(null) + val job = async { + track::features.flow.toListUntilSignal(signal) + } + + advanceUntilIdle() + track.applyOptions(LocalAudioTrackOptions(echoCancellation = false)) + advanceUntilIdle() + + signal.compareAndSet(null, Unit) + val collectedList = job.await() + + assertEquals(2, collectedList.size) + assertConstantFeatures(collectedList[0]) + assertEquals( + setOf( + AudioTrackFeature.TF_NOISE_SUPPRESSION, + AudioTrackFeature.TF_AUTO_GAIN_CONTROL, + ), + collectedList[1], + ) + } + + @Test + fun featuresFlowEmitsOnProcessorChange() = runTest { + val audioProcessingController = MockAudioProcessingController() + val track = createMockLocalAudioTrack(audioProcessingController = audioProcessingController) + val signal = MutableStateFlow(null) + val job = async { + track::features.flow.toListUntilSignal(signal) + } + + advanceUntilIdle() + audioProcessingController.capturePostProcessor = krispProcessor() + advanceUntilIdle() + + signal.compareAndSet(null, Unit) + val collectedList = job.await() + + assertEquals(2, collectedList.size) + assertConstantFeatures(collectedList[0]) + assertTrue(collectedList[1].contains(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION)) + } + + @Test + fun featuresFlowEmitsOnBypass() = runTest { + val audioProcessingController = MockAudioProcessingController() + val track = createMockLocalAudioTrack(audioProcessingController = audioProcessingController) + val signal = MutableStateFlow(null) + val job = async { + track::features.flow.toListUntilSignal(signal) + } + + advanceUntilIdle() + audioProcessingController.capturePostProcessor = krispProcessor() + advanceUntilIdle() + audioProcessingController.bypassCapturePostProcessing = true + advanceUntilIdle() + + signal.compareAndSet(null, Unit) + val collectedList = job.await() + + assertEquals(3, collectedList.size) + assertConstantFeatures(collectedList[0]) + assertTrue(collectedList[1].contains(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION)) + assertConstantFeatures(collectedList[2]) + assertFalse(collectedList[2].contains(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION)) + } + + @Test + fun applyOptionsUpdatesFeatures() = runTest { + val track = createMockLocalAudioTrack() + advanceUntilIdle() + assertConstantFeatures(track.features) + + val updatedOptions = LocalAudioTrackOptions(echoCancellation = false) + assertTrue(track.applyOptions(updatedOptions).isSuccess) + advanceUntilIdle() + + assertEquals(updatedOptions, track.options) + assertEquals( + setOf( + AudioTrackFeature.TF_NOISE_SUPPRESSION, + AudioTrackFeature.TF_AUTO_GAIN_CONTROL, + ), + track.features, + ) + } + + @Test + fun applyOptionsPassesProcessingOptionsToRtcTrack() = runTest { + val mediaTrack = MockAudioStreamTrack() + val track = createMockLocalAudioTrack(mediaTrack = mediaTrack) + val updatedOptions = LocalAudioTrackOptions( + echoCancellation = false, + noiseSuppression = false, + autoGainControl = true, + highPassFilter = false, + ) + + assertTrue(track.applyOptions(updatedOptions).isSuccess) + + val appliedOptions = mediaTrack.lastAudioProcessingOptions + assertEquals(false, appliedOptions?.echoCancellation) + assertEquals(false, appliedOptions?.noiseSuppression) + assertEquals(true, appliedOptions?.autoGainControl) + assertEquals(false, appliedOptions?.highPassFilter) + } + + @Test + fun applyOptionsFailurePreservesFeatures() = runTest { + val mediaTrack = MockAudioStreamTrack().apply { + audioProcessingOptionsResult = AudioProcessingOptionsResult.rejected( + AudioProcessingOptionsResult.Code.APPLY_FAILED, + "failed", + ) + } + val track = createMockLocalAudioTrack(mediaTrack = mediaTrack) + advanceUntilIdle() + val originalFeatures = track.features + + val result = track.applyOptions(LocalAudioTrackOptions(echoCancellation = false)) + advanceUntilIdle() + + assertTrue(result.isFailure) + assertIsClass(TrackException.MediaException::class.java, result.exceptionOrNull()) + assertEquals(originalFeatures, track.features) + assertTrue(track.options.echoCancellation) + } + + @Test + fun applyOptionsFailureUsesFallbackMessage() = runTest { + val mediaTrack = MockAudioStreamTrack().apply { + audioProcessingOptionsResult = AudioProcessingOptionsResult.rejected( + AudioProcessingOptionsResult.Code.APPLY_FAILED, + "", + ) + } + val track = createMockLocalAudioTrack(mediaTrack = mediaTrack) + + val result = track.applyOptions(LocalAudioTrackOptions(echoCancellation = false)) + + assertTrue(result.isFailure) + assertIsClass(TrackException.MediaException::class.java, result.exceptionOrNull()) + assertEquals( + "Failed to apply audio processing options (${AudioProcessingOptionsResult.Code.APPLY_FAILED})", + result.exceptionOrNull()?.message, + ) + } + + @Test + fun applyOptionsOnDisposedTrackFails() = runTest { + val track = createMockLocalAudioTrack() + track.dispose() + + val result = track.applyOptions(LocalAudioTrackOptions(echoCancellation = false)) + + assertTrue(result.isFailure) + assertIsClass(TrackException.InvalidTrackStateException::class.java, result.exceptionOrNull()) + } + + @Test + fun sendsInitialAudioTrackFeatures() = runTest { + connect() + + wsFactory.ws.clearRequests() + room.localParticipant.publishAudioTrack( + track = createMockLocalAudioTrack(), + ) + + advanceUntilIdle() + assertEquals(2, wsFactory.ws.sentRequests.size) + + val features = lastUpdateAudioTrackFeatures() + assertConstantFeatures(features) + } + + @Test + fun sendsUpdatedAudioTrackFeatures() = runTest { + connect() + + val audioProcessingController = MockAudioProcessingController() + room.localParticipant.publishAudioTrack( + track = createMockLocalAudioTrack(audioProcessingController = audioProcessingController), + ) + + advanceUntilIdle() + wsFactory.ws.clearRequests() + + audioProcessingController.capturePostProcessor = krispProcessor() + assertEquals(1, wsFactory.ws.sentRequests.size) + + val features = lastUpdateAudioTrackFeatures() + assertEquals(4, features.size) + assertTrue(features.contains(AudioTrackFeature.TF_ECHO_CANCELLATION)) + assertTrue(features.contains(AudioTrackFeature.TF_NOISE_SUPPRESSION)) + assertTrue(features.contains(AudioTrackFeature.TF_AUTO_GAIN_CONTROL)) + assertTrue(features.contains(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION)) + } + + @Test + fun bypassUpdatesAudioFeatures() = runTest { + connect() + + val audioProcessingController = MockAudioProcessingController() + room.localParticipant.publishAudioTrack( + track = createMockLocalAudioTrack(audioProcessingController = audioProcessingController), + ) + + advanceUntilIdle() + wsFactory.ws.clearRequests() + + audioProcessingController.capturePostProcessor = krispProcessor() + assertEquals(1, wsFactory.ws.sentRequests.size) + + wsFactory.ws.clearRequests() + + audioProcessingController.bypassCapturePostProcessing = true + assertEquals(1, wsFactory.ws.sentRequests.size) + + val features = lastUpdateAudioTrackFeatures() + assertConstantFeatures(features) + assertFalse(features.contains(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION)) + } + + @Test + fun sendsUpdatedFeaturesOnApplyOptions() = runTest { + connect() + + val track = createMockLocalAudioTrack() + room.localParticipant.publishAudioTrack(track = track) + + advanceUntilIdle() + wsFactory.ws.clearRequests() + + assertTrue(track.applyOptions(LocalAudioTrackOptions(echoCancellation = false)).isSuccess) + advanceUntilIdle() + assertEquals(1, wsFactory.ws.sentRequests.size) + + val features = lastUpdateAudioTrackFeatures() + assertEquals( + setOf( + AudioTrackFeature.TF_NOISE_SUPPRESSION, + AudioTrackFeature.TF_AUTO_GAIN_CONTROL, + ), + features, + ) + } + + private fun lastUpdateAudioTrackFeatures(): Set { + val requestString = wsFactory.ws.sentRequests.last().toPBByteString() + val sentRequest = LivekitRtc.SignalRequest.newBuilder() + .mergeFrom(requestString) + .build() + + assertTrue(sentRequest.hasUpdateAudioTrack()) + return sentRequest.updateAudioTrack.featuresList.toSet() + } + + private fun assertConstantFeatures(features: Collection) { + assertEquals(3, features.size) + assertTrue(features.contains(AudioTrackFeature.TF_ECHO_CANCELLATION)) + assertTrue(features.contains(AudioTrackFeature.TF_NOISE_SUPPRESSION)) + assertTrue(features.contains(AudioTrackFeature.TF_AUTO_GAIN_CONTROL)) + } + + private fun krispProcessor() = object : AudioProcessorInterface { + override fun isEnabled(): Boolean = true + + override fun getName(): String = "krisp_noise_cancellation" + + override fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int) {} + + override fun resetAudioProcessing(newRate: Int) {} + + override fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer) {} + } + + private fun otherProcessor() = object : AudioProcessorInterface { + override fun isEnabled(): Boolean = true + + override fun getName(): String = "other_processor" + + override fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int) {} + + override fun resetAudioProcessing(newRate: Int) {} + + override fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer) {} + } +}