diff --git a/app/build.gradle.kts b/app/build.gradle.kts index c4f5c46..3557b12 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -61,6 +61,7 @@ dependencies { implementation(libs.androidx.camera.mlkit.vision) implementation(libs.androidx.camera.extensions) implementation(libs.material3) + implementation(libs.stream.webrtc.android) testImplementation(libs.junit) androidTestImplementation(libs.androidx.junit) androidTestImplementation(libs.androidx.espresso.core) diff --git a/app/src/main/java/com/example/tvcontroller/MainActivity.kt b/app/src/main/java/com/example/tvcontroller/MainActivity.kt index 57c1a29..0faa057 100644 --- a/app/src/main/java/com/example/tvcontroller/MainActivity.kt +++ b/app/src/main/java/com/example/tvcontroller/MainActivity.kt @@ -5,6 +5,9 @@ import android.util.Log import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import androidx.activity.enableEdgeToEdge +import androidx.camera.core.ImageAnalysis +import androidx.camera.view.CameraController +import androidx.camera.view.LifecycleCameraController import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding import androidx.compose.material3.Icon @@ -21,8 +24,13 @@ import androidx.navigation.compose.currentBackStackEntryAsState import androidx.navigation.compose.rememberNavController import com.example.tvcontroller.ui.theme.TVControllerTheme import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.painterResource import androidx.core.app.ActivityCompat +import androidx.core.content.ContextCompat +import androidx.lifecycle.LifecycleOwner +import androidx.lifecycle.lifecycleScope import com.example.tvcontroller.services.BluetoothService import com.example.tvcontroller.services.CameraService import com.example.tvcontroller.services.ControllerService @@ -30,6 +38,10 @@ import com.example.tvcontroller.services.DeviceService import com.example.tvcontroller.ui.views.CameraView import com.example.tvcontroller.ui.views.RemoteView import com.example.tvcontroller.ui.views.SettingsView +import com.example.tvcontroller.webrtc.CameraXCapturer +import com.example.tvcontroller.webrtc.RtcPeerConnection +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch const val TAG = "MainActivity" @@ -42,10 +54,22 @@ class MainActivity : ComponentActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - bluetoothService = BluetoothService(this.applicationContext) - deviceService = DeviceService(this.applicationContext) - cameraService = CameraService(this.applicationContext) - controllerService = ControllerService(this.applicationContext, bluetoothService) + val lifecycleOwner: LifecycleOwner = this + val rtcPeerConnection = RtcPeerConnection(applicationContext) + val cameraController = + LifecycleCameraController(applicationContext).apply { + setEnabledUseCases(CameraController.IMAGE_ANALYSIS) + setImageAnalysisAnalyzer( + ContextCompat.getMainExecutor(applicationContext), + rtcPeerConnection.cameraXCapturer + ) + bindToLifecycle(lifecycleOwner) + + } + bluetoothService = BluetoothService(applicationContext) + deviceService = DeviceService(applicationContext) + cameraService = CameraService(applicationContext) + controllerService = ControllerService(applicationContext, bluetoothService) checkPermissions() enableEdgeToEdge() setContent { @@ -53,7 +77,9 @@ class MainActivity : ComponentActivity() { TvControllerApp( deviceService = deviceService, controllerService = controllerService, - bluetoothService = bluetoothService + bluetoothService = bluetoothService, + rtcPeerConnection = rtcPeerConnection, + cameraController = cameraController ) } } @@ -76,7 +102,9 @@ fun TvControllerApp( navController: NavHostController = rememberNavController(), deviceService: DeviceService, controllerService: ControllerService, - bluetoothService: BluetoothService + bluetoothService: BluetoothService, + rtcPeerConnection: RtcPeerConnection, + cameraController: LifecycleCameraController ) { val backStackEntry by navController.currentBackStackEntryAsState() val currentScreen = Screen.valueOf(backStackEntry?.destination?.route ?: Screen.Camera.name) @@ -122,11 +150,11 @@ fun TvControllerApp( }) { innerPadding -> NavHost( navController = navController, - startDestination = Screen.Remote.name, + startDestination = Screen.Settings.name, modifier = Modifier.padding(innerPadding) ) { composable(route = Screen.Camera.name) { - CameraView() + CameraView(cameraController) } composable(route = Screen.Remote.name) { RemoteView( @@ -136,7 +164,8 @@ fun TvControllerApp( composable(route = Screen.Settings.name) { SettingsView( deviceService = deviceService, - bluetoothService = bluetoothService + bluetoothService = bluetoothService, + rtcPeerConnection = rtcPeerConnection ) } } diff --git a/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt b/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt index e100c9e..de8fc6d 100644 --- a/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt +++ b/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt @@ -12,13 +12,11 @@ fun CameraPreview( controller: LifecycleCameraController, modifier: Modifier = Modifier ) { - val lifecycleOwner = LocalLifecycleOwner.current AndroidView( modifier = modifier, factory = { PreviewView(it).apply { this.controller = controller - controller.bindToLifecycle(lifecycleOwner) } }, ) diff --git a/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt b/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt index 02ab23f..71bd97b 100644 --- a/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt +++ b/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt @@ -1,34 +1,26 @@ package com.example.tvcontroller.ui.views +import androidx.camera.core.ImageAnalysis import androidx.camera.view.CameraController import androidx.camera.view.LifecycleCameraController -import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding -import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.remember -import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.unit.dp +import androidx.core.content.ContextCompat import com.example.tvcontroller.ui.components.CameraPreview @Composable -fun CameraView() { - val context = LocalContext.current - val controller = remember { - LifecycleCameraController(context).apply { - setEnabledUseCases(CameraController.VIDEO_CAPTURE) - } - } +fun CameraView(cameraController: LifecycleCameraController) { Box( modifier = Modifier .fillMaxSize() .padding(all = 16.dp), ) { - CameraPreview(controller = controller, modifier = Modifier.fillMaxSize()) + CameraPreview(controller = cameraController, modifier = Modifier.fillMaxSize()) } } diff --git a/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt b/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt new file mode 100644 index 0000000..6fd9cef --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt @@ -0,0 +1,113 @@ +package com.example.tvcontroller.webrtc + +import android.graphics.ImageFormat.YUV_420_888 +import android.util.Log +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageProxy +import org.webrtc.CapturerObserver +import org.webrtc.VideoFrame +import java.nio.ByteBuffer + +class CameraXCapturer : ImageAnalysis.Analyzer { + private var capturerObserver: CapturerObserver? = null + + fun setCapturerObserver(capturerObserver: CapturerObserver) { + this.capturerObserver = capturerObserver + } + + override fun analyze(image: ImageProxy) { + if (image.format != YUV_420_888) throw Exception("Unsupported format") + Log.i("CameraXCapturer", "Received image from CameraX") + var videoFrame = imageProxyToVideoFrame(image) + capturerObserver?.onFrameCaptured(videoFrame) ?: image.close() + } + + fun imageProxyToVideoFrame(image: ImageProxy): VideoFrame { + var buffer = object : VideoFrame.I420Buffer { + private var refCount = 0 + override fun getWidth(): Int { + return image.width + } + + override fun getHeight(): Int { + return image.height + } + + override fun toI420(): VideoFrame.I420Buffer? { + return this + } + + override fun retain() { + refCount++ + } + + override fun release() { + refCount-- + if (refCount == 0) { + image.close() + } + } + + override fun cropAndScale( + cropX: Int, + cropY: Int, + cropWidth: Int, + cropHeight: Int, + scaleWidth: Int, + scaleHeight: Int + ): VideoFrame.Buffer? { + return this + } + + override fun getDataY(): ByteBuffer? { + val format = image.format + if (format == YUV_420_888) { + return image.planes[0].buffer + } + return image.planes[0].buffer + } + + override fun getDataU(): ByteBuffer? { + val format = image.format + if (format == YUV_420_888) { + return image.planes[1].buffer + } + return image.planes[0].buffer + } + + override fun getDataV(): ByteBuffer? { + val format = image.format + if (format == YUV_420_888) { + return image.planes[2].buffer + } + return image.planes[0].buffer + } + + override fun getStrideY(): Int { + val format = image.format + if (format == YUV_420_888) { + return image.planes[0].pixelStride + } + return image.planes[0].pixelStride + } + + override fun getStrideU(): Int { + val format = image.format + if (format == YUV_420_888) { + return image.planes[1].pixelStride + } + return image.planes[0].pixelStride + } + + override fun getStrideV(): Int { + val format = image.format + if (format == YUV_420_888) { + return image.planes[2].pixelStride + } + return image.planes[0].pixelStride + } + } + var videoFrame = VideoFrame(buffer, 0, 0) + return videoFrame + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt b/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt new file mode 100644 index 0000000..fb2194b --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt @@ -0,0 +1,138 @@ +package com.example.tvcontroller.webrtc + +import android.content.Context +import android.hardware.camera2.CameraManager +import android.util.Log +import org.webrtc.AudioTrack +import org.webrtc.Camera2Capturer +import org.webrtc.DataChannel +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.PeerConnectionFactory +import org.webrtc.PeerConnectionFactory.InitializationOptions +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import org.webrtc.VideoTrack + +const val TAG = "RtcPeerConnection" + +class RtcPeerConnection(private val context: Context) : PeerConnection.Observer { + private val peerConnectionFactory by lazy { initializeFactory() } + private val iceServers by lazy { initializeIceServers() } + val cameraXCapturer by lazy { CameraXCapturer() } + private val audioTrack by lazy { createAudioTrack() } + private val videoTrack by lazy { createVideoTrack() } + private var peerConnection: PeerConnection? = null + + private fun initializeFactory(): PeerConnectionFactory { + val initOptions = InitializationOptions.builder(context).createInitializationOptions() + PeerConnectionFactory.initialize(initOptions) + + val options = PeerConnectionFactory.Options() + val peerConnectionFactory = + PeerConnectionFactory.builder().setOptions(options).createPeerConnectionFactory() + return peerConnectionFactory + } + + private fun initializeIceServers(): ArrayList { + val iceServers = ArrayList() + iceServers.add( + PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer() + ) + return iceServers + } + + private fun createAudioTrack(): AudioTrack { + val audioConstraints = MediaConstraints() + val audioSource = peerConnectionFactory.createAudioSource(audioConstraints) + val localAudioTrack = peerConnectionFactory.createAudioTrack("audio_track", audioSource) + return localAudioTrack + } + + private fun createVideoTrack(): VideoTrack { + val videoSource = peerConnectionFactory.createVideoSource(false) + cameraXCapturer.setCapturerObserver(videoSource.capturerObserver) + val localVideoTrack = peerConnectionFactory.createVideoTrack("video_track", videoSource) + localVideoTrack.setEnabled(true) + return localVideoTrack + } + + fun connect() { + Log.i(TAG, "Connecting rtc peer connection") + var rtcConfig = PeerConnection.RTCConfiguration(iceServers) + var peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this) + + var mediaConstraints = MediaConstraints() + mediaConstraints.mandatory.add( + MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true") + ); + mediaConstraints.mandatory.add( + MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true") + ); + peerConnection?.addTrack(audioTrack) + peerConnection?.addTrack(videoTrack) + peerConnection?.createOffer(object : SdpObserver { + override fun onCreateSuccess(sessionDescription: SessionDescription?) { + Log.i(TAG, "onCreateSuccess: ${sessionDescription?.description}") + } + + override fun onSetSuccess() { + Log.i(TAG, "onSetSuccess") + TODO("Not yet implemented") + } + + override fun onCreateFailure(p0: String?) { + Log.i(TAG, "onCreateFailure: $p0") + TODO("Not yet implemented") + } + + override fun onSetFailure(p0: String?) { + Log.i(TAG, "onSetFailure: $p0") + TODO("Not yet implemented") + } + + }, mediaConstraints) + } + + override fun onSignalingChange(p0: PeerConnection.SignalingState?) { + Log.i(TAG, "onSignalingChange: $p0") + } + + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { + Log.i(TAG, "onIceConnectionChange: $p0") + } + + override fun onIceConnectionReceivingChange(p0: Boolean) { + Log.i(TAG, "onIceConnectionReceivingChange: $p0") + } + + override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) { + Log.i(TAG, "onIceGatheringChange: $p0") + } + + override fun onIceCandidate(p0: IceCandidate?) { + Log.i(TAG, "onIceCandidate: $p0") + } + + override fun onIceCandidatesRemoved(p0: Array?) { + Log.i(TAG, "onIceCandidatesRemoved: $p0") + } + + override fun onAddStream(p0: MediaStream?) { + Log.i(TAG, "onAddStream: $p0") + } + + override fun onRemoveStream(p0: MediaStream?) { + Log.i(TAG, "onRemoveStream: $p0") + } + + override fun onDataChannel(p0: DataChannel?) { + Log.i(TAG, "onDataChannel: $p0") + } + + override fun onRenegotiationNeeded() { + Log.i(TAG, "onRenegotiationNeeded") + } +} \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5b3339d..240bdfa 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,6 +12,7 @@ activityCompose = "1.10.1" composeBom = "2025.03.01" material3 = "1.4.0-alpha11" navigationCompose = "2.8.9" +streamWebrtcAndroid = "1.3.8" [libraries] androidx-camera-camera2 = { module = "androidx.camera:camera-camera2", version.ref = "cameraCore" } @@ -40,6 +41,7 @@ ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" } ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-websockets = { module = "io.ktor:ktor-client-websockets", version.ref = "ktor" } material3 = { module = "androidx.compose.material3:material3", version.ref = "material3" } +stream-webrtc-android = { module = "io.getstream:stream-webrtc-android", version.ref = "streamWebrtcAndroid" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" }