feat: use camerax with webrtc lib

app/build.gradle.kts

@@ -61,6 +61,7 @@ dependencies {
     implementation(libs.androidx.camera.mlkit.vision)
     implementation(libs.androidx.camera.extensions)
     implementation(libs.material3)
+    implementation(libs.stream.webrtc.android)
     testImplementation(libs.junit)
     androidTestImplementation(libs.androidx.junit)
     androidTestImplementation(libs.androidx.espresso.core)

app/src/main/java/com/example/tvcontroller/MainActivity.kt

@@ -5,6 +5,8 @@ import android.util.Log
 import androidx.activity.ComponentActivity
 import androidx.activity.compose.setContent
 import androidx.activity.enableEdgeToEdge
+import androidx.camera.view.CameraController
+import androidx.camera.view.LifecycleCameraController
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.padding
 import androidx.compose.material3.Icon
@@ -13,23 +15,29 @@ import androidx.compose.material3.NavigationBarItem
 import androidx.compose.material3.Scaffold
 import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.getValue
 import androidx.compose.ui.Modifier
+import androidx.compose.ui.res.painterResource
+import androidx.core.app.ActivityCompat
+import androidx.core.content.ContextCompat
+import androidx.lifecycle.LifecycleOwner
+import androidx.lifecycle.lifecycleScope
 import androidx.navigation.NavHostController
 import androidx.navigation.compose.NavHost
 import androidx.navigation.compose.composable
 import androidx.navigation.compose.currentBackStackEntryAsState
 import androidx.navigation.compose.rememberNavController
-import com.example.tvcontroller.ui.theme.TVControllerTheme
-import androidx.compose.runtime.getValue
-import androidx.compose.ui.res.painterResource
-import androidx.core.app.ActivityCompat
 import com.example.tvcontroller.services.BluetoothService
 import com.example.tvcontroller.services.CameraService
 import com.example.tvcontroller.services.ControllerService
 import com.example.tvcontroller.services.DeviceService
+import com.example.tvcontroller.ui.theme.TVControllerTheme
 import com.example.tvcontroller.ui.views.CameraView
 import com.example.tvcontroller.ui.views.RemoteView
 import com.example.tvcontroller.ui.views.SettingsView
+import com.example.tvcontroller.webrtc.RtcPeerConnection
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 
 
 const val TAG = "MainActivity"
@@ -42,18 +50,36 @@ class MainActivity : ComponentActivity() {
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
-        bluetoothService = BluetoothService(this.applicationContext)
+        val lifecycleOwner: LifecycleOwner = this
-        deviceService = DeviceService(this.applicationContext)
+        val rtcPeerConnection = RtcPeerConnection(applicationContext)
-        cameraService = CameraService(this.applicationContext)
+        val cameraController =
-        controllerService = ControllerService(this.applicationContext, bluetoothService)
+            LifecycleCameraController(applicationContext).apply {
+                setEnabledUseCases(CameraController.IMAGE_ANALYSIS)
+                setImageAnalysisAnalyzer(
+                    ContextCompat.getMainExecutor(applicationContext),
+                    rtcPeerConnection.cameraXCapturer
+                )
+                bindToLifecycle(lifecycleOwner)
+
+            }
+        bluetoothService = BluetoothService(applicationContext)
+        deviceService = DeviceService(applicationContext)
+        cameraService = CameraService(applicationContext)
+        controllerService = ControllerService(applicationContext, bluetoothService)
         checkPermissions()
+
+        lifecycleScope.launch(Dispatchers.IO) {
+           deviceService.initialize()
+        }
         enableEdgeToEdge()
         setContent {
             TVControllerTheme {
                 TvControllerApp(
                     deviceService = deviceService,
                     controllerService = controllerService,
-                    bluetoothService = bluetoothService
+                    bluetoothService = bluetoothService,
+                    rtcPeerConnection = rtcPeerConnection,
+                    cameraController = cameraController
                 )
             }
         }
@@ -76,7 +102,9 @@ fun TvControllerApp(
     navController: NavHostController = rememberNavController(),
     deviceService: DeviceService,
     controllerService: ControllerService,
-    bluetoothService: BluetoothService
+    bluetoothService: BluetoothService,
+    rtcPeerConnection: RtcPeerConnection,
+    cameraController: LifecycleCameraController
 ) {
     val backStackEntry by navController.currentBackStackEntryAsState()
     val currentScreen = Screen.valueOf(backStackEntry?.destination?.route ?: Screen.Camera.name)
@@ -122,11 +150,11 @@ fun TvControllerApp(
     }) { innerPadding ->
         NavHost(
             navController = navController,
-            startDestination = Screen.Remote.name,
+            startDestination = Screen.Settings.name,
             modifier = Modifier.padding(innerPadding)
         ) {
             composable(route = Screen.Camera.name) {
-                CameraView()
+                CameraView(cameraController)
             }
             composable(route = Screen.Remote.name) {
                 RemoteView(

app/src/main/java/com/example/tvcontroller/services/DeviceService.kt

@@ -10,10 +10,13 @@ import io.ktor.client.call.body
 import io.ktor.client.plugins.cookies.HttpCookies
 import io.ktor.client.plugins.websocket.WebSockets
 import io.ktor.client.plugins.websocket.webSocket
+import io.ktor.client.request.HttpRequestBuilder
+import io.ktor.client.request.cookie
 import io.ktor.client.request.headers
 import io.ktor.client.request.request
 import io.ktor.client.request.setBody
 import io.ktor.client.statement.HttpResponse
+import io.ktor.http.Cookie
 import io.ktor.http.HttpMethod
 import io.ktor.websocket.Frame
 import io.ktor.websocket.readText
@@ -25,15 +28,18 @@ private const val TAG = "DeviceService"
 
 class DeviceService(private val context: Context) {
     private var client = HttpClient(CIO) {
-        install(HttpCookies)
         install(WebSockets)
     }
     private var serverAddress: String = ""
     private var token: String = ""
     private var deviceId: String = ""
 
-    init {
+    suspend fun initialize() {
         loadPreferences()
+        if (token.isEmpty()) return
+        getIntegration()?.let {
+            connect()
+        }
     }
 
     suspend fun registerIntegration(name: String, code: String) {
@@ -44,12 +50,14 @@ class DeviceService(private val context: Context) {
         token = ""
         deviceId = ""
         try {
-            val response: HttpResponse = client.request("http://$serverAddress/api/integrations/register") {
+            val response: HttpResponse =
+                client.request("http://$serverAddress/api/integrations/register") {
                     method = HttpMethod.Post
                     setBody(requestJson.toString())
                     headers {
                         append("Content-Type", "application/json")
                     }
+                    cookie(name = "token", value = token)
                 }
 
             val body: String = response.body()
@@ -78,6 +86,7 @@ class DeviceService(private val context: Context) {
                     headers {
                         append("Authorization", "Bearer $token")
                     }
+                    cookie(name = "token", value = token)
                 }
 
             val body: String = response.body()
@@ -120,11 +129,17 @@ class DeviceService(private val context: Context) {
     fun connect() {
         Log.i(TAG, "Connecting to websocket at $serverAddress")
         runBlocking {
-            // split server address into host and port
             val (host, port) = serverAddress.split(":")
-            // if no port is specified, assume 80
             val portInt = if (port.isEmpty()) 80 else port.toInt()
-            client.webSocket(method = HttpMethod.Get, host = host, port = portInt, path = "/ws") {
+            client.webSocket(
+                method = HttpMethod.Get,
+                host = host,
+                port = portInt,
+                path = "/ws",
+                request = {
+                    cookie(name = "token", value = token)
+                }
+            ) {
                 Log.i(TAG, "Listening for incoming websocket messages")
                 while (true) {
                     val frame = incoming.receive()

app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt

@@ -12,13 +12,11 @@ fun CameraPreview(
     controller: LifecycleCameraController,
     modifier: Modifier = Modifier
 ) {
-    val lifecycleOwner = LocalLifecycleOwner.current
     AndroidView(
         modifier = modifier,
         factory = {
             PreviewView(it).apply {
                 this.controller = controller
-                controller.bindToLifecycle(lifecycleOwner)
             }
         },
     )

app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt

@@ -1,34 +1,26 @@
 package com.example.tvcontroller.ui.views
 
+import androidx.camera.core.ImageAnalysis
 import androidx.camera.view.CameraController
 import androidx.camera.view.LifecycleCameraController
-import androidx.compose.foundation.layout.Arrangement
 import androidx.compose.foundation.layout.Box
-import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.padding
-import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.remember
-import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.unit.dp
+import androidx.core.content.ContextCompat
 import com.example.tvcontroller.ui.components.CameraPreview
 
 @Composable
-fun CameraView() {
+fun CameraView(cameraController: LifecycleCameraController) {
-    val context = LocalContext.current
-    val controller = remember {
-        LifecycleCameraController(context).apply {
-            setEnabledUseCases(CameraController.VIDEO_CAPTURE)
-        }
-    }
     Box(
         modifier = Modifier
             .fillMaxSize()
             .padding(all = 16.dp),
     ) {
-        CameraPreview(controller = controller, modifier = Modifier.fillMaxSize())
+        CameraPreview(controller = cameraController, modifier = Modifier.fillMaxSize())
     }
 }

app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt

@@ -0,0 +1,113 @@
+package com.example.tvcontroller.webrtc
+
+import android.graphics.ImageFormat.YUV_420_888
+import android.util.Log
+import androidx.camera.core.ImageAnalysis
+import androidx.camera.core.ImageProxy
+import org.webrtc.CapturerObserver
+import org.webrtc.VideoFrame
+import java.nio.ByteBuffer
+
+class CameraXCapturer : ImageAnalysis.Analyzer {
+    private var capturerObserver: CapturerObserver? = null
+
+    fun setCapturerObserver(capturerObserver: CapturerObserver) {
+        this.capturerObserver = capturerObserver
+    }
+
+    override fun analyze(image: ImageProxy) {
+        if (image.format != YUV_420_888) throw Exception("Unsupported format")
+        Log.i("CameraXCapturer", "Received image from CameraX")
+        var videoFrame = imageProxyToVideoFrame(image)
+        capturerObserver?.onFrameCaptured(videoFrame) ?: image.close()
+    }
+
+    fun imageProxyToVideoFrame(image: ImageProxy): VideoFrame {
+        var buffer = object : VideoFrame.I420Buffer {
+            private var refCount = 0
+            override fun getWidth(): Int {
+                return image.width
+            }
+
+            override fun getHeight(): Int {
+                return image.height
+            }
+
+            override fun toI420(): VideoFrame.I420Buffer? {
+                return this
+            }
+
+            override fun retain() {
+                refCount++
+            }
+
+            override fun release() {
+                refCount--
+                if (refCount == 0) {
+                    image.close()
+                }
+            }
+
+            override fun cropAndScale(
+                cropX: Int,
+                cropY: Int,
+                cropWidth: Int,
+                cropHeight: Int,
+                scaleWidth: Int,
+                scaleHeight: Int
+            ): VideoFrame.Buffer? {
+                return this
+            }
+
+            override fun getDataY(): ByteBuffer? {
+                val format = image.format
+                if (format == YUV_420_888) {
+                    return image.planes[0].buffer
+                }
+                return image.planes[0].buffer
+            }
+
+            override fun getDataU(): ByteBuffer? {
+                val format = image.format
+                if (format == YUV_420_888) {
+                    return image.planes[1].buffer
+                }
+                return image.planes[0].buffer
+            }
+
+            override fun getDataV(): ByteBuffer? {
+                val format = image.format
+                if (format == YUV_420_888) {
+                    return image.planes[2].buffer
+                }
+                return image.planes[0].buffer
+            }
+
+            override fun getStrideY(): Int {
+                val format = image.format
+                if (format == YUV_420_888) {
+                    return image.planes[0].pixelStride
+                }
+                return image.planes[0].pixelStride
+            }
+
+            override fun getStrideU(): Int {
+                val format = image.format
+                if (format == YUV_420_888) {
+                    return image.planes[1].pixelStride
+                }
+                return image.planes[0].pixelStride
+            }
+
+            override fun getStrideV(): Int {
+                val format = image.format
+                if (format == YUV_420_888) {
+                    return image.planes[2].pixelStride
+                }
+                return image.planes[0].pixelStride
+            }
+        }
+        var videoFrame = VideoFrame(buffer, 0, 0)
+        return videoFrame
+    }
+}

app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt

@@ -0,0 +1,138 @@
+package com.example.tvcontroller.webrtc
+
+import android.content.Context
+import android.hardware.camera2.CameraManager
+import android.util.Log
+import org.webrtc.AudioTrack
+import org.webrtc.Camera2Capturer
+import org.webrtc.DataChannel
+import org.webrtc.IceCandidate
+import org.webrtc.MediaConstraints
+import org.webrtc.MediaStream
+import org.webrtc.PeerConnection
+import org.webrtc.PeerConnectionFactory
+import org.webrtc.PeerConnectionFactory.InitializationOptions
+import org.webrtc.SdpObserver
+import org.webrtc.SessionDescription
+import org.webrtc.VideoTrack
+
+const val TAG = "RtcPeerConnection"
+
+class RtcPeerConnection(private val context: Context) : PeerConnection.Observer {
+    private val peerConnectionFactory by lazy { initializeFactory() }
+    private val iceServers by lazy { initializeIceServers() }
+    val cameraXCapturer by lazy { CameraXCapturer() }
+    private val audioTrack by lazy { createAudioTrack() }
+    private val videoTrack by lazy { createVideoTrack() }
+    private var peerConnection: PeerConnection? = null
+
+    private fun initializeFactory(): PeerConnectionFactory {
+        val initOptions = InitializationOptions.builder(context).createInitializationOptions()
+        PeerConnectionFactory.initialize(initOptions)
+
+        val options = PeerConnectionFactory.Options()
+        val peerConnectionFactory =
+            PeerConnectionFactory.builder().setOptions(options).createPeerConnectionFactory()
+        return peerConnectionFactory
+    }
+
+    private fun initializeIceServers(): ArrayList<PeerConnection.IceServer> {
+        val iceServers = ArrayList<PeerConnection.IceServer>()
+        iceServers.add(
+            PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer()
+        )
+        return iceServers
+    }
+
+    private fun createAudioTrack(): AudioTrack {
+        val audioConstraints = MediaConstraints()
+        val audioSource = peerConnectionFactory.createAudioSource(audioConstraints)
+        val localAudioTrack = peerConnectionFactory.createAudioTrack("audio_track", audioSource)
+        return localAudioTrack
+    }
+
+    private fun createVideoTrack(): VideoTrack {
+        val videoSource = peerConnectionFactory.createVideoSource(false)
+        cameraXCapturer.setCapturerObserver(videoSource.capturerObserver)
+        val localVideoTrack = peerConnectionFactory.createVideoTrack("video_track", videoSource)
+        localVideoTrack.setEnabled(true)
+        return localVideoTrack
+    }
+
+    fun connect() {
+        Log.i(TAG, "Connecting rtc peer connection")
+        var rtcConfig = PeerConnection.RTCConfiguration(iceServers)
+        var peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this)
+
+        var mediaConstraints = MediaConstraints()
+        mediaConstraints.mandatory.add(
+            MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")
+        );
+        mediaConstraints.mandatory.add(
+            MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")
+        );
+        peerConnection?.addTrack(audioTrack)
+        peerConnection?.addTrack(videoTrack)
+        peerConnection?.createOffer(object : SdpObserver {
+            override fun onCreateSuccess(sessionDescription: SessionDescription?) {
+                Log.i(TAG, "onCreateSuccess: ${sessionDescription?.description}")
+            }
+
+            override fun onSetSuccess() {
+                Log.i(TAG, "onSetSuccess")
+                TODO("Not yet implemented")
+            }
+
+            override fun onCreateFailure(p0: String?) {
+                Log.i(TAG, "onCreateFailure: $p0")
+                TODO("Not yet implemented")
+            }
+
+            override fun onSetFailure(p0: String?) {
+                Log.i(TAG, "onSetFailure: $p0")
+                TODO("Not yet implemented")
+            }
+
+        }, mediaConstraints)
+    }
+
+    override fun onSignalingChange(p0: PeerConnection.SignalingState?) {
+        Log.i(TAG, "onSignalingChange: $p0")
+    }
+
+    override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) {
+        Log.i(TAG, "onIceConnectionChange: $p0")
+    }
+
+    override fun onIceConnectionReceivingChange(p0: Boolean) {
+        Log.i(TAG, "onIceConnectionReceivingChange: $p0")
+    }
+
+    override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) {
+        Log.i(TAG, "onIceGatheringChange: $p0")
+    }
+
+    override fun onIceCandidate(p0: IceCandidate?) {
+        Log.i(TAG, "onIceCandidate: $p0")
+    }
+
+    override fun onIceCandidatesRemoved(p0: Array<out IceCandidate?>?) {
+        Log.i(TAG, "onIceCandidatesRemoved: $p0")
+    }
+
+    override fun onAddStream(p0: MediaStream?) {
+        Log.i(TAG, "onAddStream: $p0")
+    }
+
+    override fun onRemoveStream(p0: MediaStream?) {
+        Log.i(TAG, "onRemoveStream: $p0")
+    }
+
+    override fun onDataChannel(p0: DataChannel?) {
+        Log.i(TAG, "onDataChannel: $p0")
+    }
+
+    override fun onRenegotiationNeeded() {
+        Log.i(TAG, "onRenegotiationNeeded")
+    }
+}

gradle/libs.versions.toml

@@ -12,6 +12,7 @@ activityCompose = "1.10.1"
 composeBom = "2025.03.01"
 material3 = "1.4.0-alpha11"
 navigationCompose = "2.8.9"
+streamWebrtcAndroid = "1.3.8"
 
 [libraries]
 androidx-camera-camera2 = { module = "androidx.camera:camera-camera2", version.ref = "cameraCore" }
@@ -40,6 +41,7 @@ ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" }
 ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" }
 ktor-client-websockets = { module = "io.ktor:ktor-client-websockets", version.ref = "ktor" }
 material3 = { module = "androidx.compose.material3:material3", version.ref = "material3" }
+stream-webrtc-android = { module = "io.getstream:stream-webrtc-android", version.ref = "streamWebrtcAndroid" }
 
 [plugins]
 android-application = { id = "com.android.application", version.ref = "agp" }