From 11838f0abd07eaf4a080c912a32799270dafc937 Mon Sep 17 00:00:00 2001 From: Fritz Heiden Date: Fri, 4 Apr 2025 11:25:08 +0200 Subject: [PATCH] refactor: structure webrtc code properly --- .../com/example/tvcontroller/MainActivity.kt | 87 ++---- .../example/tvcontroller/client/WebClient.kt | 54 ++++ .../tvcontroller/client/WebsocketClient.kt | 54 ++++ .../services/ControllerService.kt | 1 - .../tvcontroller/services/DeviceService.kt | 239 +++------------ .../services/webrtc/RtcPeerConnection.kt | 191 ++++++++++++ .../services/webrtc/WebRtcService.kt | 123 ++++++++ .../ui/views/SettingsViewModel.kt | 11 +- .../tvcontroller/webrtc/CameraXCapturer.kt | 113 ------- .../tvcontroller/webrtc/RtcPeerConnection.kt | 282 ------------------ 10 files changed, 506 insertions(+), 649 deletions(-) create mode 100644 app/src/main/java/com/example/tvcontroller/client/WebClient.kt create mode 100644 app/src/main/java/com/example/tvcontroller/client/WebsocketClient.kt create mode 100644 app/src/main/java/com/example/tvcontroller/services/webrtc/RtcPeerConnection.kt create mode 100644 app/src/main/java/com/example/tvcontroller/services/webrtc/WebRtcService.kt delete mode 100644 app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt delete mode 100644 app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt diff --git a/app/src/main/java/com/example/tvcontroller/MainActivity.kt b/app/src/main/java/com/example/tvcontroller/MainActivity.kt index 267f56f..e94aebd 100644 --- a/app/src/main/java/com/example/tvcontroller/MainActivity.kt +++ b/app/src/main/java/com/example/tvcontroller/MainActivity.kt @@ -5,8 +5,6 @@ import android.util.Log import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import androidx.activity.enableEdgeToEdge -import androidx.camera.view.CameraController -import androidx.camera.view.LifecycleCameraController import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding import androidx.compose.material3.Icon @@ -19,54 +17,46 @@ import androidx.compose.runtime.getValue import androidx.compose.ui.Modifier import androidx.compose.ui.res.painterResource import androidx.core.app.ActivityCompat -import androidx.core.content.ContextCompat -import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.lifecycleScope import androidx.navigation.NavHostController import androidx.navigation.compose.NavHost import androidx.navigation.compose.composable import androidx.navigation.compose.currentBackStackEntryAsState import androidx.navigation.compose.rememberNavController +import com.example.tvcontroller.client.WebClient +import com.example.tvcontroller.client.WebsocketClient import com.example.tvcontroller.services.BluetoothService -import com.example.tvcontroller.services.CameraService import com.example.tvcontroller.services.ControllerService import com.example.tvcontroller.services.DeviceService +import com.example.tvcontroller.services.webrtc.WebRtcService import com.example.tvcontroller.ui.theme.TVControllerTheme import com.example.tvcontroller.ui.views.CameraView import com.example.tvcontroller.ui.views.RemoteView import com.example.tvcontroller.ui.views.SettingsView -import com.example.tvcontroller.webrtc.RtcPeerConnection import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch - const val TAG = "MainActivity" class MainActivity : ComponentActivity() { - private lateinit var bluetoothService: BluetoothService - private lateinit var deviceService: DeviceService - private lateinit var cameraService: CameraService - private lateinit var controllerService: ControllerService + private val webClient by lazy { WebClient() } + private val websocketClient by lazy { WebsocketClient(webClient.client) } + private val webRtcService by lazy { WebRtcService(applicationContext, websocketClient) } + private val bluetoothService by lazy { BluetoothService(applicationContext) } + private val deviceService by lazy { DeviceService(applicationContext, webClient) } + private val controllerService by lazy { ControllerService(bluetoothService) } override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - val lifecycleOwner: LifecycleOwner = this - val rtcPeerConnection = RtcPeerConnection(applicationContext) - val cameraController = - LifecycleCameraController(applicationContext).apply { - setEnabledUseCases(CameraController.IMAGE_ANALYSIS) - bindToLifecycle(lifecycleOwner) - } - bluetoothService = BluetoothService(applicationContext) - deviceService = DeviceService(applicationContext) - deviceService.rtcPeerConnection = rtcPeerConnection - rtcPeerConnection.deviceService = deviceService - cameraService = CameraService(applicationContext) - controllerService = ControllerService(applicationContext, bluetoothService) checkPermissions() lifecycleScope.launch { - deviceService.initialize() + deviceService.initialize() + if (deviceService.serverAddress.isEmpty() || deviceService.token.isEmpty()) return@launch + lifecycleScope.launch(Dispatchers.IO) { + websocketClient.connect(deviceService.serverAddress, deviceService.token) + } + webRtcService.connect() } enableEdgeToEdge() setContent { @@ -75,8 +65,7 @@ class MainActivity : ComponentActivity() { deviceService = deviceService, controllerService = controllerService, bluetoothService = bluetoothService, - rtcPeerConnection = rtcPeerConnection, - cameraController = cameraController + webRtcService = webRtcService ) } } @@ -84,9 +73,6 @@ class MainActivity : ComponentActivity() { private fun checkPermissions() { Log.i(TAG, "Checking permissions") - if (!cameraService.hasRequiredPermissions()) { - ActivityCompat.requestPermissions(this, CameraService.CAMERAX_PERMISSIONS, 0) - } if (!bluetoothService.hasRequiredPermissions()) { Log.i(TAG, "Requesting Bluetooth permissions") ActivityCompat.requestPermissions(this, BluetoothService.BLUETOOTH_PERMISSIONS, 0) @@ -100,8 +86,7 @@ fun TvControllerApp( deviceService: DeviceService, controllerService: ControllerService, bluetoothService: BluetoothService, - rtcPeerConnection: RtcPeerConnection, - cameraController: LifecycleCameraController + webRtcService: WebRtcService ) { val backStackEntry by navController.currentBackStackEntryAsState() val currentScreen = Screen.valueOf(backStackEntry?.destination?.route ?: Screen.Camera.name) @@ -111,37 +96,25 @@ fun TvControllerApp( Scaffold(modifier = Modifier.fillMaxSize(), bottomBar = { NavigationBar { NavigationBarItem( - onClick = { navController.navigate(Screen.Camera.name) }, - icon = { + onClick = { navController.navigate(Screen.Camera.name) }, icon = { Icon( - baselineCamera24, - contentDescription = "Camera" + baselineCamera24, contentDescription = "Camera" ) - }, - label = { Text("Camera") }, - selected = currentScreen == Screen.Camera + }, label = { Text("Camera") }, selected = currentScreen == Screen.Camera ) NavigationBarItem( - onClick = { navController.navigate(Screen.Remote.name) }, - icon = { + onClick = { navController.navigate(Screen.Remote.name) }, icon = { Icon( - baselineRemote24, - contentDescription = "Remote" + baselineRemote24, contentDescription = "Remote" ) - }, - label = { Text("Remote") }, - selected = currentScreen == Screen.Remote + }, label = { Text("Remote") }, selected = currentScreen == Screen.Remote ) NavigationBarItem( - onClick = { navController.navigate(Screen.Settings.name) }, - icon = { + onClick = { navController.navigate(Screen.Settings.name) }, icon = { Icon( - baselineSettings24, - contentDescription = "Settings" + baselineSettings24, contentDescription = "Settings" ) - }, - label = { Text("Settings") }, - selected = currentScreen == Screen.Settings + }, label = { Text("Settings") }, selected = currentScreen == Screen.Settings ) } }) { innerPadding -> @@ -151,7 +124,10 @@ fun TvControllerApp( modifier = Modifier.padding(innerPadding) ) { composable(route = Screen.Camera.name) { - CameraView(eglBaseContext = rtcPeerConnection.eglBaseContext, videoTrack = rtcPeerConnection.videoTrack) + CameraView( + eglBaseContext = webRtcService.eglBaseContext, + videoTrack = webRtcService.videoTrack + ) } composable(route = Screen.Remote.name) { RemoteView( @@ -160,8 +136,7 @@ fun TvControllerApp( } composable(route = Screen.Settings.name) { SettingsView( - deviceService = deviceService, - bluetoothService = bluetoothService + deviceService = deviceService, bluetoothService = bluetoothService ) } } diff --git a/app/src/main/java/com/example/tvcontroller/client/WebClient.kt b/app/src/main/java/com/example/tvcontroller/client/WebClient.kt new file mode 100644 index 0000000..7d20e93 --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/client/WebClient.kt @@ -0,0 +1,54 @@ +package com.example.tvcontroller.client + +import android.util.Log +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.websocket.WebSockets +import io.ktor.client.request.cookie +import io.ktor.client.request.headers +import io.ktor.client.request.request +import io.ktor.client.request.setBody +import io.ktor.client.statement.HttpResponse +import io.ktor.http.HttpMethod +import org.json.JSONObject + +private const val TAG = "WebClient" + +class WebClient { + val client = HttpClient(CIO) { + install(WebSockets) + } + var defaultCookies = mutableMapOf() + + suspend fun sendRequest( + url: String, + method: HttpMethod, + headers: Map = emptyMap(), + body: String? = "" + ): HttpResponse? { + try { + val response = client.request(url) { + this.method = method + setBody(body) + headers { + headers.forEach { (key, value) -> + append(key, value) + } + } + defaultCookies.forEach { (key, value) -> + cookie(name = key, value = value) + } + } + return response + } catch (e: Exception) { + Log.e(TAG, "error sending json request", e) + } + return null + } + + suspend fun sendJsonRequest(url: String, method: HttpMethod, json: JSONObject): HttpResponse? { + val headers = mutableMapOf() + headers.put("Content-Type", "application/json") + return sendRequest(url, method, headers = headers, body = json.toString()) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/client/WebsocketClient.kt b/app/src/main/java/com/example/tvcontroller/client/WebsocketClient.kt new file mode 100644 index 0000000..b3fed0a --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/client/WebsocketClient.kt @@ -0,0 +1,54 @@ +package com.example.tvcontroller.client + +import android.util.Log +import io.ktor.client.HttpClient +import io.ktor.client.plugins.websocket.webSocket +import io.ktor.client.request.cookie +import io.ktor.http.HttpMethod +import io.ktor.websocket.DefaultWebSocketSession +import io.ktor.websocket.Frame +import io.ktor.websocket.readText +import org.json.JSONObject + +private const val TAG = "WebsocketClient" + +class WebsocketClient(private val client: HttpClient) { + private var websocket: DefaultWebSocketSession? = null + private val dataHandlers = mutableListOf<(String) -> Unit>() + + suspend fun connect(serverAddress: String, token: String) { + Log.i(TAG, "Connecting to websocket at $serverAddress") + val (host, port) = serverAddress.split(":") + val portInt = if (port.isEmpty()) 80 else port.toInt() + client.webSocket( + method = HttpMethod.Get, + host = host, + port = portInt, + path = "/ws", + request = { + cookie(name = "token", value = token) + } + ) { + Log.i(TAG, "Listening for incoming websocket messages") + websocket = this + while (true) { + val frame = incoming.receive() + Log.d(TAG, "Received frame: $frame") + if (frame is Frame.Text) { + val dataString = frame.readText() + dataHandlers.forEach { it(dataString) } + } + } + } + } + + fun onData(handler: (String) -> Unit) { + Log.d(TAG, "Adding data handler") + dataHandlers.add(handler) + } + + suspend fun sendJson(json: JSONObject) { + val frame = Frame.Text(json.toString()) + websocket?.send(frame) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/services/ControllerService.kt b/app/src/main/java/com/example/tvcontroller/services/ControllerService.kt index b198457..9c7d9ec 100644 --- a/app/src/main/java/com/example/tvcontroller/services/ControllerService.kt +++ b/app/src/main/java/com/example/tvcontroller/services/ControllerService.kt @@ -7,7 +7,6 @@ import org.json.JSONObject class ControllerService( - private val context: Context, private val bluetoothService: BluetoothService ) { private val samsungCommands = mutableMapOf() diff --git a/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt b/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt index c710a1b..71700e0 100644 --- a/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt +++ b/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt @@ -3,110 +3,76 @@ package com.example.tvcontroller.services import android.content.Context import android.content.Context.MODE_PRIVATE import android.util.Log +import com.example.tvcontroller.client.WebClient import com.example.tvcontroller.data.Integration -import com.example.tvcontroller.webrtc.RtcPeerConnection -import io.ktor.client.HttpClient import io.ktor.client.call.body -import io.ktor.client.engine.cio.CIO -import io.ktor.client.plugins.websocket.WebSockets -import io.ktor.client.plugins.websocket.webSocket -import io.ktor.client.request.cookie -import io.ktor.client.request.headers -import io.ktor.client.request.request -import io.ktor.client.request.setBody -import io.ktor.client.statement.HttpResponse import io.ktor.http.HttpMethod -import io.ktor.websocket.DefaultWebSocketSession -import io.ktor.websocket.Frame -import io.ktor.websocket.readText -import kotlinx.coroutines.runBlocking import org.json.JSONObject -import org.webrtc.IceCandidate private const val SHARED_PREFERENCES_NAME = "devices"; private const val TAG = "DeviceService" -class DeviceService(private val context: Context) { - private var client = HttpClient(CIO) { - install(WebSockets) - } - private var websocket: DefaultWebSocketSession? = null - private var serverAddress: String = "" - private var token: String = "" +class DeviceService(private val context: Context, private val client: WebClient) { + var serverAddress: String = "" + var token: String = "" + private set(value) { + field = value + updateDefaultCookies() + } private var deviceId: String = "" - var rtcPeerConnection: RtcPeerConnection? = null suspend fun initialize() { loadPreferences() if (token.isEmpty()) return - getIntegration()?.let { - connect() - } + getIntegration() + } + + private fun updateDefaultCookies() { + Log.i(TAG, "Updating default cookies with token $token") + client.defaultCookies["token"] = token + Log.i(TAG, "Default cookies: ${client.defaultCookies}") } suspend fun registerIntegration(name: String, code: String) { Log.i(TAG, "Creating integration for $name with code $code at $serverAddress") - val requestJson = JSONObject() - requestJson.put("name", name) - requestJson.put("code", code) - token = "" - deviceId = "" - try { - val response: HttpResponse = - client.request("http://$serverAddress/api/integrations/register") { - method = HttpMethod.Post - setBody(requestJson.toString()) - headers { - append("Content-Type", "application/json") - } - cookie(name = "token", value = token) - } - - val body: String = response.body() - val responseJson = JSONObject(body) - if (response.status.value != 200) { - val error = responseJson.getString("error") - Log.e(TAG, "Error getting integration: ${response.status.value} $error") - return - } - token = responseJson.getString("token") - deviceId = responseJson.getString("id") - savePreferences() - - Log.i(TAG, "Response: ${response.status.value} $body") - } catch (e: Exception) { - Log.e(TAG, "Error registering integration", e) + val requestJson = JSONObject().apply { + put("name", name) + put("code", code) } + val response = + client.sendJsonRequest( + "http://$serverAddress/api/integrations", + HttpMethod.Post, + requestJson + ) ?: return + + val responseJson = JSONObject(response.body()) + if (response.status.value != 200) { + val error = responseJson.getString("error") + Log.e(TAG, "Error getting integration: ${response.status.value} $error") + return + } + token = responseJson.getString("token") + deviceId = responseJson.getString("id") + savePreferences() } suspend fun getIntegration(): Integration? { - Log.i(TAG, "Getting integration $deviceId at $serverAddress") - try { - val response: HttpResponse = - client.request("http://$serverAddress/api/integrations/$deviceId") { - method = HttpMethod.Get - headers { - append("Authorization", "Bearer $token") - } - cookie(name = "token", value = token) - } + val response = + client.sendRequest("http://$serverAddress/api/integrations/$deviceId", HttpMethod.Get) + ?: return null - val body: String = response.body() - val responseJson = JSONObject(body) - if (response.status.value != 200) { - val error = responseJson.getString("error") - Log.e(TAG, "Error getting integration: ${response.status.value} $error") - return null - } - val integration = Integration( - responseJson.getString("id"), - responseJson.getString("name") - ) - return integration - } catch (e: Exception) { - Log.e(TAG, "Error getting integration", e) + val responseJson = JSONObject(response.body()) + if (response.status.value != 200) { + val error = responseJson.getString("error") + Log.e(TAG, "Error getting integration: ${response.status.value} $error") + return null } - return null + val integration = Integration( + responseJson.getString("id"), + responseJson.getString("name") + ) + return integration } private fun loadPreferences() { @@ -128,117 +94,6 @@ class DeviceService(private val context: Context) { } } - suspend fun connect() { - Log.i(TAG, "Connecting to websocket at $serverAddress") - val (host, port) = serverAddress.split(":") - val portInt = if (port.isEmpty()) 80 else port.toInt() - client.webSocket( - method = HttpMethod.Get, - host = host, - port = portInt, - path = "/ws", - request = { - cookie(name = "token", value = token) - } - ) { - Log.i(TAG, "Listening for incoming websocket messages") - websocket = this - while (true) { - val frame = incoming.receive() - if (frame is Frame.Text) { - val dataString = frame.readText() - val dataJson = JSONObject(dataString) - val senderId = dataJson.getString("sender") - val message = dataJson.getJSONObject("message") - val type = message.getString("type") - when (type) { - RtcPeerConnection.TYPE_OFFER -> { - Log.i(TAG, "Received offer from $senderId") - val sdp = message.getString("sdp") - rtcPeerConnection?.connect(senderId, sdp) - } - - RtcPeerConnection.TYPE_ICE_CANDIDATE -> { - Log.i(TAG, "Received ice candidate") - val candidateString = message.getString("candidate") - val candidateJson = JSONObject(candidateString) - val sdpMid = candidateJson.getString("sdpMid") - val sdpMLineIndex = candidateJson.getInt("sdpMLineIndex") - val sdp = candidateJson.getString("candidate") - val candidate = IceCandidate(sdpMid, sdpMLineIndex, sdp) - Log.i(TAG, "Candidate: $candidate") - rtcPeerConnection?.addIceCandidate(candidate) - } - } - } - } - } - } - - fun sendWebRtcAnswer(targetId: String, sdp: String) { - val messageJson = JSONObject() - messageJson.put("type", TYPE_SIGNALING) - messageJson.put("target", targetId) - messageJson.put("message", JSONObject().apply { - put("sdp", sdp) - put("type", RtcPeerConnection.TYPE_ANSWER) - }) - - runBlocking { - Log.i(TAG, "Sending answer") - val frame = Frame.Text(messageJson.toString()) - websocket?.send(frame) - } - } - - fun sendWebRtcOffer(targetId: String, sdp: String) { - val messageJson = JSONObject() - messageJson.put("type", TYPE_SIGNALING) - messageJson.put("target", targetId) - messageJson.put("message", JSONObject().apply { - put("sdp", sdp) - put("type", RtcPeerConnection.TYPE_OFFER) - }) - - runBlocking { - Log.i(TAG, "Sending offer") - val frame = Frame.Text(messageJson.toString()) - websocket?.send(frame) - } - } - - fun sendWebRtcIceCandidate(targetId: String, candidate: IceCandidate) { - val messageJson = JSONObject() - messageJson.put("type", TYPE_SIGNALING) - messageJson.put("target", targetId) - messageJson.put("message", JSONObject().apply { - put("candidate", JSONObject().apply { - put("sdpMid", candidate.sdpMid) - put("sdpMLineIndex", candidate.sdpMLineIndex) - put("candidate", candidate.sdp) - }) - put("type", RtcPeerConnection.TYPE_ICE_CANDIDATE) - }) - - runBlocking { - Log.i(TAG, "Sending ice candidate") - val frame = Frame.Text(messageJson.toString()) - websocket?.send(frame) - } - } - - fun setServerAddress(url: String) { - serverAddress = url - } - - fun getServerAddress(): String { - return serverAddress - } - - fun getToken(): String { - return token - } - companion object { const val TYPE_SIGNALING = "signaling" } diff --git a/app/src/main/java/com/example/tvcontroller/services/webrtc/RtcPeerConnection.kt b/app/src/main/java/com/example/tvcontroller/services/webrtc/RtcPeerConnection.kt new file mode 100644 index 0000000..ec6dbf5 --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/services/webrtc/RtcPeerConnection.kt @@ -0,0 +1,191 @@ +package com.example.tvcontroller.services.webrtc + +import android.content.Context +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraManager +import android.util.Log +import org.webrtc.AudioTrack +import org.webrtc.Camera2Capturer +import org.webrtc.DataChannel +import org.webrtc.DefaultVideoDecoderFactory +import org.webrtc.EglBase +import org.webrtc.HardwareVideoEncoderFactory +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.MediaStream +import org.webrtc.MediaStreamTrack +import org.webrtc.PeerConnection +import org.webrtc.PeerConnectionFactory +import org.webrtc.PeerConnectionFactory.InitializationOptions +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import org.webrtc.SimulcastVideoEncoderFactory +import org.webrtc.SoftwareVideoEncoderFactory +import org.webrtc.SurfaceTextureHelper +import org.webrtc.VideoSource +import org.webrtc.VideoTrack + +private const val TAG = "RtcPeerConnection" + +class RtcPeerConnection(private val context: Context) { + private val peerConnectionFactory by lazy { initializeFactory() } + private var iceServers = ArrayList() + var eglBaseContext: EglBase.Context = EglBase.create().eglBaseContext + private val cameraManager by lazy { context.getSystemService(Context.CAMERA_SERVICE) as CameraManager } + private val videoCapturer by lazy { createCameraCapturer() } + private val surfaceTextureHelper by lazy { createSurfaceTextureHelper() } + private val videoSource by lazy { createVideoSource() } + private val videoDecoderFactory by lazy { DefaultVideoDecoderFactory(eglBaseContext) } + private val videoEncoderFactory by lazy { + SimulcastVideoEncoderFactory( + HardwareVideoEncoderFactory(eglBaseContext, true, true), + SoftwareVideoEncoderFactory() + ) + } + private var peerConnection: PeerConnection? = null + private val iceCandidateHandlers = ArrayList<((IceCandidate) -> Unit)>() + + + fun initialize() { + var observer = object : PeerConnection.Observer { + override fun onSignalingChange(p0: PeerConnection.SignalingState?) {} + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) {} + override fun onIceConnectionReceivingChange(p0: Boolean) {} + override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) {} + override fun onIceCandidate(iceCandidate: IceCandidate?) { + iceCandidateHandlers.forEach { it(iceCandidate!!) } + } + override fun onIceCandidatesRemoved(p0: Array?) {} + override fun onAddStream(p0: MediaStream?) {} + override fun onRemoveStream(p0: MediaStream?) {} + override fun onDataChannel(p0: DataChannel?) {} + override fun onRenegotiationNeeded() {} + } + var rtcConfig = PeerConnection.RTCConfiguration(iceServers) + peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, observer) + } + + fun setIceServers(iceServerStrings: Array) { + iceServers = ArrayList() + iceServerStrings.forEach { + iceServers.add( + PeerConnection.IceServer.builder(it).createIceServer() + ) + } + } + + fun createAudioTrack(): AudioTrack { + val audioConstraints = MediaConstraints() + val audioSource = peerConnectionFactory.createAudioSource(audioConstraints) + val audioTrack = peerConnectionFactory.createAudioTrack("audio_track", audioSource) + return audioTrack + } + + fun addTack(track: MediaStreamTrack) { + peerConnection?.addTrack(track) + } + + fun createVideoTrack(): VideoTrack { + val videoTrack = peerConnectionFactory.createVideoTrack("video_track", videoSource) + return videoTrack + } + + fun setLocalDescription(sessionDescription: SessionDescription, callback: () -> Unit) { + val onLocalDescriptionSet = object : SdpObserver { + override fun onSetSuccess() { + callback() + } + + override fun onCreateSuccess(p0: SessionDescription?) {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + peerConnection?.setLocalDescription(onLocalDescriptionSet, sessionDescription) + } + + fun setRemoteDescription(sessionDescription: SessionDescription, callback: () -> Unit) { + val onRemoteDescriptionSet = object : SdpObserver { + override fun onSetSuccess() { + callback() + } + + override fun onCreateSuccess(p0: SessionDescription?) {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + peerConnection?.setRemoteDescription(onRemoteDescriptionSet, sessionDescription) + } + + fun createAnswer(mediaConstraints: MediaConstraints, callback: (SessionDescription) -> Unit) { + val onAnswerCreated = object : SdpObserver { + override fun onCreateSuccess(sessionDescription: SessionDescription?) { + callback(sessionDescription!!) + } + + override fun onSetSuccess() {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + peerConnection?.createAnswer(onAnswerCreated, mediaConstraints) + } + + fun addIceCandidate(iceCandidate: IceCandidate) { + Log.i(TAG, "Adding ice candidate $iceCandidate to $peerConnection") + peerConnection?.addIceCandidate(iceCandidate) + } + + fun onIceCandidate(handler: (IceCandidate) -> Unit) { + iceCandidateHandlers.add(handler) + } + + private fun initializeFactory(): PeerConnectionFactory { + val initOptions = InitializationOptions.builder(context).createInitializationOptions() + PeerConnectionFactory.initialize(initOptions) + + val options = PeerConnectionFactory.Options() + val peerConnectionFactory = + PeerConnectionFactory.builder().setVideoDecoderFactory(videoDecoderFactory) + .setVideoEncoderFactory(videoEncoderFactory).setOptions(options) + .createPeerConnectionFactory() + return peerConnectionFactory + } + + private fun createCameraCapturer(): Camera2Capturer { + val ids = cameraManager.cameraIdList + var foundCamera = false; + var cameraId = "" + for (id in ids) { + val characteristics = cameraManager.getCameraCharacteristics(id) + val facing = characteristics.get(CameraCharacteristics.LENS_FACING) + if (facing == CameraCharacteristics.LENS_FACING_BACK) { + cameraId = id + foundCamera = true + break + } + } + if (!foundCamera) { + cameraId = ids.first() + } + val cameraCapturer = Camera2Capturer(context, cameraId, null) + return cameraCapturer + } + + private fun createSurfaceTextureHelper(): SurfaceTextureHelper { + val surfaceTextureHelper = + SurfaceTextureHelper.create("SurfaceTextureHelperThread", eglBaseContext) + return surfaceTextureHelper + } + + private fun createVideoSource(): VideoSource { + val videoSource = peerConnectionFactory.createVideoSource(false) + videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver) + videoCapturer.startCapture(1920, 1080, 30) + return videoSource + } + + companion object { + const val TYPE_OFFER = "offer" + const val TYPE_ANSWER = "answer" + const val TYPE_ICE_CANDIDATE = "ice_candidate" + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/services/webrtc/WebRtcService.kt b/app/src/main/java/com/example/tvcontroller/services/webrtc/WebRtcService.kt new file mode 100644 index 0000000..7365939 --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/services/webrtc/WebRtcService.kt @@ -0,0 +1,123 @@ +package com.example.tvcontroller.services.webrtc + +import android.content.Context +import android.util.Log +import com.example.tvcontroller.client.WebsocketClient +import com.example.tvcontroller.services.DeviceService +import kotlinx.coroutines.runBlocking +import org.json.JSONObject +import org.webrtc.EglBase +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.SessionDescription + +private const val TAG = "WebRtcService" + +class WebRtcService(private val context: Context, private val websocketClient: WebsocketClient) { + private val rtcPeerConnection by lazy { createRtcPeerConnection() } + val videoTrack by lazy { rtcPeerConnection.createVideoTrack() } + val audioTrack by lazy { rtcPeerConnection.createAudioTrack() } + val eglBaseContext: EglBase.Context + get() = rtcPeerConnection.eglBaseContext + private var peerId: String = "" + + fun connect() { + Log.i(TAG, "Connecting to signaling server") + websocketClient.onData(this::handleData) + } + + private fun createRtcPeerConnection(): RtcPeerConnection { + val iceServers = arrayOf("stun:stun.l.google.com:19302") + val webRtcService = this + val rtcPeerConnection = RtcPeerConnection(context).apply { + setIceServers(iceServers) + onIceCandidate(webRtcService::sendIceCandidate) + initialize() + } + return rtcPeerConnection + } + + private fun handleOffer(sdp: String) { + var mediaConstraints = MediaConstraints() + val remoteSessionDescription = SessionDescription(SessionDescription.Type.OFFER, sdp) + rtcPeerConnection.apply { + addTack(audioTrack) + addTack(videoTrack) + setRemoteDescription(remoteSessionDescription) { + createAnswer(mediaConstraints) { localSessionDescription -> + setLocalDescription(localSessionDescription) { + sendAnswer( + peerId, localSessionDescription.description ?: "" + ) + } + } + } + } + } + + private fun handleData(data: String) { + Log.d(TAG, "Received data: $data") + val dataJson = JSONObject(data) + val senderId = dataJson.getString("sender") + val message = dataJson.getJSONObject("message") + val type = message.getString("type") + when (type) { + RtcPeerConnection.TYPE_OFFER -> { + Log.i(TAG, "Received offer from $senderId") + val sdp = message.getString("sdp") + peerId = senderId + handleOffer(sdp) + } + + RtcPeerConnection.TYPE_ICE_CANDIDATE -> { + val candidateString = message.getString("candidate") + handleReceiveIceCandidate(candidateString) + } + } + } + + private fun handleReceiveIceCandidate(candidateString: String) { + Log.i(TAG, "Received ice candidate") + val candidateJson = JSONObject(candidateString) + val sdpMid = candidateJson.getString("sdpMid") + val sdpMLineIndex = candidateJson.getInt("sdpMLineIndex") + val sdp = candidateJson.getString("candidate") + val candidate = IceCandidate(sdpMid, sdpMLineIndex, sdp) + Log.i(TAG, "Candidate: $candidate") + rtcPeerConnection.addIceCandidate(candidate) + } + + private fun sendIceCandidate(iceCandidate: IceCandidate) { + val messageJson = JSONObject() + messageJson.put("type", DeviceService.Companion.TYPE_SIGNALING) + messageJson.put("target", peerId) + messageJson.put("message", JSONObject().apply { + put("candidate", JSONObject().apply { + put("sdpMid", iceCandidate.sdpMid) + put("sdpMLineIndex", iceCandidate.sdpMLineIndex) + put("candidate", iceCandidate.sdp) + }) + put("type", RtcPeerConnection.TYPE_ICE_CANDIDATE) + }) + + runBlocking { + Log.i(TAG, "Sending ice candidate") + websocketClient.sendJson(messageJson) + } + } + + private fun sendAnswer(targetId: String, sdp: String) { + val messageJson = JSONObject() + messageJson.put("type", DeviceService.Companion.TYPE_SIGNALING) + messageJson.put("target", targetId) + messageJson.put("message", JSONObject().apply { + put("sdp", sdp) + put("type", RtcPeerConnection.TYPE_ANSWER) + }) + + runBlocking { + Log.i(TAG, "Sending answer") + websocketClient.sendJson(messageJson) + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/ui/views/SettingsViewModel.kt b/app/src/main/java/com/example/tvcontroller/ui/views/SettingsViewModel.kt index cf62376..d066b30 100644 --- a/app/src/main/java/com/example/tvcontroller/ui/views/SettingsViewModel.kt +++ b/app/src/main/java/com/example/tvcontroller/ui/views/SettingsViewModel.kt @@ -15,11 +15,13 @@ import com.example.tvcontroller.services.DeviceService import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch +private const val TAG = "SettingsViewModel" + class SettingsViewModel( private val deviceService: DeviceService, private val bluetoothService: BluetoothService ) : ViewModel() { - var serverAddress by mutableStateOf(deviceService.getServerAddress()) + var serverAddress by mutableStateOf(deviceService.serverAddress) private set var deviceName by mutableStateOf(Build.MANUFACTURER + " " + Build.MODEL) private set @@ -48,17 +50,16 @@ class SettingsViewModel( fun connect() { //Log.i("SettingsScreen", "Save settings: $serverUrl, $deviceName, $registrationCode") viewModelScope.launch { - deviceService.setServerAddress(serverAddress) + deviceService.serverAddress = serverAddress deviceService.registerIntegration(deviceName, registrationCode) updateConnectionState() updateDeviceInfo() - deviceService.connect() } } private fun updateConnectionState() { - Log.i("SettingsViewModel", "Device token: ${deviceService.getToken()}") - connectionState = if (deviceService.getToken().isEmpty()) { + Log.i(TAG, "Device token: ${deviceService.token}") + connectionState = if (deviceService.token.isEmpty()) { Settings.ConnectionState.Unregistered } else { Settings.ConnectionState.Registered diff --git a/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt b/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt deleted file mode 100644 index 96f7b65..0000000 --- a/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt +++ /dev/null @@ -1,113 +0,0 @@ -package com.example.tvcontroller.webrtc - -import android.graphics.ImageFormat.YUV_420_888 -import android.util.Log -import androidx.camera.core.ImageAnalysis -import androidx.camera.core.ImageProxy -import org.webrtc.CapturerObserver -import org.webrtc.VideoFrame -import java.nio.ByteBuffer - -private const val TAG = "CameraXCapturer" - -class CameraXCapturer : ImageAnalysis.Analyzer { - var capturerObserver: CapturerObserver? = null - - override fun analyze(image: ImageProxy) { - if (image.format != YUV_420_888) throw Exception("Unsupported format") - var videoFrame = imageProxyToVideoFrame(image) - Log.i(TAG, "Handing frame to capturer observer $capturerObserver") - capturerObserver?.onFrameCaptured(videoFrame) ?: image.close() - Log.i(TAG, "Frame handled by capturer observer") - image.close() - } - - fun imageProxyToVideoFrame(image: ImageProxy): VideoFrame { - var buffer = object : VideoFrame.I420Buffer { - private var refCount = 0 - override fun getWidth(): Int { - return image.width - } - - override fun getHeight(): Int { - return image.height - } - - override fun toI420(): VideoFrame.I420Buffer? { - return this - } - - override fun retain() { - refCount++ - } - - override fun release() { - refCount-- - if (refCount == 0) { - image.close() - } - } - - override fun cropAndScale( - cropX: Int, - cropY: Int, - cropWidth: Int, - cropHeight: Int, - scaleWidth: Int, - scaleHeight: Int - ): VideoFrame.Buffer? { - return this - } - - override fun getDataY(): ByteBuffer? { - val format = image.format - if (format == YUV_420_888) { - return image.planes[0].buffer - } - return image.planes[0].buffer - } - - override fun getDataU(): ByteBuffer? { - val format = image.format - if (format == YUV_420_888) { - return image.planes[1].buffer - } - return image.planes[0].buffer - } - - override fun getDataV(): ByteBuffer? { - val format = image.format - if (format == YUV_420_888) { - return image.planes[2].buffer - } - return image.planes[0].buffer - } - - override fun getStrideY(): Int { - val format = image.format - if (format == YUV_420_888) { - return image.planes[0].pixelStride - } - return image.planes[0].pixelStride - } - - override fun getStrideU(): Int { - val format = image.format - if (format == YUV_420_888) { - return image.planes[1].pixelStride - } - return image.planes[0].pixelStride - } - - override fun getStrideV(): Int { - val format = image.format - if (format == YUV_420_888) { - return image.planes[2].pixelStride - } - return image.planes[0].pixelStride - } - } - var videoFrame = VideoFrame(buffer, 0, 0) - return videoFrame - } -} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt b/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt deleted file mode 100644 index 81096a8..0000000 --- a/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt +++ /dev/null @@ -1,282 +0,0 @@ -package com.example.tvcontroller.webrtc - -import android.content.Context -import android.hardware.camera2.CameraCharacteristics -import android.hardware.camera2.CameraManager -import android.util.Log -import androidx.compose.ui.Modifier -import com.example.tvcontroller.services.DeviceService -import org.webrtc.AudioTrack -import org.webrtc.Camera2Capturer -import org.webrtc.DataChannel -import org.webrtc.DefaultVideoDecoderFactory -import org.webrtc.DefaultVideoEncoderFactory -import org.webrtc.EglBase -import org.webrtc.HardwareVideoEncoderFactory -import org.webrtc.IceCandidate -import org.webrtc.MediaConstraints -import org.webrtc.MediaStream -import org.webrtc.PeerConnection -import org.webrtc.PeerConnectionFactory -import org.webrtc.PeerConnectionFactory.InitializationOptions -import org.webrtc.RtpSender -import org.webrtc.RtpTransceiver -import org.webrtc.SdpObserver -import org.webrtc.SessionDescription -import org.webrtc.SimulcastVideoEncoderFactory -import org.webrtc.SoftwareVideoEncoderFactory -import org.webrtc.SurfaceTextureHelper -import org.webrtc.VideoSource -import org.webrtc.VideoTrack -import kotlin.getValue - -private const val TAG = "RtcPeerConnection" - -class RtcPeerConnection(private val context: Context) : PeerConnection.Observer { - private val peerConnectionFactory by lazy { initializeFactory() } - private val iceServers by lazy { initializeIceServers() } - private val audioTrack by lazy { createAudioTrack() } - val videoTrack by lazy { createVideoTrack() } - var eglBaseContext: EglBase.Context = EglBase.create().eglBaseContext - private val cameraManager by lazy { context.getSystemService(Context.CAMERA_SERVICE) as CameraManager } - private val videoCapturer by lazy { createCameraCapturer() } - private val surfaceTextureHelper by lazy { createSurfaceTextureHelper() } - private val videoSource by lazy { createVideoSource() } - private val videoDecoderFactory by lazy { DefaultVideoDecoderFactory(eglBaseContext) } - private val videoEncoderFactory by lazy { - SimulcastVideoEncoderFactory( - HardwareVideoEncoderFactory(eglBaseContext, true, true), - SoftwareVideoEncoderFactory() - ) - } - private var peerConnection: PeerConnection? = null - var deviceService: DeviceService? = null - - private var peerId: String = "" - - private var offer = "" - - private fun initializeFactory(): PeerConnectionFactory { - val initOptions = InitializationOptions.builder(context).createInitializationOptions() - PeerConnectionFactory.initialize(initOptions) - - val options = PeerConnectionFactory.Options() - val peerConnectionFactory = - PeerConnectionFactory.builder().setVideoDecoderFactory(videoDecoderFactory) - .setVideoEncoderFactory(videoEncoderFactory).setOptions(options) - .createPeerConnectionFactory() - return peerConnectionFactory - } - - private fun initializeIceServers(): ArrayList { - val iceServers = ArrayList() - iceServers.add( - PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer() - ) - return iceServers - } - - private fun createAudioTrack(): AudioTrack { - val audioConstraints = MediaConstraints() - val audioSource = peerConnectionFactory.createAudioSource(audioConstraints) - val localAudioTrack = peerConnectionFactory.createAudioTrack("audio_track", audioSource) - return localAudioTrack - } - - private fun createVideoTrack(): VideoTrack { - val localVideoTrack = peerConnectionFactory.createVideoTrack("video_track", videoSource) - //localVideoTrack.setEnabled(true) - return localVideoTrack - } - - private fun createCameraCapturer(): Camera2Capturer { - val ids = cameraManager.cameraIdList - var foundCamera = false; - var cameraId = "" - for (id in ids) { - val characteristics = cameraManager.getCameraCharacteristics(id) - val facing = characteristics.get(CameraCharacteristics.LENS_FACING) - if (facing == CameraCharacteristics.LENS_FACING_BACK) { - cameraId = id - foundCamera = true - break - } - } - if (!foundCamera) { - cameraId = ids.first() - } - val cameraCapturer = Camera2Capturer(context, cameraId, null) - return cameraCapturer - } - - private fun createSurfaceTextureHelper(): SurfaceTextureHelper { - val surfaceTextureHelper = - SurfaceTextureHelper.create("SurfaceTextureHelperThread", eglBaseContext) - return surfaceTextureHelper - } - - private fun createVideoSource(): VideoSource { - val videoSource = peerConnectionFactory.createVideoSource(false) - videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver) - videoCapturer.startCapture(1920, 1080, 30) - return videoSource - } - - fun connect(targetId: String, sdp: String) { - peerId = targetId - offer = sdp - Log.i(TAG, "Connecting rtc peer connection") - var rtcConfig = PeerConnection.RTCConfiguration(iceServers) - Log.i(TAG, "Creating peer connection") - peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this) - Log.i(TAG, "Peer connection created") - //peerConnection?.addTransceiver(audioTrack, RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.SEND_ONLY)) - //peerConnection?.addTransceiver( videoTrack, RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.SEND_ONLY) ) - peerConnection?.addTrack(audioTrack) - peerConnection?.addTrack(videoTrack) - handleOffer(sdp) - } - - fun handleOffer(sdp: String) { - var mediaConstraints = MediaConstraints() - var localSessionDescription: SessionDescription? = null - val remoteSessionDescription = SessionDescription(SessionDescription.Type.OFFER, sdp) - - Log.i(TAG, "Handling offer $sdp") - val onLocalDescriptionSet = object : SdpObserver { - override fun onSetSuccess() { - Log.i(TAG, "Local description set") - peerConnection?.transceivers?.forEach { - - Log.i(TAG, "${it.mediaType} Transceiver: ${it.currentDirection}") - } - deviceService?.sendWebRtcAnswer( - peerId, localSessionDescription?.description ?: "" - ) - } - - override fun onCreateSuccess(sessionDescription: SessionDescription?) {} - override fun onCreateFailure(p0: String?) {} - override fun onSetFailure(p0: String?) {} - } - - val onAnswerCreated = object : SdpObserver { - override fun onCreateSuccess(sessionDescription: SessionDescription?) { - Log.i(TAG, "Answer created") - localSessionDescription = sessionDescription - peerConnection?.setLocalDescription(onLocalDescriptionSet, sessionDescription) - } - - override fun onSetSuccess() {} - override fun onCreateFailure(p0: String?) {} - override fun onSetFailure(p0: String?) {} - } - val onRemoteDescriptionSet = object : SdpObserver { - override fun onSetSuccess() { - Log.i(TAG, "Remote description set") - peerConnection?.createAnswer(onAnswerCreated, mediaConstraints) - } - - override fun onCreateSuccess(p0: SessionDescription?) {} - override fun onCreateFailure(p0: String?) {} - override fun onSetFailure(p0: String?) {} - } - Log.i(TAG, "Setting remote description") - peerConnection?.setRemoteDescription(onRemoteDescriptionSet, remoteSessionDescription) - } - - fun createOffer() { - var mediaConstraints = MediaConstraints() - var localSessionDescription: SessionDescription? = null - - val onLocalDescriptionSet = object : SdpObserver { - override fun onSetSuccess() { - Log.i(TAG, "Local description set") - peerConnection?.transceivers?.forEach { - Log.i(TAG, "${it.mediaType} Transceiver: ${it.currentDirection}") - } - deviceService?.sendWebRtcOffer( - peerId, localSessionDescription?.description ?: "" - ) - } - - override fun onCreateSuccess(sessionDescription: SessionDescription?) {} - override fun onCreateFailure(p0: String?) {} - override fun onSetFailure(p0: String?) {} - } - val onOfferCreated = object : SdpObserver { - override fun onCreateSuccess(sessionDescription: SessionDescription?) { - Log.i(TAG, "Offer created") - peerConnection?.setLocalDescription(onLocalDescriptionSet, sessionDescription) - } - - override fun onSetSuccess() {} - override fun onCreateFailure(p0: String?) {} - override fun onSetFailure(p0: String?) {} - } - Log.i(TAG, "Creating offer") - peerConnection?.createOffer(onOfferCreated, mediaConstraints) - } - - fun addIceCandidate(iceCandidate: IceCandidate) { - Log.i(TAG, "Adding ice candidate $iceCandidate to $peerConnection") - peerConnection?.addIceCandidate(iceCandidate) - } - - override fun onSignalingChange(p0: PeerConnection.SignalingState?) { - Log.i(TAG, "onSignalingChange: $p0") - } - - override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { - Log.i(TAG, "onIceConnectionChange: $p0") - if (p0 == PeerConnection.IceConnectionState.CONNECTED) { - videoCapturer.startCapture(1280, 720, 30) - } - } - - override fun onIceConnectionReceivingChange(p0: Boolean) { - Log.i(TAG, "onIceConnectionReceivingChange: $p0") - } - - override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) { - Log.i(TAG, "onIceGatheringChange: $p0") - } - - override fun onIceCandidate(p0: IceCandidate?) { - Log.i(TAG, "onIceCandidate: $p0") - p0?.let { - deviceService?.sendWebRtcIceCandidate(peerId, it) - } - } - - override fun onIceCandidatesRemoved(p0: Array?) { - Log.i(TAG, "onIceCandidatesRemoved: $p0") - } - - override fun onAddStream(p0: MediaStream?) { - Log.i(TAG, "onAddStream: $p0") - } - - override fun onRemoveStream(p0: MediaStream?) { - Log.i(TAG, "onRemoveStream: $p0") - } - - override fun onDataChannel(p0: DataChannel?) { - Log.i(TAG, "onDataChannel: $p0") - } - - override fun onRenegotiationNeeded() { - Log.i(TAG, "onRenegotiationNeeded") - //if (offer.isNotEmpty()) { - // handleOffer(offer) - //} else { - // createOffer() - //} - } - - companion object { - const val TYPE_OFFER = "offer" - const val TYPE_ANSWER = "answer" - const val TYPE_ICE_CANDIDATE = "ice_candidate" - } -} \ No newline at end of file