diff --git a/app/build.gradle.kts b/app/build.gradle.kts index c4f5c46..3557b12 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -61,6 +61,7 @@ dependencies { implementation(libs.androidx.camera.mlkit.vision) implementation(libs.androidx.camera.extensions) implementation(libs.material3) + implementation(libs.stream.webrtc.android) testImplementation(libs.junit) androidTestImplementation(libs.androidx.junit) androidTestImplementation(libs.androidx.espresso.core) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 57a7af3..decedb2 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -12,6 +12,7 @@ + diff --git a/app/src/main/java/com/example/tvcontroller/MainActivity.kt b/app/src/main/java/com/example/tvcontroller/MainActivity.kt index 66f754c..e94aebd 100644 --- a/app/src/main/java/com/example/tvcontroller/MainActivity.kt +++ b/app/src/main/java/com/example/tvcontroller/MainActivity.kt @@ -13,46 +13,50 @@ import androidx.compose.material3.NavigationBarItem import androidx.compose.material3.Scaffold import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue import androidx.compose.ui.Modifier +import androidx.compose.ui.res.painterResource +import androidx.core.app.ActivityCompat +import androidx.lifecycle.lifecycleScope import androidx.navigation.NavHostController import androidx.navigation.compose.NavHost import androidx.navigation.compose.composable import androidx.navigation.compose.currentBackStackEntryAsState import androidx.navigation.compose.rememberNavController -import com.example.tvcontroller.ui.theme.TVControllerTheme -import androidx.compose.runtime.getValue -import androidx.compose.ui.res.painterResource -import androidx.core.app.ActivityCompat -import androidx.lifecycle.lifecycleScope +import com.example.tvcontroller.client.WebClient +import com.example.tvcontroller.client.WebsocketClient import com.example.tvcontroller.services.BluetoothService -import com.example.tvcontroller.services.CameraService import com.example.tvcontroller.services.ControllerService import com.example.tvcontroller.services.DeviceService +import com.example.tvcontroller.services.webrtc.WebRtcService +import com.example.tvcontroller.ui.theme.TVControllerTheme import com.example.tvcontroller.ui.views.CameraView import com.example.tvcontroller.ui.views.RemoteView import com.example.tvcontroller.ui.views.SettingsView import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch - const val TAG = "MainActivity" class MainActivity : ComponentActivity() { - private lateinit var bluetoothService: BluetoothService - private lateinit var deviceService: DeviceService - private lateinit var cameraService: CameraService - private lateinit var controllerService: ControllerService + private val webClient by lazy { WebClient() } + private val websocketClient by lazy { WebsocketClient(webClient.client) } + private val webRtcService by lazy { WebRtcService(applicationContext, websocketClient) } + private val bluetoothService by lazy { BluetoothService(applicationContext) } + private val deviceService by lazy { DeviceService(applicationContext, webClient) } + private val controllerService by lazy { ControllerService(bluetoothService) } override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - bluetoothService = BluetoothService(this.applicationContext) - deviceService = DeviceService(this.applicationContext) - cameraService = CameraService(this.applicationContext) - controllerService = ControllerService(this.applicationContext, bluetoothService) checkPermissions() - lifecycleScope.launch(Dispatchers.IO) { - deviceService.initialize() + lifecycleScope.launch { + deviceService.initialize() + if (deviceService.serverAddress.isEmpty() || deviceService.token.isEmpty()) return@launch + lifecycleScope.launch(Dispatchers.IO) { + websocketClient.connect(deviceService.serverAddress, deviceService.token) + } + webRtcService.connect() } enableEdgeToEdge() setContent { @@ -60,7 +64,8 @@ class MainActivity : ComponentActivity() { TvControllerApp( deviceService = deviceService, controllerService = controllerService, - bluetoothService = bluetoothService + bluetoothService = bluetoothService, + webRtcService = webRtcService ) } } @@ -68,9 +73,6 @@ class MainActivity : ComponentActivity() { private fun checkPermissions() { Log.i(TAG, "Checking permissions") - if (!cameraService.hasRequiredPermissions()) { - ActivityCompat.requestPermissions(this, CameraService.CAMERAX_PERMISSIONS, 0) - } if (!bluetoothService.hasRequiredPermissions()) { Log.i(TAG, "Requesting Bluetooth permissions") ActivityCompat.requestPermissions(this, BluetoothService.BLUETOOTH_PERMISSIONS, 0) @@ -83,7 +85,8 @@ fun TvControllerApp( navController: NavHostController = rememberNavController(), deviceService: DeviceService, controllerService: ControllerService, - bluetoothService: BluetoothService + bluetoothService: BluetoothService, + webRtcService: WebRtcService ) { val backStackEntry by navController.currentBackStackEntryAsState() val currentScreen = Screen.valueOf(backStackEntry?.destination?.route ?: Screen.Camera.name) @@ -93,47 +96,38 @@ fun TvControllerApp( Scaffold(modifier = Modifier.fillMaxSize(), bottomBar = { NavigationBar { NavigationBarItem( - onClick = { navController.navigate(Screen.Camera.name) }, - icon = { + onClick = { navController.navigate(Screen.Camera.name) }, icon = { Icon( - baselineCamera24, - contentDescription = "Camera" + baselineCamera24, contentDescription = "Camera" ) - }, - label = { Text("Camera") }, - selected = currentScreen == Screen.Camera + }, label = { Text("Camera") }, selected = currentScreen == Screen.Camera ) NavigationBarItem( - onClick = { navController.navigate(Screen.Remote.name) }, - icon = { + onClick = { navController.navigate(Screen.Remote.name) }, icon = { Icon( - baselineRemote24, - contentDescription = "Remote" + baselineRemote24, contentDescription = "Remote" ) - }, - label = { Text("Remote") }, - selected = currentScreen == Screen.Remote + }, label = { Text("Remote") }, selected = currentScreen == Screen.Remote ) NavigationBarItem( - onClick = { navController.navigate(Screen.Settings.name) }, - icon = { + onClick = { navController.navigate(Screen.Settings.name) }, icon = { Icon( - baselineSettings24, - contentDescription = "Settings" + baselineSettings24, contentDescription = "Settings" ) - }, - label = { Text("Settings") }, - selected = currentScreen == Screen.Settings + }, label = { Text("Settings") }, selected = currentScreen == Screen.Settings ) } }) { innerPadding -> NavHost( navController = navController, - startDestination = Screen.Remote.name, + startDestination = Screen.Settings.name, modifier = Modifier.padding(innerPadding) ) { composable(route = Screen.Camera.name) { - CameraView() + CameraView( + eglBaseContext = webRtcService.eglBaseContext, + videoTrack = webRtcService.videoTrack + ) } composable(route = Screen.Remote.name) { RemoteView( @@ -142,8 +136,7 @@ fun TvControllerApp( } composable(route = Screen.Settings.name) { SettingsView( - deviceService = deviceService, - bluetoothService = bluetoothService + deviceService = deviceService, bluetoothService = bluetoothService ) } } diff --git a/app/src/main/java/com/example/tvcontroller/client/WebClient.kt b/app/src/main/java/com/example/tvcontroller/client/WebClient.kt new file mode 100644 index 0000000..7d20e93 --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/client/WebClient.kt @@ -0,0 +1,54 @@ +package com.example.tvcontroller.client + +import android.util.Log +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.websocket.WebSockets +import io.ktor.client.request.cookie +import io.ktor.client.request.headers +import io.ktor.client.request.request +import io.ktor.client.request.setBody +import io.ktor.client.statement.HttpResponse +import io.ktor.http.HttpMethod +import org.json.JSONObject + +private const val TAG = "WebClient" + +class WebClient { + val client = HttpClient(CIO) { + install(WebSockets) + } + var defaultCookies = mutableMapOf() + + suspend fun sendRequest( + url: String, + method: HttpMethod, + headers: Map = emptyMap(), + body: String? = "" + ): HttpResponse? { + try { + val response = client.request(url) { + this.method = method + setBody(body) + headers { + headers.forEach { (key, value) -> + append(key, value) + } + } + defaultCookies.forEach { (key, value) -> + cookie(name = key, value = value) + } + } + return response + } catch (e: Exception) { + Log.e(TAG, "error sending json request", e) + } + return null + } + + suspend fun sendJsonRequest(url: String, method: HttpMethod, json: JSONObject): HttpResponse? { + val headers = mutableMapOf() + headers.put("Content-Type", "application/json") + return sendRequest(url, method, headers = headers, body = json.toString()) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/client/WebsocketClient.kt b/app/src/main/java/com/example/tvcontroller/client/WebsocketClient.kt new file mode 100644 index 0000000..b3fed0a --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/client/WebsocketClient.kt @@ -0,0 +1,54 @@ +package com.example.tvcontroller.client + +import android.util.Log +import io.ktor.client.HttpClient +import io.ktor.client.plugins.websocket.webSocket +import io.ktor.client.request.cookie +import io.ktor.http.HttpMethod +import io.ktor.websocket.DefaultWebSocketSession +import io.ktor.websocket.Frame +import io.ktor.websocket.readText +import org.json.JSONObject + +private const val TAG = "WebsocketClient" + +class WebsocketClient(private val client: HttpClient) { + private var websocket: DefaultWebSocketSession? = null + private val dataHandlers = mutableListOf<(String) -> Unit>() + + suspend fun connect(serverAddress: String, token: String) { + Log.i(TAG, "Connecting to websocket at $serverAddress") + val (host, port) = serverAddress.split(":") + val portInt = if (port.isEmpty()) 80 else port.toInt() + client.webSocket( + method = HttpMethod.Get, + host = host, + port = portInt, + path = "/ws", + request = { + cookie(name = "token", value = token) + } + ) { + Log.i(TAG, "Listening for incoming websocket messages") + websocket = this + while (true) { + val frame = incoming.receive() + Log.d(TAG, "Received frame: $frame") + if (frame is Frame.Text) { + val dataString = frame.readText() + dataHandlers.forEach { it(dataString) } + } + } + } + } + + fun onData(handler: (String) -> Unit) { + Log.d(TAG, "Adding data handler") + dataHandlers.add(handler) + } + + suspend fun sendJson(json: JSONObject) { + val frame = Frame.Text(json.toString()) + websocket?.send(frame) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/services/ControllerService.kt b/app/src/main/java/com/example/tvcontroller/services/ControllerService.kt index b198457..9c7d9ec 100644 --- a/app/src/main/java/com/example/tvcontroller/services/ControllerService.kt +++ b/app/src/main/java/com/example/tvcontroller/services/ControllerService.kt @@ -7,7 +7,6 @@ import org.json.JSONObject class ControllerService( - private val context: Context, private val bluetoothService: BluetoothService ) { private val samsungCommands = mutableMapOf() diff --git a/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt b/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt index 9399bc5..71700e0 100644 --- a/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt +++ b/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt @@ -3,108 +3,76 @@ package com.example.tvcontroller.services import android.content.Context import android.content.Context.MODE_PRIVATE import android.util.Log +import com.example.tvcontroller.client.WebClient import com.example.tvcontroller.data.Integration -import io.ktor.client.engine.cio.* -import io.ktor.client.* import io.ktor.client.call.body -import io.ktor.client.plugins.cookies.HttpCookies -import io.ktor.client.plugins.websocket.WebSockets -import io.ktor.client.plugins.websocket.webSocket -import io.ktor.client.request.HttpRequestBuilder -import io.ktor.client.request.cookie -import io.ktor.client.request.headers -import io.ktor.client.request.request -import io.ktor.client.request.setBody -import io.ktor.client.statement.HttpResponse -import io.ktor.http.Cookie import io.ktor.http.HttpMethod -import io.ktor.websocket.Frame -import io.ktor.websocket.readText -import kotlinx.coroutines.runBlocking import org.json.JSONObject private const val SHARED_PREFERENCES_NAME = "devices"; private const val TAG = "DeviceService" -class DeviceService(private val context: Context) { - private var client = HttpClient(CIO) { - install(WebSockets) - } - private var serverAddress: String = "" - private var token: String = "" +class DeviceService(private val context: Context, private val client: WebClient) { + var serverAddress: String = "" + var token: String = "" + private set(value) { + field = value + updateDefaultCookies() + } private var deviceId: String = "" suspend fun initialize() { loadPreferences() if (token.isEmpty()) return - getIntegration()?.let { - connect() - } + getIntegration() + } + + private fun updateDefaultCookies() { + Log.i(TAG, "Updating default cookies with token $token") + client.defaultCookies["token"] = token + Log.i(TAG, "Default cookies: ${client.defaultCookies}") } suspend fun registerIntegration(name: String, code: String) { Log.i(TAG, "Creating integration for $name with code $code at $serverAddress") - val requestJson = JSONObject() - requestJson.put("name", name) - requestJson.put("code", code) - token = "" - deviceId = "" - try { - val response: HttpResponse = - client.request("http://$serverAddress/api/integrations/register") { - method = HttpMethod.Post - setBody(requestJson.toString()) - headers { - append("Content-Type", "application/json") - } - cookie(name = "token", value = token) - } - - val body: String = response.body() - val responseJson = JSONObject(body) - if (response.status.value != 200) { - val error = responseJson.getString("error") - Log.e(TAG, "Error getting integration: ${response.status.value} $error") - return - } - token = responseJson.getString("token") - deviceId = responseJson.getString("id") - savePreferences() - - Log.i(TAG, "Response: ${response.status.value} $body") - } catch (e: Exception) { - Log.e(TAG, "Error registering integration", e) + val requestJson = JSONObject().apply { + put("name", name) + put("code", code) } + val response = + client.sendJsonRequest( + "http://$serverAddress/api/integrations", + HttpMethod.Post, + requestJson + ) ?: return + + val responseJson = JSONObject(response.body()) + if (response.status.value != 200) { + val error = responseJson.getString("error") + Log.e(TAG, "Error getting integration: ${response.status.value} $error") + return + } + token = responseJson.getString("token") + deviceId = responseJson.getString("id") + savePreferences() } suspend fun getIntegration(): Integration? { - Log.i(TAG, "Getting integration $deviceId at $serverAddress") - try { - val response: HttpResponse = - client.request("http://$serverAddress/api/integrations/$deviceId") { - method = HttpMethod.Get - headers { - append("Authorization", "Bearer $token") - } - cookie(name = "token", value = token) - } + val response = + client.sendRequest("http://$serverAddress/api/integrations/$deviceId", HttpMethod.Get) + ?: return null - val body: String = response.body() - val responseJson = JSONObject(body) - if (response.status.value != 200) { - val error = responseJson.getString("error") - Log.e(TAG, "Error getting integration: ${response.status.value} $error") - return null - } - val integration = Integration( - responseJson.getString("id"), - responseJson.getString("name") - ) - return integration - } catch (e: Exception) { - Log.e(TAG, "Error getting integration", e) + val responseJson = JSONObject(response.body()) + if (response.status.value != 200) { + val error = responseJson.getString("error") + Log.e(TAG, "Error getting integration: ${response.status.value} $error") + return null } - return null + val integration = Integration( + responseJson.getString("id"), + responseJson.getString("name") + ) + return integration } private fun loadPreferences() { @@ -126,41 +94,7 @@ class DeviceService(private val context: Context) { } } - fun connect() { - Log.i(TAG, "Connecting to websocket at $serverAddress") - runBlocking { - val (host, port) = serverAddress.split(":") - val portInt = if (port.isEmpty()) 80 else port.toInt() - client.webSocket( - method = HttpMethod.Get, - host = host, - port = portInt, - path = "/ws", - request = { - cookie(name = "token", value = token) - } - ) { - Log.i(TAG, "Listening for incoming websocket messages") - while (true) { - val frame = incoming.receive() - if (frame is Frame.Text) { - val message = frame.readText() - Log.i(TAG, "Received message: $message") - } - } - } - } - } - - fun setServerAddress(url: String) { - serverAddress = url - } - - fun getServerAddress(): String { - return serverAddress - } - - fun getToken(): String { - return token + companion object { + const val TYPE_SIGNALING = "signaling" } } diff --git a/app/src/main/java/com/example/tvcontroller/services/webrtc/RtcPeerConnection.kt b/app/src/main/java/com/example/tvcontroller/services/webrtc/RtcPeerConnection.kt new file mode 100644 index 0000000..ec6dbf5 --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/services/webrtc/RtcPeerConnection.kt @@ -0,0 +1,191 @@ +package com.example.tvcontroller.services.webrtc + +import android.content.Context +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraManager +import android.util.Log +import org.webrtc.AudioTrack +import org.webrtc.Camera2Capturer +import org.webrtc.DataChannel +import org.webrtc.DefaultVideoDecoderFactory +import org.webrtc.EglBase +import org.webrtc.HardwareVideoEncoderFactory +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.MediaStream +import org.webrtc.MediaStreamTrack +import org.webrtc.PeerConnection +import org.webrtc.PeerConnectionFactory +import org.webrtc.PeerConnectionFactory.InitializationOptions +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import org.webrtc.SimulcastVideoEncoderFactory +import org.webrtc.SoftwareVideoEncoderFactory +import org.webrtc.SurfaceTextureHelper +import org.webrtc.VideoSource +import org.webrtc.VideoTrack + +private const val TAG = "RtcPeerConnection" + +class RtcPeerConnection(private val context: Context) { + private val peerConnectionFactory by lazy { initializeFactory() } + private var iceServers = ArrayList() + var eglBaseContext: EglBase.Context = EglBase.create().eglBaseContext + private val cameraManager by lazy { context.getSystemService(Context.CAMERA_SERVICE) as CameraManager } + private val videoCapturer by lazy { createCameraCapturer() } + private val surfaceTextureHelper by lazy { createSurfaceTextureHelper() } + private val videoSource by lazy { createVideoSource() } + private val videoDecoderFactory by lazy { DefaultVideoDecoderFactory(eglBaseContext) } + private val videoEncoderFactory by lazy { + SimulcastVideoEncoderFactory( + HardwareVideoEncoderFactory(eglBaseContext, true, true), + SoftwareVideoEncoderFactory() + ) + } + private var peerConnection: PeerConnection? = null + private val iceCandidateHandlers = ArrayList<((IceCandidate) -> Unit)>() + + + fun initialize() { + var observer = object : PeerConnection.Observer { + override fun onSignalingChange(p0: PeerConnection.SignalingState?) {} + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) {} + override fun onIceConnectionReceivingChange(p0: Boolean) {} + override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) {} + override fun onIceCandidate(iceCandidate: IceCandidate?) { + iceCandidateHandlers.forEach { it(iceCandidate!!) } + } + override fun onIceCandidatesRemoved(p0: Array?) {} + override fun onAddStream(p0: MediaStream?) {} + override fun onRemoveStream(p0: MediaStream?) {} + override fun onDataChannel(p0: DataChannel?) {} + override fun onRenegotiationNeeded() {} + } + var rtcConfig = PeerConnection.RTCConfiguration(iceServers) + peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, observer) + } + + fun setIceServers(iceServerStrings: Array) { + iceServers = ArrayList() + iceServerStrings.forEach { + iceServers.add( + PeerConnection.IceServer.builder(it).createIceServer() + ) + } + } + + fun createAudioTrack(): AudioTrack { + val audioConstraints = MediaConstraints() + val audioSource = peerConnectionFactory.createAudioSource(audioConstraints) + val audioTrack = peerConnectionFactory.createAudioTrack("audio_track", audioSource) + return audioTrack + } + + fun addTack(track: MediaStreamTrack) { + peerConnection?.addTrack(track) + } + + fun createVideoTrack(): VideoTrack { + val videoTrack = peerConnectionFactory.createVideoTrack("video_track", videoSource) + return videoTrack + } + + fun setLocalDescription(sessionDescription: SessionDescription, callback: () -> Unit) { + val onLocalDescriptionSet = object : SdpObserver { + override fun onSetSuccess() { + callback() + } + + override fun onCreateSuccess(p0: SessionDescription?) {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + peerConnection?.setLocalDescription(onLocalDescriptionSet, sessionDescription) + } + + fun setRemoteDescription(sessionDescription: SessionDescription, callback: () -> Unit) { + val onRemoteDescriptionSet = object : SdpObserver { + override fun onSetSuccess() { + callback() + } + + override fun onCreateSuccess(p0: SessionDescription?) {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + peerConnection?.setRemoteDescription(onRemoteDescriptionSet, sessionDescription) + } + + fun createAnswer(mediaConstraints: MediaConstraints, callback: (SessionDescription) -> Unit) { + val onAnswerCreated = object : SdpObserver { + override fun onCreateSuccess(sessionDescription: SessionDescription?) { + callback(sessionDescription!!) + } + + override fun onSetSuccess() {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + peerConnection?.createAnswer(onAnswerCreated, mediaConstraints) + } + + fun addIceCandidate(iceCandidate: IceCandidate) { + Log.i(TAG, "Adding ice candidate $iceCandidate to $peerConnection") + peerConnection?.addIceCandidate(iceCandidate) + } + + fun onIceCandidate(handler: (IceCandidate) -> Unit) { + iceCandidateHandlers.add(handler) + } + + private fun initializeFactory(): PeerConnectionFactory { + val initOptions = InitializationOptions.builder(context).createInitializationOptions() + PeerConnectionFactory.initialize(initOptions) + + val options = PeerConnectionFactory.Options() + val peerConnectionFactory = + PeerConnectionFactory.builder().setVideoDecoderFactory(videoDecoderFactory) + .setVideoEncoderFactory(videoEncoderFactory).setOptions(options) + .createPeerConnectionFactory() + return peerConnectionFactory + } + + private fun createCameraCapturer(): Camera2Capturer { + val ids = cameraManager.cameraIdList + var foundCamera = false; + var cameraId = "" + for (id in ids) { + val characteristics = cameraManager.getCameraCharacteristics(id) + val facing = characteristics.get(CameraCharacteristics.LENS_FACING) + if (facing == CameraCharacteristics.LENS_FACING_BACK) { + cameraId = id + foundCamera = true + break + } + } + if (!foundCamera) { + cameraId = ids.first() + } + val cameraCapturer = Camera2Capturer(context, cameraId, null) + return cameraCapturer + } + + private fun createSurfaceTextureHelper(): SurfaceTextureHelper { + val surfaceTextureHelper = + SurfaceTextureHelper.create("SurfaceTextureHelperThread", eglBaseContext) + return surfaceTextureHelper + } + + private fun createVideoSource(): VideoSource { + val videoSource = peerConnectionFactory.createVideoSource(false) + videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver) + videoCapturer.startCapture(1920, 1080, 30) + return videoSource + } + + companion object { + const val TYPE_OFFER = "offer" + const val TYPE_ANSWER = "answer" + const val TYPE_ICE_CANDIDATE = "ice_candidate" + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/services/webrtc/WebRtcService.kt b/app/src/main/java/com/example/tvcontroller/services/webrtc/WebRtcService.kt new file mode 100644 index 0000000..7365939 --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/services/webrtc/WebRtcService.kt @@ -0,0 +1,123 @@ +package com.example.tvcontroller.services.webrtc + +import android.content.Context +import android.util.Log +import com.example.tvcontroller.client.WebsocketClient +import com.example.tvcontroller.services.DeviceService +import kotlinx.coroutines.runBlocking +import org.json.JSONObject +import org.webrtc.EglBase +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.SessionDescription + +private const val TAG = "WebRtcService" + +class WebRtcService(private val context: Context, private val websocketClient: WebsocketClient) { + private val rtcPeerConnection by lazy { createRtcPeerConnection() } + val videoTrack by lazy { rtcPeerConnection.createVideoTrack() } + val audioTrack by lazy { rtcPeerConnection.createAudioTrack() } + val eglBaseContext: EglBase.Context + get() = rtcPeerConnection.eglBaseContext + private var peerId: String = "" + + fun connect() { + Log.i(TAG, "Connecting to signaling server") + websocketClient.onData(this::handleData) + } + + private fun createRtcPeerConnection(): RtcPeerConnection { + val iceServers = arrayOf("stun:stun.l.google.com:19302") + val webRtcService = this + val rtcPeerConnection = RtcPeerConnection(context).apply { + setIceServers(iceServers) + onIceCandidate(webRtcService::sendIceCandidate) + initialize() + } + return rtcPeerConnection + } + + private fun handleOffer(sdp: String) { + var mediaConstraints = MediaConstraints() + val remoteSessionDescription = SessionDescription(SessionDescription.Type.OFFER, sdp) + rtcPeerConnection.apply { + addTack(audioTrack) + addTack(videoTrack) + setRemoteDescription(remoteSessionDescription) { + createAnswer(mediaConstraints) { localSessionDescription -> + setLocalDescription(localSessionDescription) { + sendAnswer( + peerId, localSessionDescription.description ?: "" + ) + } + } + } + } + } + + private fun handleData(data: String) { + Log.d(TAG, "Received data: $data") + val dataJson = JSONObject(data) + val senderId = dataJson.getString("sender") + val message = dataJson.getJSONObject("message") + val type = message.getString("type") + when (type) { + RtcPeerConnection.TYPE_OFFER -> { + Log.i(TAG, "Received offer from $senderId") + val sdp = message.getString("sdp") + peerId = senderId + handleOffer(sdp) + } + + RtcPeerConnection.TYPE_ICE_CANDIDATE -> { + val candidateString = message.getString("candidate") + handleReceiveIceCandidate(candidateString) + } + } + } + + private fun handleReceiveIceCandidate(candidateString: String) { + Log.i(TAG, "Received ice candidate") + val candidateJson = JSONObject(candidateString) + val sdpMid = candidateJson.getString("sdpMid") + val sdpMLineIndex = candidateJson.getInt("sdpMLineIndex") + val sdp = candidateJson.getString("candidate") + val candidate = IceCandidate(sdpMid, sdpMLineIndex, sdp) + Log.i(TAG, "Candidate: $candidate") + rtcPeerConnection.addIceCandidate(candidate) + } + + private fun sendIceCandidate(iceCandidate: IceCandidate) { + val messageJson = JSONObject() + messageJson.put("type", DeviceService.Companion.TYPE_SIGNALING) + messageJson.put("target", peerId) + messageJson.put("message", JSONObject().apply { + put("candidate", JSONObject().apply { + put("sdpMid", iceCandidate.sdpMid) + put("sdpMLineIndex", iceCandidate.sdpMLineIndex) + put("candidate", iceCandidate.sdp) + }) + put("type", RtcPeerConnection.TYPE_ICE_CANDIDATE) + }) + + runBlocking { + Log.i(TAG, "Sending ice candidate") + websocketClient.sendJson(messageJson) + } + } + + private fun sendAnswer(targetId: String, sdp: String) { + val messageJson = JSONObject() + messageJson.put("type", DeviceService.Companion.TYPE_SIGNALING) + messageJson.put("target", targetId) + messageJson.put("message", JSONObject().apply { + put("sdp", sdp) + put("type", RtcPeerConnection.TYPE_ANSWER) + }) + + runBlocking { + Log.i(TAG, "Sending answer") + websocketClient.sendJson(messageJson) + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt b/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt index e100c9e..5516f0a 100644 --- a/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt +++ b/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt @@ -3,23 +3,62 @@ package com.example.tvcontroller.ui.components import androidx.camera.view.LifecycleCameraController import androidx.camera.view.PreviewView import androidx.compose.runtime.Composable +import androidx.compose.runtime.MutableState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier import androidx.compose.ui.viewinterop.AndroidView -import androidx.lifecycle.compose.LocalLifecycleOwner +import org.webrtc.EglBase +import org.webrtc.RendererCommon +import org.webrtc.VideoTrack @Composable fun CameraPreview( - controller: LifecycleCameraController, - modifier: Modifier = Modifier + eglBaseContext: EglBase.Context, videoTrack: VideoTrack, modifier: Modifier = Modifier ) { - val lifecycleOwner = LocalLifecycleOwner.current + val trackState: MutableState = remember { mutableStateOf(null) } + var view: VideoTextureViewRenderer? by remember { mutableStateOf(null) } + AndroidView( - modifier = modifier, factory = { - PreviewView(it).apply { - this.controller = controller - controller.bindToLifecycle(lifecycleOwner) + VideoTextureViewRenderer(it).apply { + init( + eglBaseContext, + object : RendererCommon.RendererEvents { + override fun onFirstFrameRendered() = Unit + override fun onFrameResolutionChanged(p0: Int, p1: Int, p2: Int) = Unit + } + ) + setupVideo(trackState, videoTrack, this) + view = this } }, + update = { v -> setupVideo(trackState, videoTrack, v) }, + modifier = modifier ) } + +private fun cleanTrack( + view: VideoTextureViewRenderer?, + trackState: MutableState +) { + view?.let { trackState.value?.removeSink(it) } + trackState.value = null +} + +private fun setupVideo( + trackState: MutableState, + track: VideoTrack, + renderer: VideoTextureViewRenderer +) { + if (trackState.value == track) { + return + } + + cleanTrack(renderer, trackState) + + trackState.value = track + track.addSink(renderer) +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/ui/components/VideoTextureViewRenderer.kt b/app/src/main/java/com/example/tvcontroller/ui/components/VideoTextureViewRenderer.kt new file mode 100644 index 0000000..81cedcb --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/ui/components/VideoTextureViewRenderer.kt @@ -0,0 +1,191 @@ +package com.example.tvcontroller.ui.components +/* + * Copyright 2023 Stream.IO, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import android.content.Context +import android.content.res.Resources +import android.graphics.SurfaceTexture +import android.os.Handler +import android.os.Looper +import android.util.AttributeSet +import android.view.TextureView +import android.view.TextureView.SurfaceTextureListener +import org.webrtc.EglBase +import org.webrtc.EglRenderer +import org.webrtc.GlRectDrawer +import org.webrtc.RendererCommon.RendererEvents +import org.webrtc.ThreadUtils +import org.webrtc.VideoFrame +import org.webrtc.VideoSink +import java.util.concurrent.CountDownLatch + +/** + * Custom [TextureView] used to render local/incoming videos on the screen. + */ +open class VideoTextureViewRenderer @JvmOverloads constructor( + context: Context, + attrs: AttributeSet? = null +) : TextureView(context, attrs), VideoSink, SurfaceTextureListener { + + /** + * Cached resource name. + */ + private val resourceName: String = getResourceName() + + /** + * Renderer used to render the video. + */ + private val eglRenderer: EglRenderer = EglRenderer(resourceName) + + /** + * Callback used for reporting render events. + */ + private var rendererEvents: RendererEvents? = null + + /** + * Handler to access the UI thread. + */ + private val uiThreadHandler = Handler(Looper.getMainLooper()) + + /** + * Whether the first frame has been rendered or not. + */ + private var isFirstFrameRendered = false + + /** + * The rotated [VideoFrame] width. + */ + private var rotatedFrameWidth = 0 + + /** + * The rotated [VideoFrame] height. + */ + private var rotatedFrameHeight = 0 + + /** + * The rotated [VideoFrame] rotation. + */ + private var frameRotation = 0 + + init { + surfaceTextureListener = this + } + + /** + * Called when a new frame is received. Sends the frame to be rendered. + * + * @param videoFrame The [VideoFrame] received from WebRTC connection to draw on the screen. + */ + override fun onFrame(videoFrame: VideoFrame) { + eglRenderer.onFrame(videoFrame) + updateFrameData(videoFrame) + } + + /** + * Updates the frame data and notifies [rendererEvents] about the changes. + */ + private fun updateFrameData(videoFrame: VideoFrame) { + if (isFirstFrameRendered) { + rendererEvents?.onFirstFrameRendered() + isFirstFrameRendered = true + } + + if (videoFrame.rotatedWidth != rotatedFrameWidth || + videoFrame.rotatedHeight != rotatedFrameHeight || + videoFrame.rotation != frameRotation + ) { + rotatedFrameWidth = videoFrame.rotatedWidth + rotatedFrameHeight = videoFrame.rotatedHeight + frameRotation = videoFrame.rotation + + uiThreadHandler.post { + rendererEvents?.onFrameResolutionChanged( + rotatedFrameWidth, + rotatedFrameHeight, + frameRotation + ) + } + } + } + + /** + * After the view is laid out we need to set the correct layout aspect ratio to the renderer so that the image + * is scaled correctly. + */ + override fun onLayout(changed: Boolean, left: Int, top: Int, right: Int, bottom: Int) { + eglRenderer.setLayoutAspectRatio((right - left) / (bottom.toFloat() - top)) + } + + /** + * Initialise the renderer. Should be called from the main thread. + * + * @param sharedContext [EglBase.Context] + * @param rendererEvents Sets the render event listener. + */ + fun init( + sharedContext: EglBase.Context, + rendererEvents: RendererEvents + ) { + ThreadUtils.checkIsOnMainThread() + this.rendererEvents = rendererEvents + eglRenderer.init(sharedContext, EglBase.CONFIG_PLAIN, GlRectDrawer()) + } + + /** + * [SurfaceTextureListener] callback that lets us know when a surface texture is ready and we can draw on it. + */ + override fun onSurfaceTextureAvailable( + surfaceTexture: SurfaceTexture, + width: Int, + height: Int + ) { + eglRenderer.createEglSurface(surfaceTexture) + } + + /** + * [SurfaceTextureListener] callback that lets us know when a surface texture is destroyed we need to stop drawing + * on it. + */ + override fun onSurfaceTextureDestroyed(surfaceTexture: SurfaceTexture): Boolean { + val completionLatch = CountDownLatch(1) + eglRenderer.releaseEglSurface { completionLatch.countDown() } + ThreadUtils.awaitUninterruptibly(completionLatch) + return true + } + + override fun onSurfaceTextureSizeChanged( + surfaceTexture: SurfaceTexture, + width: Int, + height: Int + ) { + } + + override fun onSurfaceTextureUpdated(surfaceTexture: SurfaceTexture) {} + + override fun onDetachedFromWindow() { + eglRenderer.release() + super.onDetachedFromWindow() + } + + private fun getResourceName(): String { + return try { + resources.getResourceEntryName(id) + ": " + } catch (e: Resources.NotFoundException) { + "" + } + } +} diff --git a/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt b/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt index 02ab23f..940e886 100644 --- a/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt +++ b/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt @@ -1,34 +1,22 @@ package com.example.tvcontroller.ui.views -import androidx.camera.view.CameraController -import androidx.camera.view.LifecycleCameraController -import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding -import androidx.compose.material3.Text import androidx.compose.runtime.Composable -import androidx.compose.runtime.remember -import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.unit.dp import com.example.tvcontroller.ui.components.CameraPreview +import org.webrtc.EglBase +import org.webrtc.VideoTrack @Composable -fun CameraView() { - val context = LocalContext.current - val controller = remember { - LifecycleCameraController(context).apply { - setEnabledUseCases(CameraController.VIDEO_CAPTURE) - } - } +fun CameraView(eglBaseContext: EglBase.Context, videoTrack: VideoTrack) { Box( modifier = Modifier .fillMaxSize() .padding(all = 16.dp), ) { - CameraPreview(controller = controller, modifier = Modifier.fillMaxSize()) + CameraPreview(eglBaseContext = eglBaseContext, videoTrack = videoTrack, modifier = Modifier.fillMaxSize()) } } diff --git a/app/src/main/java/com/example/tvcontroller/ui/views/SettingsViewModel.kt b/app/src/main/java/com/example/tvcontroller/ui/views/SettingsViewModel.kt index cf62376..d066b30 100644 --- a/app/src/main/java/com/example/tvcontroller/ui/views/SettingsViewModel.kt +++ b/app/src/main/java/com/example/tvcontroller/ui/views/SettingsViewModel.kt @@ -15,11 +15,13 @@ import com.example.tvcontroller.services.DeviceService import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch +private const val TAG = "SettingsViewModel" + class SettingsViewModel( private val deviceService: DeviceService, private val bluetoothService: BluetoothService ) : ViewModel() { - var serverAddress by mutableStateOf(deviceService.getServerAddress()) + var serverAddress by mutableStateOf(deviceService.serverAddress) private set var deviceName by mutableStateOf(Build.MANUFACTURER + " " + Build.MODEL) private set @@ -48,17 +50,16 @@ class SettingsViewModel( fun connect() { //Log.i("SettingsScreen", "Save settings: $serverUrl, $deviceName, $registrationCode") viewModelScope.launch { - deviceService.setServerAddress(serverAddress) + deviceService.serverAddress = serverAddress deviceService.registerIntegration(deviceName, registrationCode) updateConnectionState() updateDeviceInfo() - deviceService.connect() } } private fun updateConnectionState() { - Log.i("SettingsViewModel", "Device token: ${deviceService.getToken()}") - connectionState = if (deviceService.getToken().isEmpty()) { + Log.i(TAG, "Device token: ${deviceService.token}") + connectionState = if (deviceService.token.isEmpty()) { Settings.ConnectionState.Unregistered } else { Settings.ConnectionState.Registered diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5b3339d..240bdfa 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,6 +12,7 @@ activityCompose = "1.10.1" composeBom = "2025.03.01" material3 = "1.4.0-alpha11" navigationCompose = "2.8.9" +streamWebrtcAndroid = "1.3.8" [libraries] androidx-camera-camera2 = { module = "androidx.camera:camera-camera2", version.ref = "cameraCore" } @@ -40,6 +41,7 @@ ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" } ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-websockets = { module = "io.ktor:ktor-client-websockets", version.ref = "ktor" } material3 = { module = "androidx.compose.material3:material3", version.ref = "material3" } +stream-webrtc-android = { module = "io.getstream:stream-webrtc-android", version.ref = "streamWebrtcAndroid" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" }