From 8bf72a23ea94eea9be10e9328c67853b7f4c423b Mon Sep 17 00:00:00 2001 From: Fritz Heiden Date: Thu, 3 Apr 2025 19:28:56 +0200 Subject: [PATCH] feat: add peer connection to transfer video and audio --- app/src/main/AndroidManifest.xml | 1 + .../com/example/tvcontroller/MainActivity.kt | 11 +- .../tvcontroller/services/DeviceService.kt | 127 ++++++++--- .../ui/components/CameraPreview.kt | 53 ++++- .../ui/components/VideoTextureViewRenderer.kt | 191 ++++++++++++++++ .../tvcontroller/ui/views/CameraView.kt | 12 +- .../tvcontroller/webrtc/CameraXCapturer.kt | 12 +- .../tvcontroller/webrtc/RtcPeerConnection.kt | 204 +++++++++++++++--- 8 files changed, 530 insertions(+), 81 deletions(-) create mode 100644 app/src/main/java/com/example/tvcontroller/ui/components/VideoTextureViewRenderer.kt diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 57a7af3..decedb2 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -12,6 +12,7 @@ + diff --git a/app/src/main/java/com/example/tvcontroller/MainActivity.kt b/app/src/main/java/com/example/tvcontroller/MainActivity.kt index d22e4df..267f56f 100644 --- a/app/src/main/java/com/example/tvcontroller/MainActivity.kt +++ b/app/src/main/java/com/example/tvcontroller/MainActivity.kt @@ -55,20 +55,17 @@ class MainActivity : ComponentActivity() { val cameraController = LifecycleCameraController(applicationContext).apply { setEnabledUseCases(CameraController.IMAGE_ANALYSIS) - setImageAnalysisAnalyzer( - ContextCompat.getMainExecutor(applicationContext), - rtcPeerConnection.cameraXCapturer - ) bindToLifecycle(lifecycleOwner) - } bluetoothService = BluetoothService(applicationContext) deviceService = DeviceService(applicationContext) + deviceService.rtcPeerConnection = rtcPeerConnection + rtcPeerConnection.deviceService = deviceService cameraService = CameraService(applicationContext) controllerService = ControllerService(applicationContext, bluetoothService) checkPermissions() - lifecycleScope.launch(Dispatchers.IO) { + lifecycleScope.launch { deviceService.initialize() } enableEdgeToEdge() @@ -154,7 +151,7 @@ fun TvControllerApp( modifier = Modifier.padding(innerPadding) ) { composable(route = Screen.Camera.name) { - CameraView(cameraController) + CameraView(eglBaseContext = rtcPeerConnection.eglBaseContext, videoTrack = rtcPeerConnection.videoTrack) } composable(route = Screen.Remote.name) { RemoteView( diff --git a/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt b/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt index 9399bc5..c710a1b 100644 --- a/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt +++ b/app/src/main/java/com/example/tvcontroller/services/DeviceService.kt @@ -4,24 +4,24 @@ import android.content.Context import android.content.Context.MODE_PRIVATE import android.util.Log import com.example.tvcontroller.data.Integration -import io.ktor.client.engine.cio.* -import io.ktor.client.* +import com.example.tvcontroller.webrtc.RtcPeerConnection +import io.ktor.client.HttpClient import io.ktor.client.call.body -import io.ktor.client.plugins.cookies.HttpCookies +import io.ktor.client.engine.cio.CIO import io.ktor.client.plugins.websocket.WebSockets import io.ktor.client.plugins.websocket.webSocket -import io.ktor.client.request.HttpRequestBuilder import io.ktor.client.request.cookie import io.ktor.client.request.headers import io.ktor.client.request.request import io.ktor.client.request.setBody import io.ktor.client.statement.HttpResponse -import io.ktor.http.Cookie import io.ktor.http.HttpMethod +import io.ktor.websocket.DefaultWebSocketSession import io.ktor.websocket.Frame import io.ktor.websocket.readText import kotlinx.coroutines.runBlocking import org.json.JSONObject +import org.webrtc.IceCandidate private const val SHARED_PREFERENCES_NAME = "devices"; private const val TAG = "DeviceService" @@ -30,9 +30,11 @@ class DeviceService(private val context: Context) { private var client = HttpClient(CIO) { install(WebSockets) } + private var websocket: DefaultWebSocketSession? = null private var serverAddress: String = "" private var token: String = "" private var deviceId: String = "" + var rtcPeerConnection: RtcPeerConnection? = null suspend fun initialize() { loadPreferences() @@ -126,32 +128,105 @@ class DeviceService(private val context: Context) { } } - fun connect() { + suspend fun connect() { Log.i(TAG, "Connecting to websocket at $serverAddress") - runBlocking { - val (host, port) = serverAddress.split(":") - val portInt = if (port.isEmpty()) 80 else port.toInt() - client.webSocket( - method = HttpMethod.Get, - host = host, - port = portInt, - path = "/ws", - request = { - cookie(name = "token", value = token) - } - ) { - Log.i(TAG, "Listening for incoming websocket messages") - while (true) { - val frame = incoming.receive() - if (frame is Frame.Text) { - val message = frame.readText() - Log.i(TAG, "Received message: $message") + val (host, port) = serverAddress.split(":") + val portInt = if (port.isEmpty()) 80 else port.toInt() + client.webSocket( + method = HttpMethod.Get, + host = host, + port = portInt, + path = "/ws", + request = { + cookie(name = "token", value = token) + } + ) { + Log.i(TAG, "Listening for incoming websocket messages") + websocket = this + while (true) { + val frame = incoming.receive() + if (frame is Frame.Text) { + val dataString = frame.readText() + val dataJson = JSONObject(dataString) + val senderId = dataJson.getString("sender") + val message = dataJson.getJSONObject("message") + val type = message.getString("type") + when (type) { + RtcPeerConnection.TYPE_OFFER -> { + Log.i(TAG, "Received offer from $senderId") + val sdp = message.getString("sdp") + rtcPeerConnection?.connect(senderId, sdp) + } + + RtcPeerConnection.TYPE_ICE_CANDIDATE -> { + Log.i(TAG, "Received ice candidate") + val candidateString = message.getString("candidate") + val candidateJson = JSONObject(candidateString) + val sdpMid = candidateJson.getString("sdpMid") + val sdpMLineIndex = candidateJson.getInt("sdpMLineIndex") + val sdp = candidateJson.getString("candidate") + val candidate = IceCandidate(sdpMid, sdpMLineIndex, sdp) + Log.i(TAG, "Candidate: $candidate") + rtcPeerConnection?.addIceCandidate(candidate) + } } } } } } + fun sendWebRtcAnswer(targetId: String, sdp: String) { + val messageJson = JSONObject() + messageJson.put("type", TYPE_SIGNALING) + messageJson.put("target", targetId) + messageJson.put("message", JSONObject().apply { + put("sdp", sdp) + put("type", RtcPeerConnection.TYPE_ANSWER) + }) + + runBlocking { + Log.i(TAG, "Sending answer") + val frame = Frame.Text(messageJson.toString()) + websocket?.send(frame) + } + } + + fun sendWebRtcOffer(targetId: String, sdp: String) { + val messageJson = JSONObject() + messageJson.put("type", TYPE_SIGNALING) + messageJson.put("target", targetId) + messageJson.put("message", JSONObject().apply { + put("sdp", sdp) + put("type", RtcPeerConnection.TYPE_OFFER) + }) + + runBlocking { + Log.i(TAG, "Sending offer") + val frame = Frame.Text(messageJson.toString()) + websocket?.send(frame) + } + } + + fun sendWebRtcIceCandidate(targetId: String, candidate: IceCandidate) { + val messageJson = JSONObject() + messageJson.put("type", TYPE_SIGNALING) + messageJson.put("target", targetId) + messageJson.put("message", JSONObject().apply { + put("candidate", JSONObject().apply { + put("sdpMid", candidate.sdpMid) + put("sdpMLineIndex", candidate.sdpMLineIndex) + put("candidate", candidate.sdp) + }) + put("type", RtcPeerConnection.TYPE_ICE_CANDIDATE) + }) + + runBlocking { + Log.i(TAG, "Sending ice candidate") + val frame = Frame.Text(messageJson.toString()) + websocket?.send(frame) + } + } + fun setServerAddress(url: String) { serverAddress = url } @@ -163,4 +238,8 @@ class DeviceService(private val context: Context) { fun getToken(): String { return token } + + companion object { + const val TYPE_SIGNALING = "signaling" + } } diff --git a/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt b/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt index de8fc6d..5516f0a 100644 --- a/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt +++ b/app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt @@ -3,21 +3,62 @@ package com.example.tvcontroller.ui.components import androidx.camera.view.LifecycleCameraController import androidx.camera.view.PreviewView import androidx.compose.runtime.Composable +import androidx.compose.runtime.MutableState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier import androidx.compose.ui.viewinterop.AndroidView -import androidx.lifecycle.compose.LocalLifecycleOwner +import org.webrtc.EglBase +import org.webrtc.RendererCommon +import org.webrtc.VideoTrack @Composable fun CameraPreview( - controller: LifecycleCameraController, - modifier: Modifier = Modifier + eglBaseContext: EglBase.Context, videoTrack: VideoTrack, modifier: Modifier = Modifier ) { + val trackState: MutableState = remember { mutableStateOf(null) } + var view: VideoTextureViewRenderer? by remember { mutableStateOf(null) } + AndroidView( - modifier = modifier, factory = { - PreviewView(it).apply { - this.controller = controller + VideoTextureViewRenderer(it).apply { + init( + eglBaseContext, + object : RendererCommon.RendererEvents { + override fun onFirstFrameRendered() = Unit + override fun onFrameResolutionChanged(p0: Int, p1: Int, p2: Int) = Unit + } + ) + setupVideo(trackState, videoTrack, this) + view = this } }, + update = { v -> setupVideo(trackState, videoTrack, v) }, + modifier = modifier ) } + +private fun cleanTrack( + view: VideoTextureViewRenderer?, + trackState: MutableState +) { + view?.let { trackState.value?.removeSink(it) } + trackState.value = null +} + +private fun setupVideo( + trackState: MutableState, + track: VideoTrack, + renderer: VideoTextureViewRenderer +) { + if (trackState.value == track) { + return + } + + cleanTrack(renderer, trackState) + + trackState.value = track + track.addSink(renderer) +} \ No newline at end of file diff --git a/app/src/main/java/com/example/tvcontroller/ui/components/VideoTextureViewRenderer.kt b/app/src/main/java/com/example/tvcontroller/ui/components/VideoTextureViewRenderer.kt new file mode 100644 index 0000000..81cedcb --- /dev/null +++ b/app/src/main/java/com/example/tvcontroller/ui/components/VideoTextureViewRenderer.kt @@ -0,0 +1,191 @@ +package com.example.tvcontroller.ui.components +/* + * Copyright 2023 Stream.IO, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import android.content.Context +import android.content.res.Resources +import android.graphics.SurfaceTexture +import android.os.Handler +import android.os.Looper +import android.util.AttributeSet +import android.view.TextureView +import android.view.TextureView.SurfaceTextureListener +import org.webrtc.EglBase +import org.webrtc.EglRenderer +import org.webrtc.GlRectDrawer +import org.webrtc.RendererCommon.RendererEvents +import org.webrtc.ThreadUtils +import org.webrtc.VideoFrame +import org.webrtc.VideoSink +import java.util.concurrent.CountDownLatch + +/** + * Custom [TextureView] used to render local/incoming videos on the screen. + */ +open class VideoTextureViewRenderer @JvmOverloads constructor( + context: Context, + attrs: AttributeSet? = null +) : TextureView(context, attrs), VideoSink, SurfaceTextureListener { + + /** + * Cached resource name. + */ + private val resourceName: String = getResourceName() + + /** + * Renderer used to render the video. + */ + private val eglRenderer: EglRenderer = EglRenderer(resourceName) + + /** + * Callback used for reporting render events. + */ + private var rendererEvents: RendererEvents? = null + + /** + * Handler to access the UI thread. + */ + private val uiThreadHandler = Handler(Looper.getMainLooper()) + + /** + * Whether the first frame has been rendered or not. + */ + private var isFirstFrameRendered = false + + /** + * The rotated [VideoFrame] width. + */ + private var rotatedFrameWidth = 0 + + /** + * The rotated [VideoFrame] height. + */ + private var rotatedFrameHeight = 0 + + /** + * The rotated [VideoFrame] rotation. + */ + private var frameRotation = 0 + + init { + surfaceTextureListener = this + } + + /** + * Called when a new frame is received. Sends the frame to be rendered. + * + * @param videoFrame The [VideoFrame] received from WebRTC connection to draw on the screen. + */ + override fun onFrame(videoFrame: VideoFrame) { + eglRenderer.onFrame(videoFrame) + updateFrameData(videoFrame) + } + + /** + * Updates the frame data and notifies [rendererEvents] about the changes. + */ + private fun updateFrameData(videoFrame: VideoFrame) { + if (isFirstFrameRendered) { + rendererEvents?.onFirstFrameRendered() + isFirstFrameRendered = true + } + + if (videoFrame.rotatedWidth != rotatedFrameWidth || + videoFrame.rotatedHeight != rotatedFrameHeight || + videoFrame.rotation != frameRotation + ) { + rotatedFrameWidth = videoFrame.rotatedWidth + rotatedFrameHeight = videoFrame.rotatedHeight + frameRotation = videoFrame.rotation + + uiThreadHandler.post { + rendererEvents?.onFrameResolutionChanged( + rotatedFrameWidth, + rotatedFrameHeight, + frameRotation + ) + } + } + } + + /** + * After the view is laid out we need to set the correct layout aspect ratio to the renderer so that the image + * is scaled correctly. + */ + override fun onLayout(changed: Boolean, left: Int, top: Int, right: Int, bottom: Int) { + eglRenderer.setLayoutAspectRatio((right - left) / (bottom.toFloat() - top)) + } + + /** + * Initialise the renderer. Should be called from the main thread. + * + * @param sharedContext [EglBase.Context] + * @param rendererEvents Sets the render event listener. + */ + fun init( + sharedContext: EglBase.Context, + rendererEvents: RendererEvents + ) { + ThreadUtils.checkIsOnMainThread() + this.rendererEvents = rendererEvents + eglRenderer.init(sharedContext, EglBase.CONFIG_PLAIN, GlRectDrawer()) + } + + /** + * [SurfaceTextureListener] callback that lets us know when a surface texture is ready and we can draw on it. + */ + override fun onSurfaceTextureAvailable( + surfaceTexture: SurfaceTexture, + width: Int, + height: Int + ) { + eglRenderer.createEglSurface(surfaceTexture) + } + + /** + * [SurfaceTextureListener] callback that lets us know when a surface texture is destroyed we need to stop drawing + * on it. + */ + override fun onSurfaceTextureDestroyed(surfaceTexture: SurfaceTexture): Boolean { + val completionLatch = CountDownLatch(1) + eglRenderer.releaseEglSurface { completionLatch.countDown() } + ThreadUtils.awaitUninterruptibly(completionLatch) + return true + } + + override fun onSurfaceTextureSizeChanged( + surfaceTexture: SurfaceTexture, + width: Int, + height: Int + ) { + } + + override fun onSurfaceTextureUpdated(surfaceTexture: SurfaceTexture) {} + + override fun onDetachedFromWindow() { + eglRenderer.release() + super.onDetachedFromWindow() + } + + private fun getResourceName(): String { + return try { + resources.getResourceEntryName(id) + ": " + } catch (e: Resources.NotFoundException) { + "" + } + } +} diff --git a/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt b/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt index 71bd97b..940e886 100644 --- a/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt +++ b/app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt @@ -1,26 +1,22 @@ package com.example.tvcontroller.ui.views -import androidx.camera.core.ImageAnalysis -import androidx.camera.view.CameraController -import androidx.camera.view.LifecycleCameraController import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding import androidx.compose.runtime.Composable -import androidx.compose.runtime.remember import androidx.compose.ui.Modifier -import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.unit.dp -import androidx.core.content.ContextCompat import com.example.tvcontroller.ui.components.CameraPreview +import org.webrtc.EglBase +import org.webrtc.VideoTrack @Composable -fun CameraView(cameraController: LifecycleCameraController) { +fun CameraView(eglBaseContext: EglBase.Context, videoTrack: VideoTrack) { Box( modifier = Modifier .fillMaxSize() .padding(all = 16.dp), ) { - CameraPreview(controller = cameraController, modifier = Modifier.fillMaxSize()) + CameraPreview(eglBaseContext = eglBaseContext, videoTrack = videoTrack, modifier = Modifier.fillMaxSize()) } } diff --git a/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt b/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt index 6fd9cef..96f7b65 100644 --- a/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt +++ b/app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt @@ -8,18 +8,18 @@ import org.webrtc.CapturerObserver import org.webrtc.VideoFrame import java.nio.ByteBuffer -class CameraXCapturer : ImageAnalysis.Analyzer { - private var capturerObserver: CapturerObserver? = null +private const val TAG = "CameraXCapturer" - fun setCapturerObserver(capturerObserver: CapturerObserver) { - this.capturerObserver = capturerObserver - } +class CameraXCapturer : ImageAnalysis.Analyzer { + var capturerObserver: CapturerObserver? = null override fun analyze(image: ImageProxy) { if (image.format != YUV_420_888) throw Exception("Unsupported format") - Log.i("CameraXCapturer", "Received image from CameraX") var videoFrame = imageProxyToVideoFrame(image) + Log.i(TAG, "Handing frame to capturer observer $capturerObserver") capturerObserver?.onFrameCaptured(videoFrame) ?: image.close() + Log.i(TAG, "Frame handled by capturer observer") + image.close() } fun imageProxyToVideoFrame(image: ImageProxy): VideoFrame { diff --git a/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt b/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt index fb2194b..81096a8 100644 --- a/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt +++ b/app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt @@ -1,30 +1,60 @@ package com.example.tvcontroller.webrtc import android.content.Context +import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraManager import android.util.Log +import androidx.compose.ui.Modifier +import com.example.tvcontroller.services.DeviceService import org.webrtc.AudioTrack import org.webrtc.Camera2Capturer import org.webrtc.DataChannel +import org.webrtc.DefaultVideoDecoderFactory +import org.webrtc.DefaultVideoEncoderFactory +import org.webrtc.EglBase +import org.webrtc.HardwareVideoEncoderFactory import org.webrtc.IceCandidate import org.webrtc.MediaConstraints import org.webrtc.MediaStream import org.webrtc.PeerConnection import org.webrtc.PeerConnectionFactory import org.webrtc.PeerConnectionFactory.InitializationOptions +import org.webrtc.RtpSender +import org.webrtc.RtpTransceiver import org.webrtc.SdpObserver import org.webrtc.SessionDescription +import org.webrtc.SimulcastVideoEncoderFactory +import org.webrtc.SoftwareVideoEncoderFactory +import org.webrtc.SurfaceTextureHelper +import org.webrtc.VideoSource import org.webrtc.VideoTrack +import kotlin.getValue -const val TAG = "RtcPeerConnection" +private const val TAG = "RtcPeerConnection" class RtcPeerConnection(private val context: Context) : PeerConnection.Observer { private val peerConnectionFactory by lazy { initializeFactory() } private val iceServers by lazy { initializeIceServers() } - val cameraXCapturer by lazy { CameraXCapturer() } private val audioTrack by lazy { createAudioTrack() } - private val videoTrack by lazy { createVideoTrack() } + val videoTrack by lazy { createVideoTrack() } + var eglBaseContext: EglBase.Context = EglBase.create().eglBaseContext + private val cameraManager by lazy { context.getSystemService(Context.CAMERA_SERVICE) as CameraManager } + private val videoCapturer by lazy { createCameraCapturer() } + private val surfaceTextureHelper by lazy { createSurfaceTextureHelper() } + private val videoSource by lazy { createVideoSource() } + private val videoDecoderFactory by lazy { DefaultVideoDecoderFactory(eglBaseContext) } + private val videoEncoderFactory by lazy { + SimulcastVideoEncoderFactory( + HardwareVideoEncoderFactory(eglBaseContext, true, true), + SoftwareVideoEncoderFactory() + ) + } private var peerConnection: PeerConnection? = null + var deviceService: DeviceService? = null + + private var peerId: String = "" + + private var offer = "" private fun initializeFactory(): PeerConnectionFactory { val initOptions = InitializationOptions.builder(context).createInitializationOptions() @@ -32,7 +62,9 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer val options = PeerConnectionFactory.Options() val peerConnectionFactory = - PeerConnectionFactory.builder().setOptions(options).createPeerConnectionFactory() + PeerConnectionFactory.builder().setVideoDecoderFactory(videoDecoderFactory) + .setVideoEncoderFactory(videoEncoderFactory).setOptions(options) + .createPeerConnectionFactory() return peerConnectionFactory } @@ -52,48 +84,143 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer } private fun createVideoTrack(): VideoTrack { - val videoSource = peerConnectionFactory.createVideoSource(false) - cameraXCapturer.setCapturerObserver(videoSource.capturerObserver) val localVideoTrack = peerConnectionFactory.createVideoTrack("video_track", videoSource) - localVideoTrack.setEnabled(true) + //localVideoTrack.setEnabled(true) return localVideoTrack } - fun connect() { + private fun createCameraCapturer(): Camera2Capturer { + val ids = cameraManager.cameraIdList + var foundCamera = false; + var cameraId = "" + for (id in ids) { + val characteristics = cameraManager.getCameraCharacteristics(id) + val facing = characteristics.get(CameraCharacteristics.LENS_FACING) + if (facing == CameraCharacteristics.LENS_FACING_BACK) { + cameraId = id + foundCamera = true + break + } + } + if (!foundCamera) { + cameraId = ids.first() + } + val cameraCapturer = Camera2Capturer(context, cameraId, null) + return cameraCapturer + } + + private fun createSurfaceTextureHelper(): SurfaceTextureHelper { + val surfaceTextureHelper = + SurfaceTextureHelper.create("SurfaceTextureHelperThread", eglBaseContext) + return surfaceTextureHelper + } + + private fun createVideoSource(): VideoSource { + val videoSource = peerConnectionFactory.createVideoSource(false) + videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver) + videoCapturer.startCapture(1920, 1080, 30) + return videoSource + } + + fun connect(targetId: String, sdp: String) { + peerId = targetId + offer = sdp Log.i(TAG, "Connecting rtc peer connection") var rtcConfig = PeerConnection.RTCConfiguration(iceServers) - var peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this) - - var mediaConstraints = MediaConstraints() - mediaConstraints.mandatory.add( - MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true") - ); - mediaConstraints.mandatory.add( - MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true") - ); + Log.i(TAG, "Creating peer connection") + peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this) + Log.i(TAG, "Peer connection created") + //peerConnection?.addTransceiver(audioTrack, RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.SEND_ONLY)) + //peerConnection?.addTransceiver( videoTrack, RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.SEND_ONLY) ) peerConnection?.addTrack(audioTrack) peerConnection?.addTrack(videoTrack) - peerConnection?.createOffer(object : SdpObserver { - override fun onCreateSuccess(sessionDescription: SessionDescription?) { - Log.i(TAG, "onCreateSuccess: ${sessionDescription?.description}") - } + handleOffer(sdp) + } + fun handleOffer(sdp: String) { + var mediaConstraints = MediaConstraints() + var localSessionDescription: SessionDescription? = null + val remoteSessionDescription = SessionDescription(SessionDescription.Type.OFFER, sdp) + + Log.i(TAG, "Handling offer $sdp") + val onLocalDescriptionSet = object : SdpObserver { override fun onSetSuccess() { - Log.i(TAG, "onSetSuccess") - TODO("Not yet implemented") + Log.i(TAG, "Local description set") + peerConnection?.transceivers?.forEach { + + Log.i(TAG, "${it.mediaType} Transceiver: ${it.currentDirection}") + } + deviceService?.sendWebRtcAnswer( + peerId, localSessionDescription?.description ?: "" + ) } - override fun onCreateFailure(p0: String?) { - Log.i(TAG, "onCreateFailure: $p0") - TODO("Not yet implemented") + override fun onCreateSuccess(sessionDescription: SessionDescription?) {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + + val onAnswerCreated = object : SdpObserver { + override fun onCreateSuccess(sessionDescription: SessionDescription?) { + Log.i(TAG, "Answer created") + localSessionDescription = sessionDescription + peerConnection?.setLocalDescription(onLocalDescriptionSet, sessionDescription) } - override fun onSetFailure(p0: String?) { - Log.i(TAG, "onSetFailure: $p0") - TODO("Not yet implemented") + override fun onSetSuccess() {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + val onRemoteDescriptionSet = object : SdpObserver { + override fun onSetSuccess() { + Log.i(TAG, "Remote description set") + peerConnection?.createAnswer(onAnswerCreated, mediaConstraints) } - }, mediaConstraints) + override fun onCreateSuccess(p0: SessionDescription?) {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + Log.i(TAG, "Setting remote description") + peerConnection?.setRemoteDescription(onRemoteDescriptionSet, remoteSessionDescription) + } + + fun createOffer() { + var mediaConstraints = MediaConstraints() + var localSessionDescription: SessionDescription? = null + + val onLocalDescriptionSet = object : SdpObserver { + override fun onSetSuccess() { + Log.i(TAG, "Local description set") + peerConnection?.transceivers?.forEach { + Log.i(TAG, "${it.mediaType} Transceiver: ${it.currentDirection}") + } + deviceService?.sendWebRtcOffer( + peerId, localSessionDescription?.description ?: "" + ) + } + + override fun onCreateSuccess(sessionDescription: SessionDescription?) {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + val onOfferCreated = object : SdpObserver { + override fun onCreateSuccess(sessionDescription: SessionDescription?) { + Log.i(TAG, "Offer created") + peerConnection?.setLocalDescription(onLocalDescriptionSet, sessionDescription) + } + + override fun onSetSuccess() {} + override fun onCreateFailure(p0: String?) {} + override fun onSetFailure(p0: String?) {} + } + Log.i(TAG, "Creating offer") + peerConnection?.createOffer(onOfferCreated, mediaConstraints) + } + + fun addIceCandidate(iceCandidate: IceCandidate) { + Log.i(TAG, "Adding ice candidate $iceCandidate to $peerConnection") + peerConnection?.addIceCandidate(iceCandidate) } override fun onSignalingChange(p0: PeerConnection.SignalingState?) { @@ -102,6 +229,9 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { Log.i(TAG, "onIceConnectionChange: $p0") + if (p0 == PeerConnection.IceConnectionState.CONNECTED) { + videoCapturer.startCapture(1280, 720, 30) + } } override fun onIceConnectionReceivingChange(p0: Boolean) { @@ -114,6 +244,9 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer override fun onIceCandidate(p0: IceCandidate?) { Log.i(TAG, "onIceCandidate: $p0") + p0?.let { + deviceService?.sendWebRtcIceCandidate(peerId, it) + } } override fun onIceCandidatesRemoved(p0: Array?) { @@ -134,5 +267,16 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer override fun onRenegotiationNeeded() { Log.i(TAG, "onRenegotiationNeeded") + //if (offer.isNotEmpty()) { + // handleOffer(offer) + //} else { + // createOffer() + //} + } + + companion object { + const val TYPE_OFFER = "offer" + const val TYPE_ANSWER = "answer" + const val TYPE_ICE_CANDIDATE = "ice_candidate" } } \ No newline at end of file