feat: add peer connection to transfer video and audio

app/src/main/AndroidManifest.xml

@@ -12,6 +12,7 @@
     <uses-permission android:name="android.permission.BLUETOOTH_SCAN" />
     <uses-permission android:name="android.permission.BLUETOOTH_CONNECT" />
     <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
     <uses-permission android:name="android.permission.CAMERA" />
     <uses-permission android:name="android.permission.RECORD_AUDIO" />
 

app/src/main/java/com/example/tvcontroller/MainActivity.kt

@@ -55,20 +55,17 @@ class MainActivity : ComponentActivity() {
         val cameraController =
             LifecycleCameraController(applicationContext).apply {
                 setEnabledUseCases(CameraController.IMAGE_ANALYSIS)
-                setImageAnalysisAnalyzer(
-                    ContextCompat.getMainExecutor(applicationContext),
-                    rtcPeerConnection.cameraXCapturer
-                )
                 bindToLifecycle(lifecycleOwner)
-
             }
         bluetoothService = BluetoothService(applicationContext)
         deviceService = DeviceService(applicationContext)
+        deviceService.rtcPeerConnection = rtcPeerConnection
+        rtcPeerConnection.deviceService = deviceService
         cameraService = CameraService(applicationContext)
         controllerService = ControllerService(applicationContext, bluetoothService)
         checkPermissions()
 
-        lifecycleScope.launch(Dispatchers.IO) {
+        lifecycleScope.launch {
            deviceService.initialize()
         }
         enableEdgeToEdge()
@@ -154,7 +151,7 @@ fun TvControllerApp(
             modifier = Modifier.padding(innerPadding)
         ) {
             composable(route = Screen.Camera.name) {
-                CameraView(cameraController)
+                CameraView(eglBaseContext = rtcPeerConnection.eglBaseContext, videoTrack = rtcPeerConnection.videoTrack)
             }
             composable(route = Screen.Remote.name) {
                 RemoteView(

app/src/main/java/com/example/tvcontroller/services/DeviceService.kt

@@ -4,24 +4,24 @@ import android.content.Context
 import android.content.Context.MODE_PRIVATE
 import android.util.Log
 import com.example.tvcontroller.data.Integration
-import io.ktor.client.engine.cio.*
+import com.example.tvcontroller.webrtc.RtcPeerConnection
-import io.ktor.client.*
+import io.ktor.client.HttpClient
 import io.ktor.client.call.body
-import io.ktor.client.plugins.cookies.HttpCookies
+import io.ktor.client.engine.cio.CIO
 import io.ktor.client.plugins.websocket.WebSockets
 import io.ktor.client.plugins.websocket.webSocket
-import io.ktor.client.request.HttpRequestBuilder
 import io.ktor.client.request.cookie
 import io.ktor.client.request.headers
 import io.ktor.client.request.request
 import io.ktor.client.request.setBody
 import io.ktor.client.statement.HttpResponse
-import io.ktor.http.Cookie
 import io.ktor.http.HttpMethod
+import io.ktor.websocket.DefaultWebSocketSession
 import io.ktor.websocket.Frame
 import io.ktor.websocket.readText
 import kotlinx.coroutines.runBlocking
 import org.json.JSONObject
+import org.webrtc.IceCandidate
 
 private const val SHARED_PREFERENCES_NAME = "devices";
 private const val TAG = "DeviceService"
@@ -30,9 +30,11 @@ class DeviceService(private val context: Context) {
     private var client = HttpClient(CIO) {
         install(WebSockets)
     }
+    private var websocket: DefaultWebSocketSession? = null
     private var serverAddress: String = ""
     private var token: String = ""
     private var deviceId: String = ""
+    var rtcPeerConnection: RtcPeerConnection? = null
 
     suspend fun initialize() {
         loadPreferences()
@@ -126,32 +128,105 @@ class DeviceService(private val context: Context) {
         }
     }
 
-    fun connect() {
+    suspend fun connect() {
         Log.i(TAG, "Connecting to websocket at $serverAddress")
-        runBlocking {
+        val (host, port) = serverAddress.split(":")
-            val (host, port) = serverAddress.split(":")
+        val portInt = if (port.isEmpty()) 80 else port.toInt()
-            val portInt = if (port.isEmpty()) 80 else port.toInt()
+        client.webSocket(
-            client.webSocket(
+            method = HttpMethod.Get,
-                method = HttpMethod.Get,
+            host = host,
-                host = host,
+            port = portInt,
-                port = portInt,
+            path = "/ws",
-                path = "/ws",
+            request = {
-                request = {
+                cookie(name = "token", value = token)
-                    cookie(name = "token", value = token)
+            }
-                }
+        ) {
-            ) {
+            Log.i(TAG, "Listening for incoming websocket messages")
-                Log.i(TAG, "Listening for incoming websocket messages")
+            websocket = this
-                while (true) {
+            while (true) {
-                    val frame = incoming.receive()
+                val frame = incoming.receive()
-                    if (frame is Frame.Text) {
+                if (frame is Frame.Text) {
-                        val message = frame.readText()
+                    val dataString = frame.readText()
-                        Log.i(TAG, "Received message: $message")
+                    val dataJson = JSONObject(dataString)
+                    val senderId = dataJson.getString("sender")
+                    val message = dataJson.getJSONObject("message")
+                    val type = message.getString("type")
+                    when (type) {
+                        RtcPeerConnection.TYPE_OFFER -> {
+                            Log.i(TAG, "Received offer from $senderId")
+                            val sdp = message.getString("sdp")
+                            rtcPeerConnection?.connect(senderId, sdp)
+                        }
+
+                        RtcPeerConnection.TYPE_ICE_CANDIDATE -> {
+                            Log.i(TAG, "Received ice candidate")
+                            val candidateString = message.getString("candidate")
+                            val candidateJson = JSONObject(candidateString)
+                            val sdpMid = candidateJson.getString("sdpMid")
+                            val sdpMLineIndex = candidateJson.getInt("sdpMLineIndex")
+                            val sdp = candidateJson.getString("candidate")
+                            val candidate = IceCandidate(sdpMid, sdpMLineIndex, sdp)
+                            Log.i(TAG, "Candidate: $candidate")
+                            rtcPeerConnection?.addIceCandidate(candidate)
+                        }
                     }
                 }
             }
         }
     }
 
+    fun sendWebRtcAnswer(targetId: String, sdp: String) {
+        val messageJson = JSONObject()
+        messageJson.put("type", TYPE_SIGNALING)
+        messageJson.put("target", targetId)
+        messageJson.put("message", JSONObject().apply {
+            put("sdp", sdp)
+            put("type", RtcPeerConnection.TYPE_ANSWER)
+        })
+
+        runBlocking {
+            Log.i(TAG, "Sending answer")
+            val frame = Frame.Text(messageJson.toString())
+            websocket?.send(frame)
+        }
+    }
+
+    fun sendWebRtcOffer(targetId: String, sdp: String) {
+        val messageJson = JSONObject()
+        messageJson.put("type", TYPE_SIGNALING)
+        messageJson.put("target", targetId)
+        messageJson.put("message", JSONObject().apply {
+            put("sdp", sdp)
+            put("type", RtcPeerConnection.TYPE_OFFER)
+        })
+
+        runBlocking {
+            Log.i(TAG, "Sending offer")
+            val frame = Frame.Text(messageJson.toString())
+            websocket?.send(frame)
+        }
+    }
+
+    fun sendWebRtcIceCandidate(targetId: String, candidate: IceCandidate) {
+        val messageJson = JSONObject()
+        messageJson.put("type", TYPE_SIGNALING)
+        messageJson.put("target", targetId)
+        messageJson.put("message", JSONObject().apply {
+            put("candidate", JSONObject().apply {
+                put("sdpMid", candidate.sdpMid)
+                put("sdpMLineIndex", candidate.sdpMLineIndex)
+                put("candidate", candidate.sdp)
+            })
+            put("type", RtcPeerConnection.TYPE_ICE_CANDIDATE)
+        })
+
+        runBlocking {
+            Log.i(TAG, "Sending ice candidate")
+            val frame = Frame.Text(messageJson.toString())
+            websocket?.send(frame)
+        }
+    }
+
     fun setServerAddress(url: String) {
         serverAddress = url
     }
@@ -163,4 +238,8 @@ class DeviceService(private val context: Context) {
     fun getToken(): String {
         return token
     }
+
+    companion object {
+        const val TYPE_SIGNALING = "signaling"
+    }
 }

app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt

@@ -3,21 +3,62 @@ package com.example.tvcontroller.ui.components
 import androidx.camera.view.LifecycleCameraController
 import androidx.camera.view.PreviewView
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.MutableState
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.setValue
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.viewinterop.AndroidView
-import androidx.lifecycle.compose.LocalLifecycleOwner
+import org.webrtc.EglBase
+import org.webrtc.RendererCommon
+import org.webrtc.VideoTrack
 
 @Composable
 fun CameraPreview(
-    controller: LifecycleCameraController,
+    eglBaseContext: EglBase.Context, videoTrack: VideoTrack, modifier: Modifier = Modifier
-    modifier: Modifier = Modifier
 ) {
+    val trackState: MutableState<VideoTrack?> = remember { mutableStateOf(null) }
+    var view: VideoTextureViewRenderer? by remember { mutableStateOf(null) }
+
     AndroidView(
-        modifier = modifier,
         factory = {
-            PreviewView(it).apply {
+            VideoTextureViewRenderer(it).apply {
-                this.controller = controller
+                init(
+                    eglBaseContext,
+                    object : RendererCommon.RendererEvents {
+                        override fun onFirstFrameRendered() = Unit
+                        override fun onFrameResolutionChanged(p0: Int, p1: Int, p2: Int) = Unit
+                    }
+                )
+                setupVideo(trackState, videoTrack, this)
+                view = this
             }
         },
+        update = { v -> setupVideo(trackState, videoTrack, v) },
+        modifier = modifier
     )
 }
+
+private fun cleanTrack(
+    view: VideoTextureViewRenderer?,
+    trackState: MutableState<VideoTrack?>
+) {
+    view?.let { trackState.value?.removeSink(it) }
+    trackState.value = null
+}
+
+private fun setupVideo(
+    trackState: MutableState<VideoTrack?>,
+    track: VideoTrack,
+    renderer: VideoTextureViewRenderer
+) {
+    if (trackState.value == track) {
+        return
+    }
+
+    cleanTrack(renderer, trackState)
+
+    trackState.value = track
+    track.addSink(renderer)
+}

app/src/main/java/com/example/tvcontroller/ui/components/VideoTextureViewRenderer.kt

@@ -0,0 +1,191 @@
+package com.example.tvcontroller.ui.components
+/*
+ * Copyright 2023 Stream.IO, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import android.content.Context
+import android.content.res.Resources
+import android.graphics.SurfaceTexture
+import android.os.Handler
+import android.os.Looper
+import android.util.AttributeSet
+import android.view.TextureView
+import android.view.TextureView.SurfaceTextureListener
+import org.webrtc.EglBase
+import org.webrtc.EglRenderer
+import org.webrtc.GlRectDrawer
+import org.webrtc.RendererCommon.RendererEvents
+import org.webrtc.ThreadUtils
+import org.webrtc.VideoFrame
+import org.webrtc.VideoSink
+import java.util.concurrent.CountDownLatch
+
+/**
+ * Custom [TextureView] used to render local/incoming videos on the screen.
+ */
+open class VideoTextureViewRenderer @JvmOverloads constructor(
+    context: Context,
+    attrs: AttributeSet? = null
+) : TextureView(context, attrs), VideoSink, SurfaceTextureListener {
+
+    /**
+     * Cached resource name.
+     */
+    private val resourceName: String = getResourceName()
+
+    /**
+     * Renderer used to render the video.
+     */
+    private val eglRenderer: EglRenderer = EglRenderer(resourceName)
+
+    /**
+     * Callback used for reporting render events.
+     */
+    private var rendererEvents: RendererEvents? = null
+
+    /**
+     * Handler to access the UI thread.
+     */
+    private val uiThreadHandler = Handler(Looper.getMainLooper())
+
+    /**
+     * Whether the first frame has been rendered or not.
+     */
+    private var isFirstFrameRendered = false
+
+    /**
+     * The rotated [VideoFrame] width.
+     */
+    private var rotatedFrameWidth = 0
+
+    /**
+     * The rotated [VideoFrame] height.
+     */
+    private var rotatedFrameHeight = 0
+
+    /**
+     * The rotated [VideoFrame] rotation.
+     */
+    private var frameRotation = 0
+
+    init {
+        surfaceTextureListener = this
+    }
+
+    /**
+     * Called when a new frame is received. Sends the frame to be rendered.
+     *
+     * @param videoFrame The [VideoFrame] received from WebRTC connection to draw on the screen.
+     */
+    override fun onFrame(videoFrame: VideoFrame) {
+        eglRenderer.onFrame(videoFrame)
+        updateFrameData(videoFrame)
+    }
+
+    /**
+     * Updates the frame data and notifies [rendererEvents] about the changes.
+     */
+    private fun updateFrameData(videoFrame: VideoFrame) {
+        if (isFirstFrameRendered) {
+            rendererEvents?.onFirstFrameRendered()
+            isFirstFrameRendered = true
+        }
+
+        if (videoFrame.rotatedWidth != rotatedFrameWidth ||
+            videoFrame.rotatedHeight != rotatedFrameHeight ||
+            videoFrame.rotation != frameRotation
+        ) {
+            rotatedFrameWidth = videoFrame.rotatedWidth
+            rotatedFrameHeight = videoFrame.rotatedHeight
+            frameRotation = videoFrame.rotation
+
+            uiThreadHandler.post {
+                rendererEvents?.onFrameResolutionChanged(
+                    rotatedFrameWidth,
+                    rotatedFrameHeight,
+                    frameRotation
+                )
+            }
+        }
+    }
+
+    /**
+     * After the view is laid out we need to set the correct layout aspect ratio to the renderer so that the image
+     * is scaled correctly.
+     */
+    override fun onLayout(changed: Boolean, left: Int, top: Int, right: Int, bottom: Int) {
+        eglRenderer.setLayoutAspectRatio((right - left) / (bottom.toFloat() - top))
+    }
+
+    /**
+     * Initialise the renderer. Should be called from the main thread.
+     *
+     * @param sharedContext [EglBase.Context]
+     * @param rendererEvents Sets the render event listener.
+     */
+    fun init(
+        sharedContext: EglBase.Context,
+        rendererEvents: RendererEvents
+    ) {
+        ThreadUtils.checkIsOnMainThread()
+        this.rendererEvents = rendererEvents
+        eglRenderer.init(sharedContext, EglBase.CONFIG_PLAIN, GlRectDrawer())
+    }
+
+    /**
+     * [SurfaceTextureListener] callback that lets us know when a surface texture is ready and we can draw on it.
+     */
+    override fun onSurfaceTextureAvailable(
+        surfaceTexture: SurfaceTexture,
+        width: Int,
+        height: Int
+    ) {
+        eglRenderer.createEglSurface(surfaceTexture)
+    }
+
+    /**
+     * [SurfaceTextureListener] callback that lets us know when a surface texture is destroyed we need to stop drawing
+     * on it.
+     */
+    override fun onSurfaceTextureDestroyed(surfaceTexture: SurfaceTexture): Boolean {
+        val completionLatch = CountDownLatch(1)
+        eglRenderer.releaseEglSurface { completionLatch.countDown() }
+        ThreadUtils.awaitUninterruptibly(completionLatch)
+        return true
+    }
+
+    override fun onSurfaceTextureSizeChanged(
+        surfaceTexture: SurfaceTexture,
+        width: Int,
+        height: Int
+    ) {
+    }
+
+    override fun onSurfaceTextureUpdated(surfaceTexture: SurfaceTexture) {}
+
+    override fun onDetachedFromWindow() {
+        eglRenderer.release()
+        super.onDetachedFromWindow()
+    }
+
+    private fun getResourceName(): String {
+        return try {
+            resources.getResourceEntryName(id) + ": "
+        } catch (e: Resources.NotFoundException) {
+            ""
+        }
+    }
+}

app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt

@@ -1,26 +1,22 @@
 package com.example.tvcontroller.ui.views
 
-import androidx.camera.core.ImageAnalysis
-import androidx.camera.view.CameraController
-import androidx.camera.view.LifecycleCameraController
 import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.padding
 import androidx.compose.runtime.Composable
-import androidx.compose.runtime.remember
 import androidx.compose.ui.Modifier
-import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.unit.dp
-import androidx.core.content.ContextCompat
 import com.example.tvcontroller.ui.components.CameraPreview
+import org.webrtc.EglBase
+import org.webrtc.VideoTrack
 
 @Composable
-fun CameraView(cameraController: LifecycleCameraController) {
+fun CameraView(eglBaseContext: EglBase.Context, videoTrack: VideoTrack) {
     Box(
         modifier = Modifier
             .fillMaxSize()
             .padding(all = 16.dp),
     ) {
-        CameraPreview(controller = cameraController, modifier = Modifier.fillMaxSize())
+        CameraPreview(eglBaseContext = eglBaseContext, videoTrack = videoTrack, modifier = Modifier.fillMaxSize())
     }
 }

app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt

@@ -8,18 +8,18 @@ import org.webrtc.CapturerObserver
 import org.webrtc.VideoFrame
 import java.nio.ByteBuffer
 
-class CameraXCapturer : ImageAnalysis.Analyzer {
+private const val TAG = "CameraXCapturer"
-    private var capturerObserver: CapturerObserver? = null
 
-    fun setCapturerObserver(capturerObserver: CapturerObserver) {
+class CameraXCapturer : ImageAnalysis.Analyzer {
-        this.capturerObserver = capturerObserver
+    var capturerObserver: CapturerObserver? = null
-    }
 
     override fun analyze(image: ImageProxy) {
         if (image.format != YUV_420_888) throw Exception("Unsupported format")
-        Log.i("CameraXCapturer", "Received image from CameraX")
         var videoFrame = imageProxyToVideoFrame(image)
+        Log.i(TAG, "Handing frame to capturer observer $capturerObserver")
         capturerObserver?.onFrameCaptured(videoFrame) ?: image.close()
+        Log.i(TAG, "Frame handled by capturer observer")
+        image.close()
     }
 
     fun imageProxyToVideoFrame(image: ImageProxy): VideoFrame {

app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt

@@ -1,30 +1,60 @@
 package com.example.tvcontroller.webrtc
 
 import android.content.Context
+import android.hardware.camera2.CameraCharacteristics
 import android.hardware.camera2.CameraManager
 import android.util.Log
+import androidx.compose.ui.Modifier
+import com.example.tvcontroller.services.DeviceService
 import org.webrtc.AudioTrack
 import org.webrtc.Camera2Capturer
 import org.webrtc.DataChannel
+import org.webrtc.DefaultVideoDecoderFactory
+import org.webrtc.DefaultVideoEncoderFactory
+import org.webrtc.EglBase
+import org.webrtc.HardwareVideoEncoderFactory
 import org.webrtc.IceCandidate
 import org.webrtc.MediaConstraints
 import org.webrtc.MediaStream
 import org.webrtc.PeerConnection
 import org.webrtc.PeerConnectionFactory
 import org.webrtc.PeerConnectionFactory.InitializationOptions
+import org.webrtc.RtpSender
+import org.webrtc.RtpTransceiver
 import org.webrtc.SdpObserver
 import org.webrtc.SessionDescription
+import org.webrtc.SimulcastVideoEncoderFactory
+import org.webrtc.SoftwareVideoEncoderFactory
+import org.webrtc.SurfaceTextureHelper
+import org.webrtc.VideoSource
 import org.webrtc.VideoTrack
+import kotlin.getValue
 
-const val TAG = "RtcPeerConnection"
+private const val TAG = "RtcPeerConnection"
 
 class RtcPeerConnection(private val context: Context) : PeerConnection.Observer {
     private val peerConnectionFactory by lazy { initializeFactory() }
     private val iceServers by lazy { initializeIceServers() }
-    val cameraXCapturer by lazy { CameraXCapturer() }
     private val audioTrack by lazy { createAudioTrack() }
-    private val videoTrack by lazy { createVideoTrack() }
+    val videoTrack by lazy { createVideoTrack() }
+    var eglBaseContext: EglBase.Context = EglBase.create().eglBaseContext
+    private val cameraManager by lazy { context.getSystemService(Context.CAMERA_SERVICE) as CameraManager }
+    private val videoCapturer by lazy { createCameraCapturer() }
+    private val surfaceTextureHelper by lazy { createSurfaceTextureHelper() }
+    private val videoSource by lazy { createVideoSource() }
+    private val videoDecoderFactory by lazy { DefaultVideoDecoderFactory(eglBaseContext) }
+    private val videoEncoderFactory by lazy {
+        SimulcastVideoEncoderFactory(
+            HardwareVideoEncoderFactory(eglBaseContext, true, true),
+            SoftwareVideoEncoderFactory()
+        )
+    }
     private var peerConnection: PeerConnection? = null
+    var deviceService: DeviceService? = null
+
+    private var peerId: String = ""
+
+    private var offer = ""
 
     private fun initializeFactory(): PeerConnectionFactory {
         val initOptions = InitializationOptions.builder(context).createInitializationOptions()
@@ -32,7 +62,9 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer
 
         val options = PeerConnectionFactory.Options()
         val peerConnectionFactory =
-            PeerConnectionFactory.builder().setOptions(options).createPeerConnectionFactory()
+            PeerConnectionFactory.builder().setVideoDecoderFactory(videoDecoderFactory)
+                .setVideoEncoderFactory(videoEncoderFactory).setOptions(options)
+                .createPeerConnectionFactory()
         return peerConnectionFactory
     }
 
@@ -52,48 +84,143 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer
     }
 
     private fun createVideoTrack(): VideoTrack {
-        val videoSource = peerConnectionFactory.createVideoSource(false)
-        cameraXCapturer.setCapturerObserver(videoSource.capturerObserver)
         val localVideoTrack = peerConnectionFactory.createVideoTrack("video_track", videoSource)
-        localVideoTrack.setEnabled(true)
+        //localVideoTrack.setEnabled(true)
         return localVideoTrack
     }
 
-    fun connect() {
+    private fun createCameraCapturer(): Camera2Capturer {
+        val ids = cameraManager.cameraIdList
+        var foundCamera = false;
+        var cameraId = ""
+        for (id in ids) {
+            val characteristics = cameraManager.getCameraCharacteristics(id)
+            val facing = characteristics.get(CameraCharacteristics.LENS_FACING)
+            if (facing == CameraCharacteristics.LENS_FACING_BACK) {
+                cameraId = id
+                foundCamera = true
+                break
+            }
+        }
+        if (!foundCamera) {
+            cameraId = ids.first()
+        }
+        val cameraCapturer = Camera2Capturer(context, cameraId, null)
+        return cameraCapturer
+    }
+
+    private fun createSurfaceTextureHelper(): SurfaceTextureHelper {
+        val surfaceTextureHelper =
+            SurfaceTextureHelper.create("SurfaceTextureHelperThread", eglBaseContext)
+        return surfaceTextureHelper
+    }
+
+    private fun createVideoSource(): VideoSource {
+        val videoSource = peerConnectionFactory.createVideoSource(false)
+        videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver)
+        videoCapturer.startCapture(1920, 1080, 30)
+        return videoSource
+    }
+
+    fun connect(targetId: String, sdp: String) {
+        peerId = targetId
+        offer = sdp
         Log.i(TAG, "Connecting rtc peer connection")
         var rtcConfig = PeerConnection.RTCConfiguration(iceServers)
-        var peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this)
+        Log.i(TAG, "Creating peer connection")
-
+        peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this)
-        var mediaConstraints = MediaConstraints()
+        Log.i(TAG, "Peer connection created")
-        mediaConstraints.mandatory.add(
+        //peerConnection?.addTransceiver(audioTrack, RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.SEND_ONLY))
-            MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")
+        //peerConnection?.addTransceiver( videoTrack, RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.SEND_ONLY) )
-        );
-        mediaConstraints.mandatory.add(
-            MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")
-        );
         peerConnection?.addTrack(audioTrack)
         peerConnection?.addTrack(videoTrack)
-        peerConnection?.createOffer(object : SdpObserver {
+        handleOffer(sdp)
-            override fun onCreateSuccess(sessionDescription: SessionDescription?) {
+    }
-                Log.i(TAG, "onCreateSuccess: ${sessionDescription?.description}")
-            }
 
+    fun handleOffer(sdp: String) {
+        var mediaConstraints = MediaConstraints()
+        var localSessionDescription: SessionDescription? = null
+        val remoteSessionDescription = SessionDescription(SessionDescription.Type.OFFER, sdp)
+
+        Log.i(TAG, "Handling offer $sdp")
+        val onLocalDescriptionSet = object : SdpObserver {
             override fun onSetSuccess() {
-                Log.i(TAG, "onSetSuccess")
+                Log.i(TAG, "Local description set")
-                TODO("Not yet implemented")
+                peerConnection?.transceivers?.forEach {
+
+                    Log.i(TAG, "${it.mediaType} Transceiver: ${it.currentDirection}")
+                }
+                deviceService?.sendWebRtcAnswer(
+                    peerId, localSessionDescription?.description ?: ""
+                )
             }
 
-            override fun onCreateFailure(p0: String?) {
+            override fun onCreateSuccess(sessionDescription: SessionDescription?) {}
-                Log.i(TAG, "onCreateFailure: $p0")
+            override fun onCreateFailure(p0: String?) {}
-                TODO("Not yet implemented")
+            override fun onSetFailure(p0: String?) {}
+        }
+
+        val onAnswerCreated = object : SdpObserver {
+            override fun onCreateSuccess(sessionDescription: SessionDescription?) {
+                Log.i(TAG, "Answer created")
+                localSessionDescription = sessionDescription
+                peerConnection?.setLocalDescription(onLocalDescriptionSet, sessionDescription)
             }
 
-            override fun onSetFailure(p0: String?) {
+            override fun onSetSuccess() {}
-                Log.i(TAG, "onSetFailure: $p0")
+            override fun onCreateFailure(p0: String?) {}
-                TODO("Not yet implemented")
+            override fun onSetFailure(p0: String?) {}
+        }
+        val onRemoteDescriptionSet = object : SdpObserver {
+            override fun onSetSuccess() {
+                Log.i(TAG, "Remote description set")
+                peerConnection?.createAnswer(onAnswerCreated, mediaConstraints)
             }
 
-        }, mediaConstraints)
+            override fun onCreateSuccess(p0: SessionDescription?) {}
+            override fun onCreateFailure(p0: String?) {}
+            override fun onSetFailure(p0: String?) {}
+        }
+        Log.i(TAG, "Setting remote description")
+        peerConnection?.setRemoteDescription(onRemoteDescriptionSet, remoteSessionDescription)
+    }
+
+    fun createOffer() {
+        var mediaConstraints = MediaConstraints()
+        var localSessionDescription: SessionDescription? = null
+
+        val onLocalDescriptionSet = object : SdpObserver {
+            override fun onSetSuccess() {
+                Log.i(TAG, "Local description set")
+                peerConnection?.transceivers?.forEach {
+                    Log.i(TAG, "${it.mediaType} Transceiver: ${it.currentDirection}")
+                }
+                deviceService?.sendWebRtcOffer(
+                    peerId, localSessionDescription?.description ?: ""
+                )
+            }
+
+            override fun onCreateSuccess(sessionDescription: SessionDescription?) {}
+            override fun onCreateFailure(p0: String?) {}
+            override fun onSetFailure(p0: String?) {}
+        }
+        val onOfferCreated = object : SdpObserver {
+            override fun onCreateSuccess(sessionDescription: SessionDescription?) {
+                Log.i(TAG, "Offer created")
+                peerConnection?.setLocalDescription(onLocalDescriptionSet, sessionDescription)
+            }
+
+            override fun onSetSuccess() {}
+            override fun onCreateFailure(p0: String?) {}
+            override fun onSetFailure(p0: String?) {}
+        }
+        Log.i(TAG, "Creating offer")
+        peerConnection?.createOffer(onOfferCreated, mediaConstraints)
+    }
+
+    fun addIceCandidate(iceCandidate: IceCandidate) {
+        Log.i(TAG, "Adding ice candidate $iceCandidate to $peerConnection")
+        peerConnection?.addIceCandidate(iceCandidate)
     }
 
     override fun onSignalingChange(p0: PeerConnection.SignalingState?) {
@@ -102,6 +229,9 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer
 
     override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) {
         Log.i(TAG, "onIceConnectionChange: $p0")
+        if (p0 == PeerConnection.IceConnectionState.CONNECTED) {
+            videoCapturer.startCapture(1280, 720, 30)
+        }
     }
 
     override fun onIceConnectionReceivingChange(p0: Boolean) {
@@ -114,6 +244,9 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer
 
     override fun onIceCandidate(p0: IceCandidate?) {
         Log.i(TAG, "onIceCandidate: $p0")
+        p0?.let {
+            deviceService?.sendWebRtcIceCandidate(peerId, it)
+        }
     }
 
     override fun onIceCandidatesRemoved(p0: Array<out IceCandidate?>?) {
@@ -134,5 +267,16 @@ class RtcPeerConnection(private val context: Context) : PeerConnection.Observer
 
     override fun onRenegotiationNeeded() {
         Log.i(TAG, "onRenegotiationNeeded")
+        //if (offer.isNotEmpty()) {
+        //    handleOffer(offer)
+        //} else {
+        //    createOffer()
+        //}
+    }
+
+    companion object {
+        const val TYPE_OFFER = "offer"
+        const val TYPE_ANSWER = "answer"
+        const val TYPE_ICE_CANDIDATE = "ice_candidate"
     }
 }