fritz/tv-controller-android
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use camerax with webrtc lib

Browse Source
This commit is contained in:
Fritz Heiden 2025-04-01 08:07:06 +02:00
parent 7471168a21
commit 2f90fd7b09
7 changed files with 292 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/build.gradle.kts
View File

@ -61,6 +61,7 @@ dependencies {
implementation(libs.androidx.camera.mlkit.vision)
implementation(libs.androidx.camera.extensions)
implementation(libs.material3)
implementation(libs.stream.webrtc.android)
testImplementation(libs.junit)
androidTestImplementation(libs.androidx.junit)
androidTestImplementation(libs.androidx.espresso.core)

47
app/src/main/java/com/example/tvcontroller/MainActivity.kt
View File

@ -5,6 +5,8 @@ import android.util.Log
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.activity.enableEdgeToEdge
import androidx.camera.view.CameraController
import androidx.camera.view.LifecycleCameraController
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.material3.Icon
@ -13,24 +15,27 @@ import androidx.compose.material3.NavigationBarItem
import androidx.compose.material3.Scaffold
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.res.painterResource
import androidx.core.app.ActivityCompat
import androidx.core.content.ContextCompat
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.lifecycleScope
import androidx.navigation.NavHostController
import androidx.navigation.compose.NavHost
import androidx.navigation.compose.composable
import androidx.navigation.compose.currentBackStackEntryAsState
import androidx.navigation.compose.rememberNavController
import com.example.tvcontroller.ui.theme.TVControllerTheme
import androidx.compose.runtime.getValue
import androidx.compose.ui.res.painterResource
import androidx.core.app.ActivityCompat
import androidx.lifecycle.lifecycleScope
import com.example.tvcontroller.services.BluetoothService
import com.example.tvcontroller.services.CameraService
import com.example.tvcontroller.services.ControllerService
import com.example.tvcontroller.services.DeviceService
import com.example.tvcontroller.ui.theme.TVControllerTheme
import com.example.tvcontroller.ui.views.CameraView
import com.example.tvcontroller.ui.views.RemoteView
import com.example.tvcontroller.ui.views.SettingsView
import com.example.tvcontroller.webrtc.RtcPeerConnection
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
@ -45,10 +50,22 @@ class MainActivity : ComponentActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
bluetoothService = BluetoothService(this.applicationContext)
deviceService = DeviceService(this.applicationContext)
cameraService = CameraService(this.applicationContext)
controllerService = ControllerService(this.applicationContext, bluetoothService)
val lifecycleOwner: LifecycleOwner = this
val rtcPeerConnection = RtcPeerConnection(applicationContext)
val cameraController =
LifecycleCameraController(applicationContext).apply {
setEnabledUseCases(CameraController.IMAGE_ANALYSIS)
setImageAnalysisAnalyzer(
ContextCompat.getMainExecutor(applicationContext),
rtcPeerConnection.cameraXCapturer
)
bindToLifecycle(lifecycleOwner)
}
bluetoothService = BluetoothService(applicationContext)
deviceService = DeviceService(applicationContext)
cameraService = CameraService(applicationContext)
controllerService = ControllerService(applicationContext, bluetoothService)
checkPermissions()
lifecycleScope.launch(Dispatchers.IO) {
@ -60,7 +77,9 @@ class MainActivity : ComponentActivity() {
TvControllerApp(
deviceService = deviceService,
controllerService = controllerService,
bluetoothService = bluetoothService
bluetoothService = bluetoothService,
rtcPeerConnection = rtcPeerConnection,
cameraController = cameraController
)
}
}
@ -83,7 +102,9 @@ fun TvControllerApp(
navController: NavHostController = rememberNavController(),
deviceService: DeviceService,
controllerService: ControllerService,
bluetoothService: BluetoothService
bluetoothService: BluetoothService,
rtcPeerConnection: RtcPeerConnection,
cameraController: LifecycleCameraController
) {
val backStackEntry by navController.currentBackStackEntryAsState()
val currentScreen = Screen.valueOf(backStackEntry?.destination?.route ?: Screen.Camera.name)
@ -129,11 +150,11 @@ fun TvControllerApp(
}) { innerPadding ->
NavHost(
navController = navController,
startDestination = Screen.Remote.name,
startDestination = Screen.Settings.name,
modifier = Modifier.padding(innerPadding)
) {
composable(route = Screen.Camera.name) {
CameraView()
CameraView(cameraController)
}
composable(route = Screen.Remote.name) {
RemoteView(

2
app/src/main/java/com/example/tvcontroller/ui/components/CameraPreview.kt
View File

@ -12,13 +12,11 @@ fun CameraPreview(
controller: LifecycleCameraController,
modifier: Modifier = Modifier
) {
val lifecycleOwner = LocalLifecycleOwner.current
AndroidView(
modifier = modifier,
factory = {
PreviewView(it).apply {
this.controller = controller
controller.bindToLifecycle(lifecycleOwner)
}
},
)

16
app/src/main/java/com/example/tvcontroller/ui/views/CameraView.kt
View File

@ -1,34 +1,26 @@
package com.example.tvcontroller.ui.views
import androidx.camera.core.ImageAnalysis
import androidx.camera.view.CameraController
import androidx.camera.view.LifecycleCameraController
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import com.example.tvcontroller.ui.components.CameraPreview
@Composable
fun CameraView() {
val context = LocalContext.current
val controller = remember {
LifecycleCameraController(context).apply {
setEnabledUseCases(CameraController.VIDEO_CAPTURE)
}
}
fun CameraView(cameraController: LifecycleCameraController) {
Box(
modifier = Modifier
.fillMaxSize()
.padding(all = 16.dp),
) {
CameraPreview(controller = controller, modifier = Modifier.fillMaxSize())
CameraPreview(controller = cameraController, modifier = Modifier.fillMaxSize())
}
}

113
app/src/main/java/com/example/tvcontroller/webrtc/CameraXCapturer.kt Normal file
View File

@ -0,0 +1,113 @@
package com.example.tvcontroller.webrtc
import android.graphics.ImageFormat.YUV_420_888
import android.util.Log
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageProxy
import org.webrtc.CapturerObserver
import org.webrtc.VideoFrame
import java.nio.ByteBuffer
class CameraXCapturer : ImageAnalysis.Analyzer {
private var capturerObserver: CapturerObserver? = null
fun setCapturerObserver(capturerObserver: CapturerObserver) {
this.capturerObserver = capturerObserver
}
override fun analyze(image: ImageProxy) {
if (image.format != YUV_420_888) throw Exception("Unsupported format")
Log.i("CameraXCapturer", "Received image from CameraX")
var videoFrame = imageProxyToVideoFrame(image)
capturerObserver?.onFrameCaptured(videoFrame) ?: image.close()
}
fun imageProxyToVideoFrame(image: ImageProxy): VideoFrame {
var buffer = object : VideoFrame.I420Buffer {
private var refCount = 0
override fun getWidth(): Int {
return image.width
}
override fun getHeight(): Int {
return image.height
}
override fun toI420(): VideoFrame.I420Buffer? {
return this
}
override fun retain() {
refCount++
}
override fun release() {
refCount--
if (refCount == 0) {
image.close()
}
}
override fun cropAndScale(
cropX: Int,
cropY: Int,
cropWidth: Int,
cropHeight: Int,
scaleWidth: Int,
scaleHeight: Int
): VideoFrame.Buffer? {
return this
}
override fun getDataY(): ByteBuffer? {
val format = image.format
if (format == YUV_420_888) {
return image.planes[0].buffer
}
return image.planes[0].buffer
}
override fun getDataU(): ByteBuffer? {
val format = image.format
if (format == YUV_420_888) {
return image.planes[1].buffer
}
return image.planes[0].buffer
}
override fun getDataV(): ByteBuffer? {
val format = image.format
if (format == YUV_420_888) {
return image.planes[2].buffer
}
return image.planes[0].buffer
}
override fun getStrideY(): Int {
val format = image.format
if (format == YUV_420_888) {
return image.planes[0].pixelStride
}
return image.planes[0].pixelStride
}
override fun getStrideU(): Int {
val format = image.format
if (format == YUV_420_888) {
return image.planes[1].pixelStride
}
return image.planes[0].pixelStride
}
override fun getStrideV(): Int {
val format = image.format
if (format == YUV_420_888) {
return image.planes[2].pixelStride
}
return image.planes[0].pixelStride
}
}
var videoFrame = VideoFrame(buffer, 0, 0)
return videoFrame
}
}

138
app/src/main/java/com/example/tvcontroller/webrtc/RtcPeerConnection.kt Normal file
View File

@ -0,0 +1,138 @@
package com.example.tvcontroller.webrtc
import android.content.Context
import android.hardware.camera2.CameraManager
import android.util.Log
import org.webrtc.AudioTrack
import org.webrtc.Camera2Capturer
import org.webrtc.DataChannel
import org.webrtc.IceCandidate
import org.webrtc.MediaConstraints
import org.webrtc.MediaStream
import org.webrtc.PeerConnection
import org.webrtc.PeerConnectionFactory
import org.webrtc.PeerConnectionFactory.InitializationOptions
import org.webrtc.SdpObserver
import org.webrtc.SessionDescription
import org.webrtc.VideoTrack
const val TAG = "RtcPeerConnection"
class RtcPeerConnection(private val context: Context) : PeerConnection.Observer {
private val peerConnectionFactory by lazy { initializeFactory() }
private val iceServers by lazy { initializeIceServers() }
val cameraXCapturer by lazy { CameraXCapturer() }
private val audioTrack by lazy { createAudioTrack() }
private val videoTrack by lazy { createVideoTrack() }
private var peerConnection: PeerConnection? = null
private fun initializeFactory(): PeerConnectionFactory {
val initOptions = InitializationOptions.builder(context).createInitializationOptions()
PeerConnectionFactory.initialize(initOptions)
val options = PeerConnectionFactory.Options()
val peerConnectionFactory =
PeerConnectionFactory.builder().setOptions(options).createPeerConnectionFactory()
return peerConnectionFactory
}
private fun initializeIceServers(): ArrayList<PeerConnection.IceServer> {
val iceServers = ArrayList<PeerConnection.IceServer>()
iceServers.add(
PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer()
)
return iceServers
}
private fun createAudioTrack(): AudioTrack {
val audioConstraints = MediaConstraints()
val audioSource = peerConnectionFactory.createAudioSource(audioConstraints)
val localAudioTrack = peerConnectionFactory.createAudioTrack("audio_track", audioSource)
return localAudioTrack
}
private fun createVideoTrack(): VideoTrack {
val videoSource = peerConnectionFactory.createVideoSource(false)
cameraXCapturer.setCapturerObserver(videoSource.capturerObserver)
val localVideoTrack = peerConnectionFactory.createVideoTrack("video_track", videoSource)
localVideoTrack.setEnabled(true)
return localVideoTrack
}
fun connect() {
Log.i(TAG, "Connecting rtc peer connection")
var rtcConfig = PeerConnection.RTCConfiguration(iceServers)
var peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this)
var mediaConstraints = MediaConstraints()
mediaConstraints.mandatory.add(
MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")
);
mediaConstraints.mandatory.add(
MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")
);
peerConnection?.addTrack(audioTrack)
peerConnection?.addTrack(videoTrack)
peerConnection?.createOffer(object : SdpObserver {
override fun onCreateSuccess(sessionDescription: SessionDescription?) {
Log.i(TAG, "onCreateSuccess: ${sessionDescription?.description}")
}
override fun onSetSuccess() {
Log.i(TAG, "onSetSuccess")
TODO("Not yet implemented")
}
override fun onCreateFailure(p0: String?) {
Log.i(TAG, "onCreateFailure: $p0")
TODO("Not yet implemented")
}
override fun onSetFailure(p0: String?) {
Log.i(TAG, "onSetFailure: $p0")
TODO("Not yet implemented")
}
}, mediaConstraints)
}
override fun onSignalingChange(p0: PeerConnection.SignalingState?) {
Log.i(TAG, "onSignalingChange: $p0")
}
override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) {
Log.i(TAG, "onIceConnectionChange: $p0")
}
override fun onIceConnectionReceivingChange(p0: Boolean) {
Log.i(TAG, "onIceConnectionReceivingChange: $p0")
}
override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) {
Log.i(TAG, "onIceGatheringChange: $p0")
}
override fun onIceCandidate(p0: IceCandidate?) {
Log.i(TAG, "onIceCandidate: $p0")
}
override fun onIceCandidatesRemoved(p0: Array<out IceCandidate?>?) {
Log.i(TAG, "onIceCandidatesRemoved: $p0")
}
override fun onAddStream(p0: MediaStream?) {
Log.i(TAG, "onAddStream: $p0")
}
override fun onRemoveStream(p0: MediaStream?) {
Log.i(TAG, "onRemoveStream: $p0")
}
override fun onDataChannel(p0: DataChannel?) {
Log.i(TAG, "onDataChannel: $p0")
}
override fun onRenegotiationNeeded() {
Log.i(TAG, "onRenegotiationNeeded")
}
}

2
gradle/libs.versions.toml
View File

@ -12,6 +12,7 @@ activityCompose = "1.10.1"
composeBom = "2025.03.01"
material3 = "1.4.0-alpha11"
navigationCompose = "2.8.9"
streamWebrtcAndroid = "1.3.8"
[libraries]
androidx-camera-camera2 = { module = "androidx.camera:camera-camera2", version.ref = "cameraCore" }
@ -40,6 +41,7 @@ ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" }
ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" }
ktor-client-websockets = { module = "io.ktor:ktor-client-websockets", version.ref = "ktor" }
material3 = { module = "androidx.compose.material3:material3", version.ref = "material3" }
stream-webrtc-android = { module = "io.getstream:stream-webrtc-android", version.ref = "streamWebrtcAndroid" }
[plugins]
android-application = { id = "com.android.application", version.ref = "agp" }