feat: use camerax with webrtc lib
This commit is contained in:
parent
645f8e2f04
commit
3d3f8710f2
@ -61,6 +61,7 @@ dependencies {
|
||||
implementation(libs.androidx.camera.mlkit.vision)
|
||||
implementation(libs.androidx.camera.extensions)
|
||||
implementation(libs.material3)
|
||||
implementation(libs.stream.webrtc.android)
|
||||
testImplementation(libs.junit)
|
||||
androidTestImplementation(libs.androidx.junit)
|
||||
androidTestImplementation(libs.androidx.espresso.core)
|
||||
|
||||
@ -5,6 +5,9 @@ import android.util.Log
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.setContent
|
||||
import androidx.activity.enableEdgeToEdge
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.view.CameraController
|
||||
import androidx.camera.view.LifecycleCameraController
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.material3.Icon
|
||||
@ -21,8 +24,13 @@ import androidx.navigation.compose.currentBackStackEntryAsState
|
||||
import androidx.navigation.compose.rememberNavController
|
||||
import com.example.tvcontroller.ui.theme.TVControllerTheme
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.res.painterResource
|
||||
import androidx.core.app.ActivityCompat
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
import com.example.tvcontroller.services.BluetoothService
|
||||
import com.example.tvcontroller.services.CameraService
|
||||
import com.example.tvcontroller.services.ControllerService
|
||||
@ -30,6 +38,10 @@ import com.example.tvcontroller.services.DeviceService
|
||||
import com.example.tvcontroller.ui.views.CameraView
|
||||
import com.example.tvcontroller.ui.views.RemoteView
|
||||
import com.example.tvcontroller.ui.views.SettingsView
|
||||
import com.example.tvcontroller.webrtc.CameraXCapturer
|
||||
import com.example.tvcontroller.webrtc.RtcPeerConnection
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
|
||||
const val TAG = "MainActivity"
|
||||
@ -42,10 +54,22 @@ class MainActivity : ComponentActivity() {
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
bluetoothService = BluetoothService(this.applicationContext)
|
||||
deviceService = DeviceService(this.applicationContext)
|
||||
cameraService = CameraService(this.applicationContext)
|
||||
controllerService = ControllerService(this.applicationContext, bluetoothService)
|
||||
val lifecycleOwner: LifecycleOwner = this
|
||||
val rtcPeerConnection = RtcPeerConnection(applicationContext)
|
||||
val cameraController =
|
||||
LifecycleCameraController(applicationContext).apply {
|
||||
setEnabledUseCases(CameraController.IMAGE_ANALYSIS)
|
||||
setImageAnalysisAnalyzer(
|
||||
ContextCompat.getMainExecutor(applicationContext),
|
||||
rtcPeerConnection.cameraXCapturer
|
||||
)
|
||||
bindToLifecycle(lifecycleOwner)
|
||||
|
||||
}
|
||||
bluetoothService = BluetoothService(applicationContext)
|
||||
deviceService = DeviceService(applicationContext)
|
||||
cameraService = CameraService(applicationContext)
|
||||
controllerService = ControllerService(applicationContext, bluetoothService)
|
||||
checkPermissions()
|
||||
enableEdgeToEdge()
|
||||
setContent {
|
||||
@ -53,7 +77,9 @@ class MainActivity : ComponentActivity() {
|
||||
TvControllerApp(
|
||||
deviceService = deviceService,
|
||||
controllerService = controllerService,
|
||||
bluetoothService = bluetoothService
|
||||
bluetoothService = bluetoothService,
|
||||
rtcPeerConnection = rtcPeerConnection,
|
||||
cameraController = cameraController
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -76,7 +102,9 @@ fun TvControllerApp(
|
||||
navController: NavHostController = rememberNavController(),
|
||||
deviceService: DeviceService,
|
||||
controllerService: ControllerService,
|
||||
bluetoothService: BluetoothService
|
||||
bluetoothService: BluetoothService,
|
||||
rtcPeerConnection: RtcPeerConnection,
|
||||
cameraController: LifecycleCameraController
|
||||
) {
|
||||
val backStackEntry by navController.currentBackStackEntryAsState()
|
||||
val currentScreen = Screen.valueOf(backStackEntry?.destination?.route ?: Screen.Camera.name)
|
||||
@ -122,11 +150,11 @@ fun TvControllerApp(
|
||||
}) { innerPadding ->
|
||||
NavHost(
|
||||
navController = navController,
|
||||
startDestination = Screen.Remote.name,
|
||||
startDestination = Screen.Settings.name,
|
||||
modifier = Modifier.padding(innerPadding)
|
||||
) {
|
||||
composable(route = Screen.Camera.name) {
|
||||
CameraView()
|
||||
CameraView(cameraController)
|
||||
}
|
||||
composable(route = Screen.Remote.name) {
|
||||
RemoteView(
|
||||
@ -136,7 +164,8 @@ fun TvControllerApp(
|
||||
composable(route = Screen.Settings.name) {
|
||||
SettingsView(
|
||||
deviceService = deviceService,
|
||||
bluetoothService = bluetoothService
|
||||
bluetoothService = bluetoothService,
|
||||
rtcPeerConnection = rtcPeerConnection
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,13 +12,11 @@ fun CameraPreview(
|
||||
controller: LifecycleCameraController,
|
||||
modifier: Modifier = Modifier
|
||||
) {
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
AndroidView(
|
||||
modifier = modifier,
|
||||
factory = {
|
||||
PreviewView(it).apply {
|
||||
this.controller = controller
|
||||
controller.bindToLifecycle(lifecycleOwner)
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
@ -1,34 +1,26 @@
|
||||
package com.example.tvcontroller.ui.views
|
||||
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.view.CameraController
|
||||
import androidx.camera.view.LifecycleCameraController
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.example.tvcontroller.ui.components.CameraPreview
|
||||
|
||||
@Composable
|
||||
fun CameraView() {
|
||||
val context = LocalContext.current
|
||||
val controller = remember {
|
||||
LifecycleCameraController(context).apply {
|
||||
setEnabledUseCases(CameraController.VIDEO_CAPTURE)
|
||||
}
|
||||
}
|
||||
fun CameraView(cameraController: LifecycleCameraController) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxSize()
|
||||
.padding(all = 16.dp),
|
||||
) {
|
||||
CameraPreview(controller = controller, modifier = Modifier.fillMaxSize())
|
||||
CameraPreview(controller = cameraController, modifier = Modifier.fillMaxSize())
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,113 @@
|
||||
package com.example.tvcontroller.webrtc
|
||||
|
||||
import android.graphics.ImageFormat.YUV_420_888
|
||||
import android.util.Log
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.core.ImageProxy
|
||||
import org.webrtc.CapturerObserver
|
||||
import org.webrtc.VideoFrame
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
class CameraXCapturer : ImageAnalysis.Analyzer {
|
||||
private var capturerObserver: CapturerObserver? = null
|
||||
|
||||
fun setCapturerObserver(capturerObserver: CapturerObserver) {
|
||||
this.capturerObserver = capturerObserver
|
||||
}
|
||||
|
||||
override fun analyze(image: ImageProxy) {
|
||||
if (image.format != YUV_420_888) throw Exception("Unsupported format")
|
||||
Log.i("CameraXCapturer", "Received image from CameraX")
|
||||
var videoFrame = imageProxyToVideoFrame(image)
|
||||
capturerObserver?.onFrameCaptured(videoFrame) ?: image.close()
|
||||
}
|
||||
|
||||
fun imageProxyToVideoFrame(image: ImageProxy): VideoFrame {
|
||||
var buffer = object : VideoFrame.I420Buffer {
|
||||
private var refCount = 0
|
||||
override fun getWidth(): Int {
|
||||
return image.width
|
||||
}
|
||||
|
||||
override fun getHeight(): Int {
|
||||
return image.height
|
||||
}
|
||||
|
||||
override fun toI420(): VideoFrame.I420Buffer? {
|
||||
return this
|
||||
}
|
||||
|
||||
override fun retain() {
|
||||
refCount++
|
||||
}
|
||||
|
||||
override fun release() {
|
||||
refCount--
|
||||
if (refCount == 0) {
|
||||
image.close()
|
||||
}
|
||||
}
|
||||
|
||||
override fun cropAndScale(
|
||||
cropX: Int,
|
||||
cropY: Int,
|
||||
cropWidth: Int,
|
||||
cropHeight: Int,
|
||||
scaleWidth: Int,
|
||||
scaleHeight: Int
|
||||
): VideoFrame.Buffer? {
|
||||
return this
|
||||
}
|
||||
|
||||
override fun getDataY(): ByteBuffer? {
|
||||
val format = image.format
|
||||
if (format == YUV_420_888) {
|
||||
return image.planes[0].buffer
|
||||
}
|
||||
return image.planes[0].buffer
|
||||
}
|
||||
|
||||
override fun getDataU(): ByteBuffer? {
|
||||
val format = image.format
|
||||
if (format == YUV_420_888) {
|
||||
return image.planes[1].buffer
|
||||
}
|
||||
return image.planes[0].buffer
|
||||
}
|
||||
|
||||
override fun getDataV(): ByteBuffer? {
|
||||
val format = image.format
|
||||
if (format == YUV_420_888) {
|
||||
return image.planes[2].buffer
|
||||
}
|
||||
return image.planes[0].buffer
|
||||
}
|
||||
|
||||
override fun getStrideY(): Int {
|
||||
val format = image.format
|
||||
if (format == YUV_420_888) {
|
||||
return image.planes[0].pixelStride
|
||||
}
|
||||
return image.planes[0].pixelStride
|
||||
}
|
||||
|
||||
override fun getStrideU(): Int {
|
||||
val format = image.format
|
||||
if (format == YUV_420_888) {
|
||||
return image.planes[1].pixelStride
|
||||
}
|
||||
return image.planes[0].pixelStride
|
||||
}
|
||||
|
||||
override fun getStrideV(): Int {
|
||||
val format = image.format
|
||||
if (format == YUV_420_888) {
|
||||
return image.planes[2].pixelStride
|
||||
}
|
||||
return image.planes[0].pixelStride
|
||||
}
|
||||
}
|
||||
var videoFrame = VideoFrame(buffer, 0, 0)
|
||||
return videoFrame
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,138 @@
|
||||
package com.example.tvcontroller.webrtc
|
||||
|
||||
import android.content.Context
|
||||
import android.hardware.camera2.CameraManager
|
||||
import android.util.Log
|
||||
import org.webrtc.AudioTrack
|
||||
import org.webrtc.Camera2Capturer
|
||||
import org.webrtc.DataChannel
|
||||
import org.webrtc.IceCandidate
|
||||
import org.webrtc.MediaConstraints
|
||||
import org.webrtc.MediaStream
|
||||
import org.webrtc.PeerConnection
|
||||
import org.webrtc.PeerConnectionFactory
|
||||
import org.webrtc.PeerConnectionFactory.InitializationOptions
|
||||
import org.webrtc.SdpObserver
|
||||
import org.webrtc.SessionDescription
|
||||
import org.webrtc.VideoTrack
|
||||
|
||||
const val TAG = "RtcPeerConnection"
|
||||
|
||||
class RtcPeerConnection(private val context: Context) : PeerConnection.Observer {
|
||||
private val peerConnectionFactory by lazy { initializeFactory() }
|
||||
private val iceServers by lazy { initializeIceServers() }
|
||||
val cameraXCapturer by lazy { CameraXCapturer() }
|
||||
private val audioTrack by lazy { createAudioTrack() }
|
||||
private val videoTrack by lazy { createVideoTrack() }
|
||||
private var peerConnection: PeerConnection? = null
|
||||
|
||||
private fun initializeFactory(): PeerConnectionFactory {
|
||||
val initOptions = InitializationOptions.builder(context).createInitializationOptions()
|
||||
PeerConnectionFactory.initialize(initOptions)
|
||||
|
||||
val options = PeerConnectionFactory.Options()
|
||||
val peerConnectionFactory =
|
||||
PeerConnectionFactory.builder().setOptions(options).createPeerConnectionFactory()
|
||||
return peerConnectionFactory
|
||||
}
|
||||
|
||||
private fun initializeIceServers(): ArrayList<PeerConnection.IceServer> {
|
||||
val iceServers = ArrayList<PeerConnection.IceServer>()
|
||||
iceServers.add(
|
||||
PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer()
|
||||
)
|
||||
return iceServers
|
||||
}
|
||||
|
||||
private fun createAudioTrack(): AudioTrack {
|
||||
val audioConstraints = MediaConstraints()
|
||||
val audioSource = peerConnectionFactory.createAudioSource(audioConstraints)
|
||||
val localAudioTrack = peerConnectionFactory.createAudioTrack("audio_track", audioSource)
|
||||
return localAudioTrack
|
||||
}
|
||||
|
||||
private fun createVideoTrack(): VideoTrack {
|
||||
val videoSource = peerConnectionFactory.createVideoSource(false)
|
||||
cameraXCapturer.setCapturerObserver(videoSource.capturerObserver)
|
||||
val localVideoTrack = peerConnectionFactory.createVideoTrack("video_track", videoSource)
|
||||
localVideoTrack.setEnabled(true)
|
||||
return localVideoTrack
|
||||
}
|
||||
|
||||
fun connect() {
|
||||
Log.i(TAG, "Connecting rtc peer connection")
|
||||
var rtcConfig = PeerConnection.RTCConfiguration(iceServers)
|
||||
var peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, this)
|
||||
|
||||
var mediaConstraints = MediaConstraints()
|
||||
mediaConstraints.mandatory.add(
|
||||
MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")
|
||||
);
|
||||
mediaConstraints.mandatory.add(
|
||||
MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")
|
||||
);
|
||||
peerConnection?.addTrack(audioTrack)
|
||||
peerConnection?.addTrack(videoTrack)
|
||||
peerConnection?.createOffer(object : SdpObserver {
|
||||
override fun onCreateSuccess(sessionDescription: SessionDescription?) {
|
||||
Log.i(TAG, "onCreateSuccess: ${sessionDescription?.description}")
|
||||
}
|
||||
|
||||
override fun onSetSuccess() {
|
||||
Log.i(TAG, "onSetSuccess")
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
override fun onCreateFailure(p0: String?) {
|
||||
Log.i(TAG, "onCreateFailure: $p0")
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
override fun onSetFailure(p0: String?) {
|
||||
Log.i(TAG, "onSetFailure: $p0")
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
}, mediaConstraints)
|
||||
}
|
||||
|
||||
override fun onSignalingChange(p0: PeerConnection.SignalingState?) {
|
||||
Log.i(TAG, "onSignalingChange: $p0")
|
||||
}
|
||||
|
||||
override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) {
|
||||
Log.i(TAG, "onIceConnectionChange: $p0")
|
||||
}
|
||||
|
||||
override fun onIceConnectionReceivingChange(p0: Boolean) {
|
||||
Log.i(TAG, "onIceConnectionReceivingChange: $p0")
|
||||
}
|
||||
|
||||
override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) {
|
||||
Log.i(TAG, "onIceGatheringChange: $p0")
|
||||
}
|
||||
|
||||
override fun onIceCandidate(p0: IceCandidate?) {
|
||||
Log.i(TAG, "onIceCandidate: $p0")
|
||||
}
|
||||
|
||||
override fun onIceCandidatesRemoved(p0: Array<out IceCandidate?>?) {
|
||||
Log.i(TAG, "onIceCandidatesRemoved: $p0")
|
||||
}
|
||||
|
||||
override fun onAddStream(p0: MediaStream?) {
|
||||
Log.i(TAG, "onAddStream: $p0")
|
||||
}
|
||||
|
||||
override fun onRemoveStream(p0: MediaStream?) {
|
||||
Log.i(TAG, "onRemoveStream: $p0")
|
||||
}
|
||||
|
||||
override fun onDataChannel(p0: DataChannel?) {
|
||||
Log.i(TAG, "onDataChannel: $p0")
|
||||
}
|
||||
|
||||
override fun onRenegotiationNeeded() {
|
||||
Log.i(TAG, "onRenegotiationNeeded")
|
||||
}
|
||||
}
|
||||
@ -12,6 +12,7 @@ activityCompose = "1.10.1"
|
||||
composeBom = "2025.03.01"
|
||||
material3 = "1.4.0-alpha11"
|
||||
navigationCompose = "2.8.9"
|
||||
streamWebrtcAndroid = "1.3.8"
|
||||
|
||||
[libraries]
|
||||
androidx-camera-camera2 = { module = "androidx.camera:camera-camera2", version.ref = "cameraCore" }
|
||||
@ -40,6 +41,7 @@ ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" }
|
||||
ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" }
|
||||
ktor-client-websockets = { module = "io.ktor:ktor-client-websockets", version.ref = "ktor" }
|
||||
material3 = { module = "androidx.compose.material3:material3", version.ref = "material3" }
|
||||
stream-webrtc-android = { module = "io.getstream:stream-webrtc-android", version.ref = "streamWebrtcAndroid" }
|
||||
|
||||
[plugins]
|
||||
android-application = { id = "com.android.application", version.ref = "agp" }
|
||||
|
||||
Loading…
Reference in New Issue
Block a user